gen_fast_*: Split top-level Manifests to save bandwidth

Split the potential-top-level Manifests into separate
'Manifest.files[.gz]' and 'Manifest' with the former containing large
set of (possibly unchanged) files and the latter referencing the former
and containing the timestamp.

2 files changed, 21 insertions, 14 deletions

diff --git a/utils/gen_fast_manifest.py b/utils/gen_fast_manifest.py
index 049761f..9d58e46 100755
--- a/utils/gen_fast_manifest.py
+++ b/utils/gen_fast_manifest.py
@@ -36,7 +36,7 @@ def generate_manifest_entries(out, topdir):
     for dirpath, dirs, files in os.walk(topdir):
         if dirpath != topdir:
             for f in files:
-                if f.startswith('Manifest'):
+                if f in ('Manifest', 'Manifest.gz'):
                     fp = os.path.join(dirpath, f)
                     out.append(get_manifest_entry('MANIFEST',
                             fp, os.path.relpath(fp, topdir)))
@@ -50,7 +50,7 @@ def generate_manifest_entries(out, topdir):
                 continue
         else:
             # enable compat mode for ebuild directories
-            if any(f.endswith('.ebuild') for f in files):
+            if any(f.endswith('.ebuild') and f != 'skel.ebuild' for f in files):
                 compat_mode = True
 
         # skip dot-dirs
@@ -101,10 +101,6 @@ def gen_manifest(top_dir):
     compat_mode = generate_manifest_entries(manifest_entries, top_dir)
     manifest_entries.sort()
 
-    # do not compress files which we want valid top-level Manifests
-    if top_dir.endswith('metadata/glsa') or top_dir.endswith('metadata/news'):
-        compat_mode = True
-
     manifest_data = b'\n'.join(manifest_entries) + b'\n'
     if len(manifest_data) > 4096 and not compat_mode:
         with gzip.GzipFile(os.path.join(top_dir, 'Manifest.gz'), 'wb', mtime=0) as f:
diff --git a/utils/gen_fast_metamanifest.py b/utils/gen_fast_metamanifest.py
index bd26c09..fe5d615 100755
--- a/utils/gen_fast_metamanifest.py
+++ b/utils/gen_fast_metamanifest.py
@@ -50,6 +50,20 @@ def manifest_dir_generator(iter_n):
         yield 'metadata'
 
 
+def make_toplevel(d, ts):
+    for suffix in ('.gz', ''):
+        src = os.path.join(d, 'Manifest' + suffix)
+        if os.path.exists(src):
+            dstsplit = os.path.join(d, 'Manifest.files' + suffix)
+            dsttop = os.path.join(d, 'Manifest')
+            os.rename(src, dstsplit)
+            with io.open(dsttop, 'wb') as f:
+                me = gen_fast_manifest.get_manifest_entry('MANIFEST',
+                        dstsplit, 'Manifest.files' + suffix)
+                f.write(me + b'\n' + ts)
+            break
+
+
 def gen_metamanifest(top_dir):
     os.chdir(top_dir)
 
@@ -79,13 +93,11 @@ IGNORE packages
     # chunksize
     p.map(gen_fast_manifest.gen_manifest, manifest_dir_generator(1), chunksize=64)
 
-    # timestamp into tier 1 directories
+    # special directories: we split Manifest there, and add timestamp
     ts = datetime.datetime.utcnow().strftime(
             'TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n').encode('ascii')
-    with io.open('metadata/glsa/Manifest', 'ab') as f:
-        f.write(ts)
-    with io.open('metadata/news/Manifest', 'ab') as f:
-        f.write(ts)
+    make_toplevel('metadata/glsa', ts)
+    make_toplevel('metadata/news', ts)
 
     # 2nd batch (files depending on results of 1st batch)
     # this one is fast to generate, so let's pass a list and let map()
@@ -95,11 +107,10 @@ IGNORE packages
     # finally, generate the top-level Manifest
     gen_fast_manifest.gen_manifest('.')
 
-    # final timestamp
+    # final split
     ts = datetime.datetime.utcnow().strftime(
             'TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n').encode('ascii')
-    with io.open('Manifest', 'ab') as f:
-        f.write(ts)
+    make_toplevel('', ts)
 
 
 if __name__ == '__main__':