Add ultra-fast Manifest writing scripts

2 files changed, 199 insertions, 0 deletions

diff --git a/utils/gen_fast_manifest.py b/utils/gen_fast_manifest.py
new file mode 100755
index 0000000..74cd759
--- /dev/null
+++ b/utils/gen_fast_manifest.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+# Ultra-optimized Manifest writing.
+# (c) 2017 Michał Górny
+# Licensed under the terms of 2-clause BSD license
+
+import errno
+import glob
+import gzip
+import hashlib
+import io
+import os
+import os.path
+import sys
+
+import pyblake2
+
+
+def get_manifest_entry(t, path, relpath):
+    sha512 = hashlib.sha512()
+    blake2 = pyblake2.blake2b()
+
+    with io.open(path, 'rb') as f:
+        buf = f.read()
+        sha512.update(buf)
+        blake2.update(buf)
+        size = len(buf)
+
+    return ('{} {} {} BLAKE2B {} SHA512 {}'.format(t, relpath,
+            size, blake2.hexdigest(), sha512.hexdigest())).encode('utf8')
+
+
+def generate_manifest_entries(out, topdir):
+    compat_mode = False
+
+    for dirpath, dirs, files in os.walk(topdir):
+        if dirpath != topdir:
+            for f in files:
+                if f.startswith('Manifest'):
+                    fp = os.path.join(dirpath, f)
+                    out.append(get_manifest_entry('MANIFEST',
+                            fp, os.path.relpath(fp, topdir)))
+                    # do not descend
+                    dirs.clear()
+                    skip = True
+                    break
+            else:
+                skip = False
+            if skip:
+                continue
+        else:
+            # enable compat mode for ebuild directories
+            if any(f.endswith('.ebuild') for f in files):
+                compat_mode = True
+
+        # skip dot-dirs
+        dotdirs = [d for d in dirs if d.startswith('.')]
+        for d in dotdirs:
+            dirs.remove(d)
+
+        for f in files:
+            if f.startswith('Manifest') or f.startswith('.'):
+                continue
+            fp = os.path.join(dirpath, f)
+            ep = os.path.relpath(fp, topdir)
+            ftype = 'DATA'
+            if compat_mode:
+                if f.endswith('.ebuild') and f != 'skel.ebuild':
+                    ftype = 'EBUILD'
+                elif f == 'metadata.xml':
+                    ftype = 'MISC'
+                elif ep.startswith('files/'):
+                    ftype = 'AUX'
+                    ep = ep[6:]
+            else:
+                if f in ('timestamp', 'timestamp.chk', 'timestamp.commit',
+                        'timestamp.x'):
+                    continue
+
+            out.append(get_manifest_entry(ftype, fp, ep))
+
+    return compat_mode
+
+
+def gen_manifest(top_dir):
+    manifest_entries = []
+
+    # load DIST and IGNORE entries from existing Manifest
+    had_manifest = False
+    try:
+        with io.open(os.path.join(top_dir, 'Manifest'), 'rb') as f:
+            for l in f:
+                if l.startswith(b'DIST') or l.startswith(b'IGNORE'):
+                    manifest_entries.append(l.rstrip())
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    else:
+        had_manifest = True
+
+    # generate local file entries
+    compat_mode = generate_manifest_entries(manifest_entries, top_dir)
+    manifest_entries.sort()
+
+    manifest_data = b'\n'.join(manifest_entries) + b'\n'
+    if len(manifest_data) > 4096 and not compat_mode:
+        with gzip.GzipFile(os.path.join(top_dir, 'Manifest.gz'), 'wb') as f:
+            f.write(manifest_data)
+        if had_manifest:
+            os.unlink(os.path.join(top_dir, 'Manifest'))
+    else:
+        with io.open(os.path.join(top_dir, 'Manifest'), 'wb') as f:
+            f.write(manifest_data)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('Usage: {} <directory>...'.format(sys.argv[0]))
+        sys.exit(1)
+
+    for path in sys.argv[1:]:
+        gen_manifest(path)
diff --git a/utils/gen_fast_metamanifest.py b/utils/gen_fast_metamanifest.py
new file mode 100755
index 0000000..b95c08e
--- /dev/null
+++ b/utils/gen_fast_metamanifest.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# Ultra-optimized Meta-Manifest writing.
+# (c) 2017 Michał Górny
+# Licensed under the terms of 2-clause BSD license
+
+import datetime
+import glob
+import io
+import os
+import os.path
+import subprocess
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+import gen_fast_manifest
+
+
+def manifest_dir_generator():
+    with io.open('profiles/categories', 'r') as f:
+        categories = [x.strip() for x in f]
+
+    for c in categories:
+        # all package directories
+        for d in glob.glob(os.path.join(c, '*/')):
+            yield d
+        # category directory
+        yield c
+        # md5-cache for the category
+        yield os.path.join('metadata/md5-cache', c)
+
+    # few special metadata directories
+    yield 'metadata/glsa'
+    yield 'metadata/md5-cache'
+    yield 'metadata/news'
+
+    # top-level dirs
+    yield 'metadata'
+    yield 'eclass'
+    yield 'licenses'
+    yield 'profiles'
+
+    # finally, the whole repo
+    yield '.'
+
+
+def gen_metamanifest(top_dir):
+    os.chdir(top_dir)
+    alldirs = manifest_dir_generator()
+
+    # pre-populate IGNORE entries
+    with io.open('metadata/Manifest', 'wb') as f:
+        f.write(b'''IGNORE timestamp
+IGNORE timestamp.chk
+IGNORE timestamp.commit
+IGNORE timestamp.x
+''')
+    with io.open('Manifest', 'wb') as f:
+        f.write(b'''IGNORE distfiles
+IGNORE local
+IGNORE packages
+''')
+
+    # call the fast-gen routine
+    for path in alldirs:
+        gen_fast_manifest.gen_manifest(path)
+
+    # write timestamp
+    with io.open('Manifest', 'ab') as f:
+        f.write(datetime.datetime.utcnow().strftime('TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n').encode('ascii'))
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print('Usage: {} <top-directory>'.format(sys.argv[0]))
+        sys.exit(1)
+
+    gen_metamanifest(sys.argv[1])