diff options
| -rwxr-xr-x | utils/gen_fast_metamanifest.py | 37 |
1 files changed, 18 insertions, 19 deletions
diff --git a/utils/gen_fast_metamanifest.py b/utils/gen_fast_metamanifest.py index bd26c09..c5272b4 100755 --- a/utils/gen_fast_metamanifest.py +++ b/utils/gen_fast_metamanifest.py @@ -72,25 +72,24 @@ IGNORE lost+found IGNORE packages ''') - p = multiprocessing.Pool() - - # generate 1st batch of sub-Manifests - # expecting 20000+ items, so use iterator with a reasonably large - # chunksize - p.map(gen_fast_manifest.gen_manifest, manifest_dir_generator(1), chunksize=64) - - # timestamp into tier 1 directories - ts = datetime.datetime.utcnow().strftime( - 'TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n').encode('ascii') - with io.open('metadata/glsa/Manifest', 'ab') as f: - f.write(ts) - with io.open('metadata/news/Manifest', 'ab') as f: - f.write(ts) - - # 2nd batch (files depending on results of 1st batch) - # this one is fast to generate, so let's pass a list and let map() - # choose optimal chunksize - p.map(gen_fast_manifest.gen_manifest, list(manifest_dir_generator(2))) + with multiprocessing.Pool() as p: + # generate 1st batch of sub-Manifests + # expecting 20000+ items, so use iterator with a reasonably large + # chunksize + p.map(gen_fast_manifest.gen_manifest, manifest_dir_generator(1), chunksize=64) + + # timestamp into tier 1 directories + ts = datetime.datetime.utcnow().strftime( + 'TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n').encode('ascii') + with io.open('metadata/glsa/Manifest', 'ab') as f: + f.write(ts) + with io.open('metadata/news/Manifest', 'ab') as f: + f.write(ts) + + # 2nd batch (files depending on results of 1st batch) + # this one is fast to generate, so let's pass a list and let map() + # choose optimal chunksize + p.map(gen_fast_manifest.gen_manifest, list(manifest_dir_generator(2))) # finally, generate the top-level Manifest gen_fast_manifest.gen_manifest('.') |