diff options
author | Michał Górny <mgorny@gentoo.org> | 2017-11-24 23:57:13 +0100 |
---|---|---|
committer | Michał Górny <mgorny@gentoo.org> | 2017-11-24 23:57:13 +0100 |
commit | 89cfd10d1a88d95e6f28270f60980b459d06b71a (patch) | |
tree | 27af0ef949a9feed80b792b636c25b12a31b08b9 /utils/gen_fast_metamanifest.py | |
parent | 9c698f7dd2ff18db72de3415c1fb6e08e048f952 (diff) | |
download | gemato-89cfd10d1a88d95e6f28270f60980b459d06b71a.tar.gz |
gen_fast_metamanifest: Use context manager for Pool
Diffstat (limited to 'utils/gen_fast_metamanifest.py')
-rwxr-xr-x | utils/gen_fast_metamanifest.py | 37 |
1 files changed, 18 insertions, 19 deletions
diff --git a/utils/gen_fast_metamanifest.py b/utils/gen_fast_metamanifest.py index bd26c09..c5272b4 100755 --- a/utils/gen_fast_metamanifest.py +++ b/utils/gen_fast_metamanifest.py @@ -72,25 +72,24 @@ IGNORE lost+found IGNORE packages ''') - p = multiprocessing.Pool() - - # generate 1st batch of sub-Manifests - # expecting 20000+ items, so use iterator with a reasonably large - # chunksize - p.map(gen_fast_manifest.gen_manifest, manifest_dir_generator(1), chunksize=64) - - # timestamp into tier 1 directories - ts = datetime.datetime.utcnow().strftime( - 'TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n').encode('ascii') - with io.open('metadata/glsa/Manifest', 'ab') as f: - f.write(ts) - with io.open('metadata/news/Manifest', 'ab') as f: - f.write(ts) - - # 2nd batch (files depending on results of 1st batch) - # this one is fast to generate, so let's pass a list and let map() - # choose optimal chunksize - p.map(gen_fast_manifest.gen_manifest, list(manifest_dir_generator(2))) + with multiprocessing.Pool() as p: + # generate 1st batch of sub-Manifests + # expecting 20000+ items, so use iterator with a reasonably large + # chunksize + p.map(gen_fast_manifest.gen_manifest, manifest_dir_generator(1), chunksize=64) + + # timestamp into tier 1 directories + ts = datetime.datetime.utcnow().strftime( + 'TIMESTAMP %Y-%m-%dT%H:%M:%SZ\n').encode('ascii') + with io.open('metadata/glsa/Manifest', 'ab') as f: + f.write(ts) + with io.open('metadata/news/Manifest', 'ab') as f: + f.write(ts) + + # 2nd batch (files depending on results of 1st batch) + # this one is fast to generate, so let's pass a list and let map() + # choose optimal chunksize + p.map(gen_fast_manifest.gen_manifest, list(manifest_dir_generator(2))) # finally, generate the top-level Manifest gen_fast_manifest.gen_manifest('.') |