diff options
| -rw-r--r-- | gemato/recursiveloader.py | 144 | ||||
| -rw-r--r-- | tests/test_recursiveloader.py | 73 |
2 files changed, 156 insertions, 61 deletions
diff --git a/gemato/recursiveloader.py b/gemato/recursiveloader.py index f403b7a..e5e9fdc 100644 --- a/gemato/recursiveloader.py +++ b/gemato/recursiveloader.py @@ -56,6 +56,7 @@ class ManifestRecursiveLoader(object): # TODO: allow catching OpenPGP exceptions somehow? m = self.load_manifest(os.path.basename(top_manifest_path)) self.openpgp_signed = m.openpgp_signed + self.updated_manifests = set() def load_manifest(self, relpath, verify_entry=None): """ @@ -365,11 +366,64 @@ class ManifestRecursiveLoader(object): return ret + def save_manifests(self, hashes=None): + """ + Save the Manifests modified since the last save_manifests() + call. + + @hashes specifies the requested hash set. If specified, + it overrides the hash set used in Manifest. If None, the set + specified in ManifestLoader constructor is used. If that one + is None as well, the routine reuses the existing hash set + in the entry. + """ + + if hashes is None: + hashes = self.hashes + + fixed_manifests = set() + for mpath, relpath, m in self._iter_manifests_for_path('', + recursive=True): + for e in m.entries: + if not isinstance(e, gemato.manifest + .ManifestEntryMANIFEST): + continue + + fullpath = os.path.join(relpath, e.path) + if fullpath not in self.updated_manifests: + continue + + gemato.verify.update_entry_for_path( + os.path.join(self.root_directory, fullpath), + e, + hashes=hashes, + expected_dev=self.manifest_device) + + # do not remove it from self.updated_manifests + # immediately as we may have to deal with multiple + # entries + fixed_manifests.add(fullpath) + self.updated_manifests.add(mpath) + + # we've apparently modified this Manifest, so store it now + if mpath in self.updated_manifests: + self.save_manifest(mpath) + + # now, discard all the Manifests whose entries we've updated + self.updated_manifests -= fixed_manifests + # ...and top-level Manifest which has no entries + self.updated_manifests -= set(gemato.compression + .get_potential_compressed_names('Manifest')) + # at this point, the list should be empty + assert not self.updated_manifests + def update_entry_for_path(self, path, new_entry_type='DATA', hashes=None): """ - Update the Manifest entries for @path and the appropriate - MANIFEST entries. @path must not be covered by IGNORE. + Update the Manifest entries for @path and queue the containing + Manifests for update. @path must not be covered by IGNORE. + You need to invoke save_manifests() to store the Manifest + updates afterwards. If the path exists and has a matching Manifest entry, the most specific existing entry will be updated. If the path has more @@ -399,7 +453,6 @@ class ManifestRecursiveLoader(object): """ had_entry = False - manifests_to_update = set() if hashes is None: hashes = self.hashes @@ -424,15 +477,12 @@ class ManifestRecursiveLoader(object): # we update either file at the specified path # or any relevant Manifests fullpath = os.path.join(relpath, e.path) - if fullpath == path: - if had_entry: - # duplicate entry! - entries_to_remove.append(e) - continue - # pass through - elif fullpath in manifests_to_update: - pass - else: + if fullpath != path: + continue + + if had_entry: + # duplicate entry! + entries_to_remove.append(e) continue try: @@ -451,61 +501,35 @@ class ManifestRecursiveLoader(object): else: raise err else: - manifests_to_update.add(mpath) + self.updated_manifests.add(mpath) had_entry = True if entries_to_remove: for e in entries_to_remove: m.entries.remove(e) - manifests_to_update.add(mpath) - - # we've apparently added this Manifest, so store it now - if mpath in manifests_to_update: - self.save_manifest(mpath) + self.updated_manifests.add(mpath) if not had_entry: assert hashes is not None for mpath, relpath, m in self._iter_manifests_for_path(path): # add to the first relevant Manifest - if not had_entry: - assert new_entry_type not in ( - 'DIST', 'IGNORE', 'OPTIONAL') - newpath = os.path.relpath(path, relpath) - if new_entry_type == 'AUX': - # AUX has implicit files/ prefix - assert gemato.util.path_inside_dir(newpath, - 'files') - # drop files/ prefix - newpath = os.path.relpath(newpath, 'files') - e = gemato.manifest.new_manifest_entry( - new_entry_type, newpath, 0, {}) - gemato.verify.update_entry_for_path( - os.path.join(self.root_directory, path), - e, - hashes=hashes, - expected_dev=self.manifest_device) - m.entries.append(e) - manifests_to_update.add(mpath) - had_entry = True - else: - for e in m.entries: - if not isinstance(e, gemato.manifest.ManifestEntryMANIFEST): - continue - - # we update either file at the specified path - # or any relevant Manifests - fullpath = os.path.join(relpath, e.path) - if fullpath not in manifests_to_update: - continue - - gemato.verify.update_entry_for_path( - os.path.join(self.root_directory, - fullpath), - e, - hashes=hashes, - expected_dev=self.manifest_device) - manifests_to_update.add(mpath) - - # we've apparently added this Manifest, so store it now - if mpath in manifests_to_update: - self.save_manifest(mpath) + assert new_entry_type not in ( + 'DIST', 'IGNORE', 'OPTIONAL') + newpath = os.path.relpath(path, relpath) + if new_entry_type == 'AUX': + # AUX has implicit files/ prefix + assert gemato.util.path_inside_dir(newpath, + 'files') + # drop files/ prefix + newpath = os.path.relpath(newpath, 'files') + e = gemato.manifest.new_manifest_entry( + new_entry_type, newpath, 0, {}) + gemato.verify.update_entry_for_path( + os.path.join(self.root_directory, path), + e, + hashes=hashes, + expected_dev=self.manifest_device) + m.entries.append(e) + self.updated_manifests.add(mpath) + had_entry = True + break diff --git a/tests/test_recursiveloader.py b/tests/test_recursiveloader.py index d017ef1..a8b7384 100644 --- a/tests/test_recursiveloader.py +++ b/tests/test_recursiveloader.py @@ -301,6 +301,7 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e m.update_entry_for_path('sub/stray', hashes=['SHA256', 'SHA512']) self.assertIsInstance(m.find_path_entry('sub/stray'), gemato.manifest.ManifestEntryDATA) + m.save_manifests() # relevant Manifests should have been updated with io.open(os.path.join(self.dir, 'sub/Manifest'), 'r', encoding='utf8') as f: @@ -317,6 +318,7 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e new_entry_type='MANIFEST') self.assertIsInstance(m.find_path_entry('sub/stray'), gemato.manifest.ManifestEntryMANIFEST) + m.save_manifests() # relevant Manifests should have been updated with io.open(os.path.join(self.dir, 'sub/Manifest'), 'r', encoding='utf8') as f: @@ -334,6 +336,7 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e new_entry_type='MISC') self.assertIsInstance(m.find_path_entry('sub/stray'), gemato.manifest.ManifestEntryMISC) + m.save_manifests() # relevant Manifests should have been updated with io.open(os.path.join(self.dir, 'sub/Manifest'), 'r', encoding='utf8') as f: @@ -350,6 +353,7 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e new_entry_type='EBUILD') self.assertIsInstance(m.find_path_entry('sub/stray'), gemato.manifest.ManifestEntryEBUILD) + m.save_manifests() # relevant Manifests should have been updated with io.open(os.path.join(self.dir, 'sub/Manifest'), 'r', encoding='utf8') as f: @@ -381,6 +385,7 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e self.assertListEqual( sorted(m.find_path_entry('sub/stray').checksums), ['SHA256', 'SHA512']) + m.save_manifests() # relevant Manifests should have been updated with io.open(os.path.join(self.dir, 'sub/Manifest'), 'r', encoding='utf8') as f: @@ -398,6 +403,7 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e self.assertListEqual( sorted(m.find_path_entry('sub/stray').checksums), ['MD5']) + m.save_manifests() # relevant Manifests should have been updated with io.open(os.path.join(self.dir, 'sub/Manifest'), 'r', encoding='utf8') as f: @@ -407,6 +413,25 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e self.assertNotEqual(f.read(), self.FILES['Manifest']) m.assert_directory_verifies() + def test_update_entry_for_path_discard(self): + """ + Test that files are not modified if save_manifests() + is not called. + """ + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest')) + m.update_entry_for_path('sub/stray', hashes=['SHA256', 'SHA512']) + self.assertIsInstance(m.find_path_entry('sub/stray'), + gemato.manifest.ManifestEntryDATA) + del m + # relevant Manifests should not have been touched + with io.open(os.path.join(self.dir, 'sub/Manifest'), + 'r', encoding='utf8') as f: + self.assertEqual(f.read(), self.FILES['sub/Manifest']) + with io.open(os.path.join(self.dir, 'Manifest'), + 'r', encoding='utf8') as f: + self.assertEqual(f.read(), self.FILES['Manifest']) + class MultipleManifestTest(TempDirTestCase): DIRS = ['sub'] @@ -459,6 +484,7 @@ TIMESTAMP 2017-01-01T01:01:01Z m = gemato.recursiveloader.ManifestRecursiveLoader( os.path.join(self.dir, 'Manifest')) m.update_entry_for_path('sub/foo') + m.save_manifests() # relevant Manifests should have been updated # but sub/Manifest.b should be left intact with io.open(os.path.join(self.dir, 'sub/Manifest.a'), @@ -480,10 +506,45 @@ TIMESTAMP 2017-01-01T01:01:01Z self.assertListEqual( sorted(m.find_path_entry('sub/foo').checksums), ['SHA256', 'SHA512']) + m.save_manifests() + self.assertListEqual( + sorted(m.find_path_entry('sub/Manifest.a').checksums), + ['MD5']) + self.assertListEqual( + sorted(m.find_path_entry('sub/Manifest.b').checksums), + ['MD5']) + # relevant Manifests should have been updated + # but sub/Manifest.b should be left intact + with io.open(os.path.join(self.dir, 'sub/Manifest.a'), + 'r', encoding='utf8') as f: + self.assertNotEqual(f.read(), self.FILES['sub/Manifest.a']) + with io.open(os.path.join(self.dir, 'sub/Manifest.b'), + 'r', encoding='utf8') as f: + self.assertEqual(f.read(), self.FILES['sub/Manifest.b']) + with io.open(os.path.join(self.dir, 'Manifest'), + 'r', encoding='utf8') as f: + self.assertNotEqual(f.read(), self.FILES['Manifest']) + m.assert_directory_verifies() + + def test_update_entry_for_path_hashes_plus_manifest(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest')) + m.update_entry_for_path('sub/foo', hashes=['SHA256', 'SHA512']) + # check for checksums + self.assertListEqual( + sorted(m.find_path_entry('sub/foo').checksums), + ['SHA256', 'SHA512']) self.assertListEqual( sorted(m.find_path_entry('sub/Manifest.a').checksums), + ['MD5']) + m.save_manifests(hashes=['SHA1']) + self.assertListEqual( + sorted(m.find_path_entry('sub/foo').checksums), ['SHA256', 'SHA512']) self.assertListEqual( + sorted(m.find_path_entry('sub/Manifest.a').checksums), + ['SHA1']) + self.assertListEqual( sorted(m.find_path_entry('sub/Manifest.b').checksums), ['MD5']) # relevant Manifests should have been updated @@ -510,6 +571,10 @@ TIMESTAMP 2017-01-01T01:01:01Z ['SHA256', 'SHA512']) self.assertListEqual( sorted(m.find_path_entry('sub/Manifest.a').checksums), + ['MD5']) + m.save_manifests() + self.assertListEqual( + sorted(m.find_path_entry('sub/Manifest.a').checksums), ['SHA256', 'SHA512']) self.assertListEqual( sorted(m.find_path_entry('sub/Manifest.b').checksums), @@ -536,9 +601,10 @@ TIMESTAMP 2017-01-01T01:01:01Z self.assertListEqual( sorted(m.find_path_entry('sub/foo').checksums), ['SHA1']) + m.save_manifests() self.assertListEqual( sorted(m.find_path_entry('sub/Manifest.a').checksums), - ['SHA1']) + ['SHA256', 'SHA512']) self.assertListEqual( sorted(m.find_path_entry('sub/Manifest.b').checksums), ['MD5']) @@ -767,6 +833,7 @@ AUX test.patch 0 MD5 d41d8cd98f00b204e9800998ecf8427e os.path.join(self.dir, 'Manifest')) m.update_entry_for_path('files/test.patch') self.assertIsNone(m.find_path_entry('files/test.patch')) + m.save_manifests() with io.open(os.path.join(self.dir, 'Manifest'), 'r', encoding='utf8') as f: self.assertNotEqual(f.read(), self.FILES['Manifest']) @@ -793,6 +860,7 @@ class AUXTypeFileAdditionTest(TempDirTestCase): hashes=['MD5'], new_entry_type='AUX') self.assertIsInstance(m.find_path_entry('files/test.txt'), gemato.manifest.ManifestEntryAUX) + m.save_manifests() with io.open(os.path.join(self.dir, 'Manifest'), 'r', encoding='utf8') as f: self.assertNotEqual(f.read(), self.FILES['Manifest']) @@ -858,6 +926,7 @@ DATA test 0 SHA1 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 self.assertIn( tuple(m.find_path_entry('test').checksums), (('MD5',), ('SHA1',))) + m.save_manifests() with io.open(os.path.join(self.dir, 'Manifest'), 'r', encoding='utf8') as f: self.assertNotEqual(f.read(), self.FILES['Manifest']) @@ -1083,6 +1152,7 @@ MISC foo 0 MD5 d41d8cd98f00b204e9800998ecf8427e os.path.join(self.dir, 'Manifest')) m.update_entry_for_path('foo') self.assertIsNone(m.find_path_entry('foo')) + m.save_manifests() with io.open(os.path.join(self.dir, 'Manifest'), 'r', encoding='utf8') as f: self.assertNotEqual(f.read(), self.FILES['Manifest']) @@ -1134,6 +1204,7 @@ OPTIONAL foo os.path.join(self.dir, 'Manifest')) m.update_entry_for_path('foo') self.assertIsNotNone(m.find_path_entry('foo')) + m.save_manifests() with io.open(os.path.join(self.dir, 'Manifest'), 'r', encoding='utf8') as f: self.assertEqual(f.read(), self.FILES['Manifest']) |