diff options
| -rw-r--r-- | gemato/recursiveloader.py | 121 | ||||
| -rw-r--r-- | tests/test_recursiveloader.py | 182 |
2 files changed, 302 insertions, 1 deletions
diff --git a/gemato/recursiveloader.py b/gemato/recursiveloader.py index 99ca631..59ce631 100644 --- a/gemato/recursiveloader.py +++ b/gemato/recursiveloader.py @@ -587,3 +587,124 @@ class ManifestRecursiveLoader(object): self.updated_manifests.add(mpath) return out + + def update_entries_for_directory(self, path='', hashes=None): + """ + Update the Manifest entries for the contents of directory + @path (top directory by default), recursively. Includes adding + new files and removing entries for those that no longer exist. + The behavior for various cases is the same + as for update_entry_for_path() except as noted below. + + New entries are currently created with DATA type. This will + be extended in the future. + + @hashes specifies the requested hash set. If specified, + it overrides the hash set used in Manifest. If None, the set + specified in ManifestLoader constructor is used. Either + of the two hash sets must be specified. + + If the top-level Manifest has a TIMESTAMP entry, this entry + will be updated with the current time after successful update. + """ + + if hashes is None: + hashes = self.hashes + assert hashes is not None + + entry_dict = self.get_deduplicated_file_entry_dict_for_update( + path) + it = os.walk(os.path.join(self.root_directory, path), + onerror=gemato.util.throw_exception, + followlinks=True) + + for dirpath, dirnames, filenames in it: + relpath = os.path.relpath(dirpath, self.root_directory) + # strip dot to avoid matching problems + if relpath == '.': + relpath = '' + + skip_dirs = [] + for d in dirnames: + # skip dotfiles + if d.startswith('.'): + skip_dirs.append(d) + continue + + dpath = os.path.join(relpath, d) + mpath, de = entry_dict.pop(dpath, (None, None)) + if de is None: + syspath = os.path.join(dirpath, d) + st = os.stat(syspath) + if st.st_dev != self.manifest_device: + raise gemato.exceptions.ManifestCrossDevice(syspath) + continue + + if isinstance(de, gemato.manifest.ManifestEntryIGNORE): + skip_dirs.append(d) + else: + # trigger the exception indirectly + gemato.verify.update_entry_for_path( + os.path.join(dirpath, d), + de, + hashes=hashes, + expected_dev=self.manifest_device) + assert False, "exception should have been raised" + + # skip scanning ignored directories + for d in skip_dirs: + dirnames.remove(d) + + dir_manifest = None + + for f in filenames: + # skip dotfiles + if f.startswith('.'): + continue + + fpath = os.path.join(relpath, f) + # skip top-level Manifest, we obviously can't have + # an entry for it + if fpath in (gemato.compression + .get_potential_compressed_names('Manifest')): + continue + mpath, fe = entry_dict.pop(fpath, (None, None)) + if fe is not None: + if isinstance(fe, gemato.manifest.ManifestEntryIGNORE): + continue + elif isinstance(fe, gemato.manifest.ManifestEntryOPTIONAL): + continue + else: + # find appropriate Manifest for this directory + if dir_manifest is None: + for mpath, relpath, m in (self + ._iter_manifests_for_path(fpath)): + dir_manifest = (mpath, relpath, m) + break + else: + mpath, relpath, m = dir_manifest + + fe = gemato.manifest.ManifestEntryDATA( + os.path.relpath(fpath, relpath), + 0, {}) + m.entries.append(fe) + + # update the existing entry + changed = gemato.verify.update_entry_for_path( + os.path.join(dirpath, f), + fe, + hashes=hashes, + expected_dev=self.manifest_device) + if changed: + self.updated_manifests.add(mpath) + + # check for removed files + for relpath, me in entry_dict.items(): + mpath, fe = me + if isinstance(fe, gemato.manifest.ManifestEntryIGNORE): + continue + elif isinstance(fe, gemato.manifest.ManifestEntryOPTIONAL): + continue + + self.loaded_manifests[mpath].entries.remove(fe) + self.updated_manifests.add(mpath) diff --git a/tests/test_recursiveloader.py b/tests/test_recursiveloader.py index dc0bd41..498498b 100644 --- a/tests/test_recursiveloader.py +++ b/tests/test_recursiveloader.py @@ -485,6 +485,22 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e 'r', encoding='utf8') as f: self.assertEqual(f.read(), self.FILES['Manifest']) + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest')) + m.update_entries_for_directory('', hashes=['SHA256', 'SHA512']) + self.assertIsInstance(m.find_path_entry('sub/stray'), + gemato.manifest.ManifestEntryDATA) + m.save_manifests() + # relevant Manifests should have been updated + with io.open(os.path.join(self.dir, 'sub/Manifest'), + 'r', encoding='utf8') as f: + self.assertNotEqual(f.read(), self.FILES['sub/Manifest']) + with io.open(os.path.join(self.dir, 'Manifest'), + 'r', encoding='utf8') as f: + self.assertNotEqual(f.read(), self.FILES['Manifest']) + m.assert_directory_verifies() + class MultipleManifestTest(TempDirTestCase): DIRS = ['sub'] @@ -674,6 +690,35 @@ TIMESTAMP 2017-01-01T01:01:01Z self.assertNotEqual(f.read(), self.FILES['Manifest']) m.assert_directory_verifies() + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + m.update_entries_for_directory('') + # check for checksums + self.assertListEqual( + sorted(m.find_path_entry('sub/foo').checksums), + ['SHA256', 'SHA512']) + m.save_manifests() + self.assertListEqual( + sorted(m.find_path_entry('sub/Manifest.a').checksums), + ['SHA256', 'SHA512']) + self.assertListEqual( + sorted(m.find_path_entry('sub/Manifest.b').checksums), + ['SHA256', 'SHA512']) + # relevant Manifests should have been updated + # but sub/Manifest.b should be left intact + with io.open(os.path.join(self.dir, 'sub/Manifest.a'), + 'r', encoding='utf8') as f: + self.assertNotEqual(f.read(), self.FILES['sub/Manifest.a']) + with io.open(os.path.join(self.dir, 'sub/Manifest.b'), + 'r', encoding='utf8') as f: + self.assertEqual(f.read(), self.FILES['sub/Manifest.b']) + with io.open(os.path.join(self.dir, 'Manifest'), + 'r', encoding='utf8') as f: + self.assertNotEqual(f.read(), self.FILES['Manifest']) + m.assert_directory_verifies() + class MultipleTopLevelManifestTest(TempDirTestCase): FILES = { @@ -963,6 +1008,18 @@ AUX test.patch 0 MD5 d41d8cd98f00b204e9800998ecf8427e self.assertRaises(gemato.exceptions.ManifestInvalidPath, m.update_entry_for_path, 'test.patch', hashes=['MD5']) + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + m.update_entries_for_directory('') + self.assertIsNone(m.find_path_entry('files/test.patch')) + m.save_manifests() + with io.open(os.path.join(self.dir, 'Manifest'), + 'r', encoding='utf8') as f: + self.assertNotEqual(f.read(), self.FILES['Manifest']) + m.assert_directory_verifies() + class AUXTypeFileAdditionTest(TempDirTestCase): DIRS = ['files'] @@ -1067,6 +1124,22 @@ DATA test 0 SHA1 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 self.assertNotEqual(f.read(), self.FILES['Manifest']) m.assert_directory_verifies() + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + m.update_entries_for_directory('') + self.assertEqual(m.find_path_entry('test').checksums, + { + 'SHA256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'SHA512': 'cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e', + }) + m.save_manifests() + with io.open(os.path.join(self.dir, 'Manifest'), + 'r', encoding='utf8') as f: + self.assertNotEqual(f.read(), self.FILES['Manifest']) + m.assert_directory_verifies() + class DuplicateIncompatibleDataMiscTypeFileEntryTest(TempDirTestCase): """ @@ -1348,6 +1421,18 @@ MISC foo 0 MD5 d41d8cd98f00b204e9800998ecf8427e self.assertNotEqual(f.read(), self.FILES['Manifest']) m.assert_directory_verifies() + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + m.update_entries_for_directory('') + self.assertIsNone(m.find_path_entry('foo')) + m.save_manifests() + with io.open(os.path.join(self.dir, 'Manifest'), + 'r', encoding='utf8') as f: + self.assertNotEqual(f.read(), self.FILES['Manifest']) + m.assert_directory_verifies() + class ManifestOptionalEntryTest(TempDirTestCase): """ @@ -1393,7 +1478,22 @@ OPTIONAL foo m = gemato.recursiveloader.ManifestRecursiveLoader( os.path.join(self.dir, 'Manifest')) m.update_entry_for_path('foo') - self.assertIsNotNone(m.find_path_entry('foo')) + self.assertIsInstance(m.find_path_entry('foo'), + gemato.manifest.ManifestEntryOPTIONAL) + m.save_manifests() + with io.open(os.path.join(self.dir, 'Manifest'), + 'r', encoding='utf8') as f: + self.assertEqual(f.read(), self.FILES['Manifest']) + self.assertRaises(gemato.exceptions.ManifestMismatch, + m.assert_directory_verifies, '') + + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + m.update_entries_for_directory('') + self.assertIsInstance(m.find_path_entry('foo'), + gemato.manifest.ManifestEntryOPTIONAL) m.save_manifests() with io.open(os.path.join(self.dir, 'Manifest'), 'r', encoding='utf8') as f: @@ -1445,6 +1545,13 @@ DATA sub/version 0 MD5 d41d8cd98f00b204e9800998ecf8427e gemato.cli.main(['gemato', 'verify', '--no-strict', self.dir]), 1) + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + self.assertRaises(gemato.exceptions.ManifestCrossDevice, + m.update_entries_for_directory, '') + class CrossDeviceEmptyManifestTest(TempDirTestCase): """ @@ -1488,6 +1595,13 @@ class CrossDeviceEmptyManifestTest(TempDirTestCase): gemato.cli.main(['gemato', 'verify', '--no-strict', self.dir]), 1) + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + self.assertRaises(gemato.exceptions.ManifestCrossDevice, + m.update_entries_for_directory, '') + class CrossDeviceIgnoreManifestTest(TempDirTestCase): """ @@ -1519,6 +1633,13 @@ IGNORE sub gemato.cli.main(['gemato', 'verify', self.dir]), 0) + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + m.update_entries_for_directory('') + self.assertEqual(len(m.loaded_manifests['Manifest'].entries), 1) + class DotfileManifestTest(TempDirTestCase): """ @@ -1542,6 +1663,19 @@ class DotfileManifestTest(TempDirTestCase): gemato.cli.main(['gemato', 'verify', self.dir]), 0) + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + m.update_entries_for_directory('') + self.assertIsNone(m.find_path_entry('.bar/baz')) + self.assertIsNone(m.find_path_entry('.foo')) + m.save_manifests() + with io.open(os.path.join(self.dir, 'Manifest'), + 'r', encoding='utf8') as f: + self.assertEqual(f.read(), self.FILES['Manifest']) + m.assert_directory_verifies() + class DirectoryInPlaceOfFileManifestTest(TempDirTestCase): """ @@ -1572,6 +1706,13 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e self.assertRaises(gemato.exceptions.ManifestInvalidPath, m.update_entry_for_path, 'test') + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + self.assertRaises(gemato.exceptions.ManifestInvalidPath, + m.update_entries_for_directory, '') + class UnreadableDirectoryTest(TempDirTestCase): """ @@ -1592,6 +1733,12 @@ class UnreadableDirectoryTest(TempDirTestCase): os.path.join(self.dir, 'Manifest')) self.assertRaises(OSError, m.assert_directory_verifies) + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + self.assertRaises(OSError, m.update_entries_for_directory, '') + class CompressedTopManifestTest(TempDirTestCase): """ @@ -1638,6 +1785,22 @@ DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e os.path.join(self.dir, 'Manifest.gz'), 'rb') as f: self.assertEqual(f.read(), self.MANIFEST.lstrip()) + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest.gz'), + hashes=['SHA256', 'SHA512']) + m.update_entries_for_directory('') + self.assertEqual(m.find_path_entry('test').checksums, + { + 'SHA256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'SHA512': 'cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e', + }) + m.save_manifests() + with gemato.compression.open_potentially_compressed_path( + os.path.join(self.dir, 'Manifest.gz'), 'rb') as f: + self.assertNotEqual(f.read(), self.MANIFEST.lstrip()) + m.assert_directory_verifies() + class CompressedSubManifestTest(TempDirTestCase): """ @@ -1681,3 +1844,20 @@ MANIFEST sub/Manifest.gz 78 MD5 9c158f87b2445279d7c8aac439612fba self.assertEqual( gemato.cli.main(['gemato', 'verify', self.dir]), 0) + + def test_update_entries_for_directory(self): + m = gemato.recursiveloader.ManifestRecursiveLoader( + os.path.join(self.dir, 'Manifest'), + hashes=['SHA256', 'SHA512']) + m.update_entries_for_directory('') + self.assertEqual(m.find_path_entry('sub/test').checksums, + { + 'SHA256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'SHA512': 'cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e', + }) + m.save_manifests() + with io.open(os.path.join(self.dir, 'sub/Manifest.gz'), + 'rb') as f: + self.assertNotEqual(f.read(), + base64.b64decode(self.SUB_MANIFEST)) + m.assert_directory_verifies() |