recursiveloader: Handle duplicate entries in Manifests

3 files changed, 330 insertions, 2 deletions

diff --git a/gemato/recursiveloader.py b/gemato/recursiveloader.py
index 6cc01b4..b50d2ba 100644
--- a/gemato/recursiveloader.py
+++ b/gemato/recursiveloader.py
@@ -12,6 +12,17 @@ import gemato.util
 import gemato.verify
 
 
+class ManifestIncompatibleEntry(Exception):
+    def __init__(self, e1, e2, diff):
+        msg = "Incompatible Manifest entries for {}".format(e1.path)
+        for k, d1, d2 in diff:
+            msg += "\n  {}: e1: {}, e2: {}".format(k, e1, e2)
+        super(ManifestIncompatibleEntry, self).__init__(msg)
+        self.e1 = e1
+        self.e2 = e2
+        self.diff = diff
+
+
 class ManifestRecursiveLoader(object):
     """
     A class encapsulating a tree covered by multiple Manifests.
@@ -171,6 +182,18 @@ class ManifestRecursiveLoader(object):
                 elif isinstance(e, gemato.manifest.ManifestPathEntry):
                     fullpath = os.path.join(relpath, e.path)
                     if gemato.util.path_starts_with(fullpath, path):
-                        # TODO: implement conflict detection
+                        if fullpath in out:
+                            # compare the two entries
+                            ret, diff = gemato.verify.verify_entry_compatibility(
+                                    out[fullpath], e)
+                            if not ret:
+                                raise ManifestIncompatibleEntry(out[fullpath], e, diff)
+                            # we need to construct a single entry with both checksums
+                            if diff:
+                                new_checksums = dict(e.checksums)
+                                for k, d1, d2 in diff:
+                                    if d2 is None:
+                                        new_checksums[k] = d1
+                                e = type(e)(e.path, e.size, new_checksums)
                         out[fullpath] = e
         return out
diff --git a/gemato/verify.py b/gemato/verify.py
index a34264e..eda13ba 100644
--- a/gemato/verify.py
+++ b/gemato/verify.py
@@ -137,7 +137,6 @@ def verify_entry_compatibility(e1, e2):
     hashes that are present only in one of the entries.
     """
 
-    assert e1.path == e2.path
     assert isinstance(e1, gemato.manifest.ManifestPathEntry)
     assert isinstance(e2, gemato.manifest.ManifestPathEntry)
 
diff --git a/tests/test_recursiveloader.py b/tests/test_recursiveloader.py
index d891f1f..f18a1e8 100644
--- a/tests/test_recursiveloader.py
+++ b/tests/test_recursiveloader.py
@@ -283,3 +283,309 @@ TIMESTAMP 2017-01-01T01:01:01Z
             os.path.join(self.dir, 'Manifest'))
         self.assertEqual(m.find_timestamp().ts,
                 datetime.datetime(2017, 1, 1, 1, 1, 1))
+
+
+class DuplicateFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice.
+    """
+
+    FILES = {
+        'Manifest': u'''
+DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('test').path, 'test')
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        entries = m.get_file_entry_dict('')
+        self.assertSetEqual(frozenset(entries), frozenset(('test',)))
+        self.assertEqual(entries['test'].path, 'test')
+
+
+class DuplicateManifestFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same Manifest twice.
+    """
+
+    DIRS = ['sub']
+    FILES = {
+        'Manifest': u'''
+MANIFEST sub/Manifest 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+MANIFEST sub/Manifest 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+''',
+        'sub/Manifest': u''
+    }
+
+    def test_load_sub_manifest(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertNotIn('sub/Manifest', m.loaded_manifests)
+        m.load_manifests_for_path('sub/test')
+        self.assertIn('sub/Manifest', m.loaded_manifests)
+
+
+class DuplicateManifestDATAFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same Manifest as MANIFEST
+    and DATA.
+    """
+
+    DIRS = ['sub']
+    FILES = {
+        'Manifest': u'''
+DATA sub/Manifest 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+MANIFEST sub/Manifest 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+''',
+        'sub/Manifest': u''
+    }
+
+    def test_load_sub_manifest(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertNotIn('sub/Manifest', m.loaded_manifests)
+        m.load_manifests_for_path('sub/test')
+        self.assertIn('sub/Manifest', m.loaded_manifests)
+
+
+class DuplicateFileEntryInSubManifestTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice in different
+    Manifest files.
+    """
+
+    DIRS = ['sub']
+    FILES = {
+        'Manifest': u'''
+MANIFEST sub/Manifest 50 MD5 0f7cd9ed779a4844f98d28315dd9176a
+DATA sub/test 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+''',
+        'sub/Manifest': u'''
+DATA test 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('sub/test').size, 0)
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        entries = m.get_file_entry_dict('')
+        self.assertSetEqual(frozenset(entries),
+                frozenset(('sub/test', 'sub/Manifest')))
+        self.assertEqual(entries['sub/test'].size, 0)
+
+
+class DuplicateCompatibleTypeFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice, with
+    compatible types.
+    """
+
+    FILES = {
+        'Manifest': u'''
+DATA test.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+EBUILD test.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('test.ebuild').path, 'test.ebuild')
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        entries = m.get_file_entry_dict('')
+        self.assertSetEqual(frozenset(entries), frozenset(('test.ebuild',)))
+        self.assertEqual(entries['test.ebuild'].path, 'test.ebuild')
+
+
+class DuplicateAUXTypeFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice, using AUX
+    type (because of path weirdness).
+    """
+
+    FILES = {
+        'Manifest': u'''
+DATA files/test.patch 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+AUX test.patch 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('files/test.patch').path, 'files/test.patch')
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        entries = m.get_file_entry_dict('')
+        self.assertSetEqual(frozenset(entries), frozenset(('files/test.patch',)))
+        self.assertEqual(entries['files/test.patch'].path, 'files/test.patch')
+
+
+class DuplicateFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice,
+    with different hash sets (and both of them mismatched).
+    """
+
+    FILES = {
+        'Manifest': u'''
+DATA test 0 MD5 9e107d9d372bb6826bd81d3542a419d6
+DATA test 0 SHA1 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('test').path, 'test')
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        entries = m.get_file_entry_dict('')
+        self.assertSetEqual(frozenset(entries), frozenset(('test',)))
+        self.assertEqual(entries['test'].path, 'test')
+        self.assertSetEqual(frozenset(entries['test'].checksums),
+            frozenset(('MD5', 'SHA1')))
+
+
+class DuplicateIncompatibleDataMiscTypeFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice, with
+    incompatible types.
+    """
+
+    FILES = {
+        'Manifest': u'''
+DATA test.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+MISC test.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('test.ebuild').path, 'test.ebuild')
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertRaises(gemato.recursiveloader.ManifestIncompatibleEntry,
+                m.get_file_entry_dict, '')
+
+
+class DuplicateIncompatibleDataOptionalTypeFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice, with
+    incompatible types.
+    """
+
+    FILES = {
+        'Manifest': u'''
+DATA test.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+OPTIONAL test.ebuild
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('test.ebuild').path, 'test.ebuild')
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertRaises(gemato.recursiveloader.ManifestIncompatibleEntry,
+                m.get_file_entry_dict, '')
+
+
+class DuplicateIncompatibleMiscOptionalTypeFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice, with
+    incompatible types.
+    """
+
+    FILES = {
+        'Manifest': u'''
+MISC test.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+OPTIONAL test.ebuild
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('test.ebuild').path, 'test.ebuild')
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertRaises(gemato.recursiveloader.ManifestIncompatibleEntry,
+                m.get_file_entry_dict, '')
+
+
+class DuplicateDifferentSizeFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice, with
+    different sizes.
+    """
+
+    FILES = {
+        'Manifest': u'''
+DATA test.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+DATA test.ebuild 32 MD5 d41d8cd98f00b204e9800998ecf8427e
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('test.ebuild').path, 'test.ebuild')
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertRaises(gemato.recursiveloader.ManifestIncompatibleEntry,
+                m.get_file_entry_dict, '')
+
+
+class DuplicateDifferentHashFileEntryTest(TempDirTestCase):
+    """
+    Test for specifying the entry for the same file twice, with
+    different sizes.
+    """
+
+    FILES = {
+        'Manifest': u'''
+DATA test.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e
+DATA test.ebuild 0 MD5 9e107d9d372bb6826bd81d3542a419d6
+''',
+    }
+
+    def test_find_path_entry(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertEqual(m.find_path_entry('test.ebuild').path, 'test.ebuild')
+
+    def test_get_file_entry_dict(self):
+        m = gemato.recursiveloader.ManifestRecursiveLoader(
+            os.path.join(self.dir, 'Manifest'))
+        self.assertRaises(gemato.recursiveloader.ManifestIncompatibleEntry,
+                m.get_file_entry_dict, '')