verify: Support skipping rechecksum on mtime

2 files changed, 176 insertions, 4 deletions

diff --git a/gemato/verify.py b/gemato/verify.py
index 3c5e9df..942b93e 100644
--- a/gemato/verify.py
+++ b/gemato/verify.py
@@ -120,7 +120,7 @@ def get_file_metadata(path, hashes):
         yield ret
 
 
-def verify_path(path, e, expected_dev=None):
+def verify_path(path, e, expected_dev=None, last_mtime=None):
     """
     Verify the file at system path @path against the data in entry @e.
     The path/filename is not matched against the entry -- the correct
@@ -136,6 +136,10 @@ def verify_path(path, e, expected_dev=None):
     ManifestCrossDevice exception. It can be used to verify that
     the files do not cross filesystem boundaries.
 
+    If @last_mtime is not None, it specifies the timestamp corresponding
+    to the previous file verification. If the file is not newer
+    than that, the checksum verification is skipped.
+
     Each name can be:
     - __exists__ (boolean) to indicate whether the file existed,
     - __type__ (string) as a human-readable description of file type,
@@ -183,8 +187,12 @@ def verify_path(path, e, expected_dev=None):
         if st_size != 0 and st_size != e.size:
             return (False, [('__size__', e.size, st_size)])
 
-        # 5. skip st_mtime for now
+        # 5. skip checksums if file has not changed since the last time
+        #    (and st_size != 0 since we can't trust weird filesystems)
         st_mtime = next(g)
+        if (last_mtime is not None and st_mtime <= last_mtime
+                and st_size != 0):
+            return (True, [])
 
         # 6. verify the real size from checksum data
         checksums = next(g)
@@ -206,7 +214,8 @@ def verify_path(path, e, expected_dev=None):
     return (True, [])
 
 
-def update_entry_for_path(path, e, hashes=None, expected_dev=None):
+def update_entry_for_path(path, e, hashes=None, expected_dev=None,
+        last_mtime=None):
     """
     Update the data in entry @e to match the current state of file
     at path @path. Uses hashes listed in @hashes (using Manifest names),
@@ -223,6 +232,10 @@ def update_entry_for_path(path, e, hashes=None, expected_dev=None):
     on specified device. If the device does not match, raises
     ManifestCrossDevice exception. It can be used to verify that
     the files do not cross filesystem boundaries.
+
+    If @last_mtime is not None, it specifies the timestamp corresponding
+    to the previous file update. If the file is not newer than that,
+    the checksum calculation is skipped.
     """
 
     assert e.tag not in ('IGNORE', 'OPTIONAL', 'TIMESTAMP')
@@ -251,8 +264,12 @@ def update_entry_for_path(path, e, hashes=None, expected_dev=None):
         # 4. get the apparent file size
         st_size = next(g)
 
-        # 5. skip st_mtime for now
+        # 5. skip checksums if file has not changed since the last time
+        #    (and st_size makes sense)
         st_mtime = next(g)
+        if (last_mtime is not None and st_mtime <= last_mtime
+                and st_size != 0 and st_size == e.size):
+            return False
 
         # 6. get the checksums and real size
         checksums = next(g)
diff --git a/tests/test_verify.py b/tests/test_verify.py
index 1656b23..93360b9 100644
--- a/tests/test_verify.py
+++ b/tests/test_verify.py
@@ -430,6 +430,67 @@ class EmptyFileVerificationTest(unittest.TestCase):
         self.assertEqual(e.size, 0)
         self.assertDictEqual(e.checksums, {})
 
+    def test_wrong_checksum_DATA_with_old_mtime(self):
+        """
+        Test whether the checksums are verified if last mtime is older
+        than the current one.
+        """
+        e = gemato.manifest.ManifestEntryDATA.from_list(
+                ('DATA', os.path.basename(self.path), '0',
+                    'MD5', '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1', 'da39a3ee5e6b4b0d3255bfef95601890afd80709'))
+        self.assertEqual(
+                gemato.verify.verify_path(self.path, e,
+                    last_mtime=0),
+                (False, [('MD5', '9e107d9d372bb6826bd81d3542a419d6', 'd41d8cd98f00b204e9800998ecf8427e')]))
+
+    def test_wrong_checksum_DATA_with_new_mtime(self):
+        """
+        Test whether the checksums are verified if last mtime indicates
+        that the file did not change (with st_size == 0).
+        """
+        e = gemato.manifest.ManifestEntryDATA.from_list(
+                ('DATA', os.path.basename(self.path), '0',
+                    'MD5', '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1', 'da39a3ee5e6b4b0d3255bfef95601890afd80709'))
+        st = os.stat(self.path)
+        self.assertEqual(
+                gemato.verify.verify_path(self.path, e,
+                    last_mtime=st.st_mtime),
+                (False, [('MD5', '9e107d9d372bb6826bd81d3542a419d6', 'd41d8cd98f00b204e9800998ecf8427e')]))
+
+    def test_update_with_hashes_and_old_mtime(self):
+        e = gemato.manifest.ManifestEntryDATA(
+                os.path.basename(self.path), 0, {
+                    'MD5': '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+                })
+        self.assertTrue(
+                gemato.verify.update_entry_for_path(self.path, e,
+                    ['MD5', 'SHA1'], last_mtime=0))
+        self.assertEqual(e.path, os.path.basename(self.path))
+        self.assertEqual(e.size, 0)
+        self.assertDictEqual(e.checksums, {
+                'MD5': 'd41d8cd98f00b204e9800998ecf8427e',
+                'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+            })
+
+    def test_update_with_hashes_and_new_mtime(self):
+        e = gemato.manifest.ManifestEntryDATA(
+                os.path.basename(self.path), 0, {
+                    'MD5': '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+                })
+        st = os.stat(self.path)
+        self.assertTrue(
+                gemato.verify.update_entry_for_path(self.path, e,
+                    ['MD5', 'SHA1'], last_mtime=st.st_mtime))
+        self.assertEqual(e.path, os.path.basename(self.path))
+        self.assertEqual(e.size, 0)
+        self.assertDictEqual(e.checksums, {
+                'MD5': 'd41d8cd98f00b204e9800998ecf8427e',
+                'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+            })
 
 
 class NonEmptyFileVerificationTest(unittest.TestCase):
@@ -549,6 +610,100 @@ class NonEmptyFileVerificationTest(unittest.TestCase):
                 'SHA1': '2fd4e1c67a2d28fced849ee1bb76e7391b93eb12',
             })
 
+    def test_wrong_checksum_DATA_with_old_mtime(self):
+        """
+        Test whether the checksums are verified if last mtime is older
+        than the current one.
+        """
+        e = gemato.manifest.ManifestEntryDATA.from_list(
+                ('DATA', os.path.basename(self.path), '43',
+                    'MD5', '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1', 'da39a3ee5e6b4b0d3255bfef95601890afd80709'))
+        self.assertEqual(
+                gemato.verify.verify_path(self.path, e,
+                    last_mtime=0),
+                (False, [('SHA1', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', '2fd4e1c67a2d28fced849ee1bb76e7391b93eb12')]))
+
+    def test_wrong_checksum_DATA_with_new_mtime(self):
+        """
+        Test whether the checksums are verified if last mtime indicates
+        that the file did not change.
+        """
+        e = gemato.manifest.ManifestEntryDATA.from_list(
+                ('DATA', os.path.basename(self.path), '43',
+                    'MD5', '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1', 'da39a3ee5e6b4b0d3255bfef95601890afd80709'))
+        st = os.stat(self.path)
+        self.assertEqual(
+                gemato.verify.verify_path(self.path, e,
+                    last_mtime=st.st_mtime),
+                (True, []))
+
+    def test_wrong_checksum_DATA_with_new_mtime_and_wrong_size(self):
+        """
+        Test whether the checksums are verified if last mtime indicates
+        that the file did not change but size is different.
+        """
+        e = gemato.manifest.ManifestEntryDATA.from_list(
+                ('DATA', os.path.basename(self.path), '33',
+                    'MD5', '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1', 'da39a3ee5e6b4b0d3255bfef95601890afd80709'))
+        st = os.stat(self.path)
+        self.assertEqual(
+                gemato.verify.verify_path(self.path, e,
+                    last_mtime=st.st_mtime),
+                (False, [('__size__', 33, 43)]))
+
+    def test_update_with_hashes_and_old_mtime(self):
+        e = gemato.manifest.ManifestEntryDATA(
+                os.path.basename(self.path), 0, {
+                    'MD5': '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+                })
+        self.assertTrue(
+                gemato.verify.update_entry_for_path(self.path, e,
+                    ['MD5', 'SHA1'], last_mtime=0))
+        self.assertEqual(e.path, os.path.basename(self.path))
+        self.assertEqual(e.size, 43)
+        self.assertDictEqual(e.checksums, {
+                'MD5': '9e107d9d372bb6826bd81d3542a419d6',
+                'SHA1': '2fd4e1c67a2d28fced849ee1bb76e7391b93eb12',
+            })
+
+    def test_update_with_hashes_and_new_mtime(self):
+        e = gemato.manifest.ManifestEntryDATA(
+                os.path.basename(self.path), 43, {
+                    'MD5': '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+                })
+        st = os.stat(self.path)
+        self.assertFalse(
+                gemato.verify.update_entry_for_path(self.path, e,
+                    ['MD5', 'SHA1'], last_mtime=st.st_mtime))
+        self.assertEqual(e.path, os.path.basename(self.path))
+        self.assertEqual(e.size, 43)
+        self.assertDictEqual(e.checksums, {
+                'MD5': '9e107d9d372bb6826bd81d3542a419d6',
+                'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+            })
+
+    def test_update_with_hashes_and_new_mtime_but_wrong_size(self):
+        e = gemato.manifest.ManifestEntryDATA(
+                os.path.basename(self.path), 33, {
+                    'MD5': '9e107d9d372bb6826bd81d3542a419d6',
+                    'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+                })
+        st = os.stat(self.path)
+        self.assertTrue(
+                gemato.verify.update_entry_for_path(self.path, e,
+                    ['MD5', 'SHA1'], last_mtime=st.st_mtime))
+        self.assertEqual(e.path, os.path.basename(self.path))
+        self.assertEqual(e.size, 43)
+        self.assertDictEqual(e.checksums, {
+                'MD5': '9e107d9d372bb6826bd81d3542a419d6',
+                'SHA1': '2fd4e1c67a2d28fced849ee1bb76e7391b93eb12',
+            })
+
 
 class SymbolicLinkVerificationTest(NonEmptyFileVerificationTest):
     """