Initial code for Manifest parsing

2 files changed, 472 insertions, 0 deletions

diff --git a/gemato/manifest.py b/gemato/manifest.py
new file mode 100644
index 0000000..e5b6a81
--- /dev/null
+++ b/gemato/manifest.py
@@ -0,0 +1,250 @@
+# gemato: Manifest file objects
+# vim:fileencoding=utf-8
+# (c) 2017 Michał Górny
+# Licensed under the terms of 2-clause BSD license
+
+import datetime
+import os.path
+
+
+class ManifestSyntaxError(Exception):
+    def __init__(self, message):
+        super(ManifestSyntaxError, self).__init__(message)
+
+
+class ManifestEntryTIMESTAMP(object):
+    """
+    ISO-8601 timestamp.
+    """
+
+    def __init__(self, ts):
+        assert isinstance(ts, datetime.datetime)
+        self.ts = ts
+
+    @classmethod
+    def from_list(cls, l):
+        if len(l) != 1:
+            raise ManifestSyntaxError(
+                    'TIMESTAMP line: expects 1 value, got: {}'.format(l))
+        try:
+            ts = datetime.datetime.strptime(l[0], '%Y-%m-%dT%H:%M:%SZ')
+        except ValueError:
+            raise ManifestSyntaxError(
+                    'TIMESTAMP line: expected ISO8601 timestamp, got: {}'.format(l[0]))
+        return cls(ts)
+
+
+class ManifestPathEntry(object):
+    """
+    Base class for entries using a path.
+    """
+
+    def __init__(self, path):
+        assert path[0] != '/'
+        self.path = path
+
+    @staticmethod
+    def process_path(tag, l):
+        if len(l) != 1:
+            raise ManifestSyntaxError(
+                    '{} line: expects 1 value, got: {}'.format(tag, l))
+        if not l[0] or l[0][0] == '/':
+            raise ManifestSyntaxError(
+                    '{} line: expected relative path, got: {}'.format(tag, l[0]))
+        return l[0]
+
+
+class ManifestEntryIGNORE(ManifestPathEntry):
+    """
+    Ignored path.
+    """
+
+    @classmethod
+    def from_list(cls, l):
+        return cls(cls.process_path('IGNORE', l))
+
+
+class ManifestEntryOPTIONAL(ManifestPathEntry):
+    """
+    Optional path.
+    """
+
+    def __init__(self, path):
+        super(ManifestEntryOPTIONAL, self).__init__(path)
+        self.size = None
+        self.checksums = {}
+
+    @classmethod
+    def from_list(cls, l):
+        return cls(cls.process_path('OPTIONAL', l))
+
+
+class ManifestFileEntry(ManifestPathEntry):
+    """
+    Base class for entries providing checksums for a path.
+    """
+
+    def __init__(self, path, size, checksums):
+        super(ManifestFileEntry, self).__init__(path)
+        self.size = size
+        self.checksums = checksums
+
+    @staticmethod
+    def process_checksums(tag, l):
+        if len(l) < 2:
+            raise ManifestSyntaxError(
+                    '{} line: expects at least 2 values, got: {}'.format(tag, l))
+
+        try:
+            size = int(l[1])
+            if size < 0:
+                raise ValueError()
+        except ValueError:
+            raise ManifestSyntaxError(
+                    '{} line: size must be a non-negative integer, got: {}'.format(tag, l[1]))
+
+        checksums = {}
+        it = iter(l[2:])
+        while True:
+            try:
+                ckname = next(it)
+            except StopIteration:
+                break
+            try:
+                ckval = next(it)
+            except StopIteration:
+                raise ManifestSyntaxError(
+                        '{} line: checksum {} has no value'.format(tag, ckname))
+            checksums[ckname] = ckval
+
+        return size, checksums
+
+
+class ManifestEntryMANIFEST(ManifestFileEntry):
+    """
+    Sub-Manifest file reference.
+    """
+
+    @classmethod
+    def from_list(cls, l):
+        path = cls.process_path('MANIFEST', l[:1])
+        size, checksums = cls.process_checksums('MANIFEST', l)
+        return cls(path, size, checksums)
+
+
+class ManifestEntryDATA(ManifestFileEntry):
+    """
+    Regular file reference.
+    """
+
+    @classmethod
+    def from_list(cls, l):
+        path = cls.process_path('DATA', l[:1])
+        size, checksums = cls.process_checksums('DATA', l)
+        return cls(path, size, checksums)
+
+
+class ManifestEntryMISC(ManifestFileEntry):
+    """
+    Non-obligatory file reference.
+    """
+
+    @classmethod
+    def from_list(cls, l):
+        path = cls.process_path('MISC', l[:1])
+        size, checksums = cls.process_checksums('MISC', l)
+        return cls(path, size, checksums)
+
+
+class ManifestEntryDIST(ManifestFileEntry):
+    """
+    Distfile reference.
+    """
+
+    @classmethod
+    def from_list(cls, l):
+        path = cls.process_path('DIST', l[:1])
+        if '/' in path:
+            raise ManifestSyntaxError(
+                    'DIST line: file name expected, got directory path: {}'.format(path))
+        size, checksums = cls.process_checksums('DIST', l)
+        return cls(path, size, checksums)
+
+
+class ManifestEntryEBUILD(ManifestFileEntry):
+    """
+    Deprecated ebuild file reference (equivalent to DATA).
+    """
+
+    @classmethod
+    def from_list(cls, l):
+        path = cls.process_path('EBUILD', l[:1])
+        size, checksums = cls.process_checksums('EBUILD', l)
+        return cls(path, size, checksums)
+
+
+class ManifestEntryAUX(ManifestFileEntry):
+    """
+    Deprecated AUX file reference (DATA with 'files/' prepended).
+    """
+
+    def __init__(self, aux_path, size, checksums):
+        self.aux_path = aux_path
+        super(ManifestEntryAUX, self).__init__(
+                os.path.join('files', aux_path), size, checksums)
+
+    @classmethod
+    def from_list(cls, l):
+        path = cls.process_path('AUX', l[:1])
+        size, checksums = cls.process_checksums('AUX', l)
+        return cls(path, size, checksums)
+
+
+MANIFEST_TAG_MAPPING = {
+    'TIMESTAMP': ManifestEntryTIMESTAMP,
+    'MANIFEST': ManifestEntryMANIFEST,
+    'IGNORE': ManifestEntryIGNORE,
+    'DATA': ManifestEntryDATA,
+    'MISC': ManifestEntryMISC,
+    'OPTIONAL': ManifestEntryOPTIONAL,
+    'DIST': ManifestEntryDIST,
+    'EBUILD': ManifestEntryEBUILD,
+    'AUX': ManifestEntryAUX,
+}
+
+
+class ManifestFile(object):
+    """
+    A class encapsulating a single Manifest file. It supports reading
+    from files and writing to them.
+    """
+
+    def __init__(self, f=None):
+        """
+        Create a new instance. If @f is provided, reads the entries
+        from open Manifest file @f (see load()).
+        """
+        if f is not None:
+            self.load(f)
+
+    def load(self, f):
+        """
+        Load data from file @f. The file should be open for reading
+        in text mode, and oriented at the beginning.
+        """
+
+        for l in f:
+            sl = l.strip().split()
+            # skip empty lines
+            if not sl:
+                continue
+            tag = sl.pop(0)
+            MANIFEST_TAG_MAPPING[tag].from_list(sl)
+
+
+    def dump(self, f):
+        """
+        Dump data into file @f. The file should be open for writing
+        in text mode, and truncated to zero length.
+        """
+        pass
diff --git a/tests/test_manifest.py b/tests/test_manifest.py
new file mode 100644
index 0000000..ef95e22
--- /dev/null
+++ b/tests/test_manifest.py
@@ -0,0 +1,222 @@
+# gemato: Manifest file support tests
+# vim:fileencoding=utf-8
+# (c) 2017 Michał Górny
+# Licensed under the terms of 2-clause BSD license
+
+import datetime
+import io
+import unittest
+
+import gemato.manifest
+
+
+TEST_MANIFEST = '''
+TIMESTAMP 2017-10-22T18:06:41Z
+MANIFEST eclass/Manifest 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709
+IGNORE local
+DATA myebuild-0.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709
+MISC metadata.xml 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709
+OPTIONAL ChangeLog
+DIST mydistfile.tar.gz 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709
+'''
+
+TEST_DEPRECATED_MANIFEST = '''
+EBUILD myebuild-0.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709
+MISC metadata.xml 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709
+AUX test.patch 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709
+DIST mydistfile.tar.gz 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709
+'''
+
+
+class ManifestTest(unittest.TestCase):
+    """
+    Basic tests for Manifest processing.
+    """
+
+    def test_load(self):
+        m = gemato.manifest.ManifestFile()
+        m.load(io.StringIO(TEST_MANIFEST))
+
+    def test_load_deprecated(self):
+        m = gemato.manifest.ManifestFile()
+        m.load(io.StringIO(TEST_DEPRECATED_MANIFEST))
+
+
+class ManifestEntryTest(unittest.TestCase):
+    """
+    Basic tests for Manifest entries.
+    """
+
+    file_vals = ('test', '0', 'MD5', 'd41d8cd98f00b204e9800998ecf8427e',
+            'SHA1', 'da39a3ee5e6b4b0d3255bfef95601890afd80709')
+    exp_cksums = {
+        'MD5': 'd41d8cd98f00b204e9800998ecf8427e',
+        'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+    }
+
+    def test_TIMESTAMP(self):
+        self.assertEqual(gemato.manifest.ManifestEntryTIMESTAMP.from_list(('2010-01-01T11:12:13Z',)).ts,
+                datetime.datetime(2010, 1, 1, 11, 12, 13))
+
+    def test_MANIFEST(self):
+        m = gemato.manifest.ManifestEntryMANIFEST.from_list(self.file_vals)
+        self.assertEqual(m.path, 'test')
+        self.assertEqual(m.size, 0)
+        self.assertDictEqual(m.checksums, self.exp_cksums)
+
+    def test_IGNORE(self):
+        self.assertEqual(gemato.manifest.ManifestEntryIGNORE.from_list(('test',)).path,
+                'test')
+
+    def test_DATA(self):
+        m = gemato.manifest.ManifestEntryDATA.from_list(self.file_vals)
+        self.assertEqual(m.path, 'test')
+        self.assertEqual(m.size, 0)
+        self.assertDictEqual(m.checksums, self.exp_cksums)
+
+    def test_MISC(self):
+        m = gemato.manifest.ManifestEntryMISC.from_list(self.file_vals)
+        self.assertEqual(m.path, 'test')
+        self.assertEqual(m.size, 0)
+        self.assertDictEqual(m.checksums, self.exp_cksums)
+
+    def test_OPTIONAL(self):
+        self.assertEqual(gemato.manifest.ManifestEntryOPTIONAL.from_list(('test',)).path,
+                'test')
+
+    def test_DIST(self):
+        m = gemato.manifest.ManifestEntryDIST.from_list(self.file_vals)
+        self.assertEqual(m.path, 'test')
+        self.assertEqual(m.size, 0)
+        self.assertDictEqual(m.checksums, self.exp_cksums)
+
+    def test_EBUILD(self):
+        m = gemato.manifest.ManifestEntryEBUILD.from_list(self.file_vals)
+        self.assertEqual(m.path, 'test')
+        self.assertEqual(m.size, 0)
+        self.assertDictEqual(m.checksums, self.exp_cksums)
+
+    def test_AUX(self):
+        m = gemato.manifest.ManifestEntryAUX.from_list(self.file_vals)
+        self.assertEqual(m.aux_path, 'test')
+        self.assertEqual(m.path, 'files/test')
+        self.assertEqual(m.size, 0)
+        self.assertDictEqual(m.checksums, self.exp_cksums)
+
+    def test_timestamp_invalid(self):
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('2017-10-22T18:06:41+02:00',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('2017-10-22T18:06:41',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('2017-10-22 18:06:41Z',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('20171022T180641Z',))
+
+    def test_path_invalid(self):
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMANIFEST.from_list, ('', '0'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMANIFEST.from_list, ('/foo', '0'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryIGNORE.from_list, ('',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryIGNORE.from_list, ('/foo',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryOPTIONAL.from_list, ('',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryOPTIONAL.from_list, ('/foo',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDATA.from_list, ('',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDATA.from_list, ('/foo',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMISC.from_list, ('',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMISC.from_list, ('/foo',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDIST.from_list, ('',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDIST.from_list, ('/foo',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDIST.from_list, ('foo/bar.gz',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryEBUILD.from_list, ('',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryEBUILD.from_list, ('/foo',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryAUX.from_list, ('',))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryAUX.from_list, ('/foo',))
+
+    def test_size_invalid(self):
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMANIFEST.from_list, ('foo', 'asdf'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMANIFEST.from_list, ('foo', '5ds'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMANIFEST.from_list, ('foo', '-5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDATA.from_list, ('foo', 'asdf'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDATA.from_list, ('foo', '5ds'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDATA.from_list, ('foo', '-5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMISC.from_list, ('foo', 'asdf'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMISC.from_list, ('foo', '5ds'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMISC.from_list, ('foo', '-5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDIST.from_list, ('foo', 'asdf'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDIST.from_list, ('foo', '5ds'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDIST.from_list, ('foo', '-5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryEBUILD.from_list, ('foo', 'asdf'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryEBUILD.from_list, ('foo', '5ds'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryEBUILD.from_list, ('foo', '-5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryAUX.from_list, ('foo', 'asdf'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryAUX.from_list, ('foo', '5ds'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryAUX.from_list, ('foo', '-5'))
+
+    def test_checksum_short(self):
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMANIFEST.from_list, ('foo', '0', 'md5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMANIFEST.from_list,
+                ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDATA.from_list, ('foo', '0', 'md5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDATA.from_list,
+                ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMISC.from_list, ('foo', '0', 'md5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryMISC.from_list,
+                ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDIST.from_list, ('foo', '0', 'md5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryDIST.from_list,
+                ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryEBUILD.from_list, ('foo', '0', 'md5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryEBUILD.from_list,
+                ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryAUX.from_list, ('foo', '0', 'md5'))
+        self.assertRaises(gemato.manifest.ManifestSyntaxError,
+                gemato.manifest.ManifestEntryAUX.from_list,
+                ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1'))