From 40be55d5109474920c230834618d1b20631f9dc3 Mon Sep 17 00:00:00 2001 From: Michał Górny Date: Sun, 22 Oct 2017 19:23:09 +0200 Subject: Initial code for Manifest parsing --- gemato/manifest.py | 250 +++++++++++++++++++++++++++++++++++++++++++++++++ tests/test_manifest.py | 222 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 472 insertions(+) create mode 100644 gemato/manifest.py create mode 100644 tests/test_manifest.py diff --git a/gemato/manifest.py b/gemato/manifest.py new file mode 100644 index 0000000..e5b6a81 --- /dev/null +++ b/gemato/manifest.py @@ -0,0 +1,250 @@ +# gemato: Manifest file objects +# vim:fileencoding=utf-8 +# (c) 2017 Michał Górny +# Licensed under the terms of 2-clause BSD license + +import datetime +import os.path + + +class ManifestSyntaxError(Exception): + def __init__(self, message): + super(ManifestSyntaxError, self).__init__(message) + + +class ManifestEntryTIMESTAMP(object): + """ + ISO-8601 timestamp. + """ + + def __init__(self, ts): + assert isinstance(ts, datetime.datetime) + self.ts = ts + + @classmethod + def from_list(cls, l): + if len(l) != 1: + raise ManifestSyntaxError( + 'TIMESTAMP line: expects 1 value, got: {}'.format(l)) + try: + ts = datetime.datetime.strptime(l[0], '%Y-%m-%dT%H:%M:%SZ') + except ValueError: + raise ManifestSyntaxError( + 'TIMESTAMP line: expected ISO8601 timestamp, got: {}'.format(l[0])) + return cls(ts) + + +class ManifestPathEntry(object): + """ + Base class for entries using a path. + """ + + def __init__(self, path): + assert path[0] != '/' + self.path = path + + @staticmethod + def process_path(tag, l): + if len(l) != 1: + raise ManifestSyntaxError( + '{} line: expects 1 value, got: {}'.format(tag, l)) + if not l[0] or l[0][0] == '/': + raise ManifestSyntaxError( + '{} line: expected relative path, got: {}'.format(tag, l[0])) + return l[0] + + +class ManifestEntryIGNORE(ManifestPathEntry): + """ + Ignored path. + """ + + @classmethod + def from_list(cls, l): + return cls(cls.process_path('IGNORE', l)) + + +class ManifestEntryOPTIONAL(ManifestPathEntry): + """ + Optional path. + """ + + def __init__(self, path): + super(ManifestEntryOPTIONAL, self).__init__(path) + self.size = None + self.checksums = {} + + @classmethod + def from_list(cls, l): + return cls(cls.process_path('OPTIONAL', l)) + + +class ManifestFileEntry(ManifestPathEntry): + """ + Base class for entries providing checksums for a path. + """ + + def __init__(self, path, size, checksums): + super(ManifestFileEntry, self).__init__(path) + self.size = size + self.checksums = checksums + + @staticmethod + def process_checksums(tag, l): + if len(l) < 2: + raise ManifestSyntaxError( + '{} line: expects at least 2 values, got: {}'.format(tag, l)) + + try: + size = int(l[1]) + if size < 0: + raise ValueError() + except ValueError: + raise ManifestSyntaxError( + '{} line: size must be a non-negative integer, got: {}'.format(tag, l[1])) + + checksums = {} + it = iter(l[2:]) + while True: + try: + ckname = next(it) + except StopIteration: + break + try: + ckval = next(it) + except StopIteration: + raise ManifestSyntaxError( + '{} line: checksum {} has no value'.format(tag, ckname)) + checksums[ckname] = ckval + + return size, checksums + + +class ManifestEntryMANIFEST(ManifestFileEntry): + """ + Sub-Manifest file reference. + """ + + @classmethod + def from_list(cls, l): + path = cls.process_path('MANIFEST', l[:1]) + size, checksums = cls.process_checksums('MANIFEST', l) + return cls(path, size, checksums) + + +class ManifestEntryDATA(ManifestFileEntry): + """ + Regular file reference. + """ + + @classmethod + def from_list(cls, l): + path = cls.process_path('DATA', l[:1]) + size, checksums = cls.process_checksums('DATA', l) + return cls(path, size, checksums) + + +class ManifestEntryMISC(ManifestFileEntry): + """ + Non-obligatory file reference. + """ + + @classmethod + def from_list(cls, l): + path = cls.process_path('MISC', l[:1]) + size, checksums = cls.process_checksums('MISC', l) + return cls(path, size, checksums) + + +class ManifestEntryDIST(ManifestFileEntry): + """ + Distfile reference. + """ + + @classmethod + def from_list(cls, l): + path = cls.process_path('DIST', l[:1]) + if '/' in path: + raise ManifestSyntaxError( + 'DIST line: file name expected, got directory path: {}'.format(path)) + size, checksums = cls.process_checksums('DIST', l) + return cls(path, size, checksums) + + +class ManifestEntryEBUILD(ManifestFileEntry): + """ + Deprecated ebuild file reference (equivalent to DATA). + """ + + @classmethod + def from_list(cls, l): + path = cls.process_path('EBUILD', l[:1]) + size, checksums = cls.process_checksums('EBUILD', l) + return cls(path, size, checksums) + + +class ManifestEntryAUX(ManifestFileEntry): + """ + Deprecated AUX file reference (DATA with 'files/' prepended). + """ + + def __init__(self, aux_path, size, checksums): + self.aux_path = aux_path + super(ManifestEntryAUX, self).__init__( + os.path.join('files', aux_path), size, checksums) + + @classmethod + def from_list(cls, l): + path = cls.process_path('AUX', l[:1]) + size, checksums = cls.process_checksums('AUX', l) + return cls(path, size, checksums) + + +MANIFEST_TAG_MAPPING = { + 'TIMESTAMP': ManifestEntryTIMESTAMP, + 'MANIFEST': ManifestEntryMANIFEST, + 'IGNORE': ManifestEntryIGNORE, + 'DATA': ManifestEntryDATA, + 'MISC': ManifestEntryMISC, + 'OPTIONAL': ManifestEntryOPTIONAL, + 'DIST': ManifestEntryDIST, + 'EBUILD': ManifestEntryEBUILD, + 'AUX': ManifestEntryAUX, +} + + +class ManifestFile(object): + """ + A class encapsulating a single Manifest file. It supports reading + from files and writing to them. + """ + + def __init__(self, f=None): + """ + Create a new instance. If @f is provided, reads the entries + from open Manifest file @f (see load()). + """ + if f is not None: + self.load(f) + + def load(self, f): + """ + Load data from file @f. The file should be open for reading + in text mode, and oriented at the beginning. + """ + + for l in f: + sl = l.strip().split() + # skip empty lines + if not sl: + continue + tag = sl.pop(0) + MANIFEST_TAG_MAPPING[tag].from_list(sl) + + + def dump(self, f): + """ + Dump data into file @f. The file should be open for writing + in text mode, and truncated to zero length. + """ + pass diff --git a/tests/test_manifest.py b/tests/test_manifest.py new file mode 100644 index 0000000..ef95e22 --- /dev/null +++ b/tests/test_manifest.py @@ -0,0 +1,222 @@ +# gemato: Manifest file support tests +# vim:fileencoding=utf-8 +# (c) 2017 Michał Górny +# Licensed under the terms of 2-clause BSD license + +import datetime +import io +import unittest + +import gemato.manifest + + +TEST_MANIFEST = ''' +TIMESTAMP 2017-10-22T18:06:41Z +MANIFEST eclass/Manifest 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709 +IGNORE local +DATA myebuild-0.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709 +MISC metadata.xml 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709 +OPTIONAL ChangeLog +DIST mydistfile.tar.gz 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709 +''' + +TEST_DEPRECATED_MANIFEST = ''' +EBUILD myebuild-0.ebuild 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709 +MISC metadata.xml 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709 +AUX test.patch 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709 +DIST mydistfile.tar.gz 0 MD5 d41d8cd98f00b204e9800998ecf8427e SHA1 da39a3ee5e6b4b0d3255bfef95601890afd80709 +''' + + +class ManifestTest(unittest.TestCase): + """ + Basic tests for Manifest processing. + """ + + def test_load(self): + m = gemato.manifest.ManifestFile() + m.load(io.StringIO(TEST_MANIFEST)) + + def test_load_deprecated(self): + m = gemato.manifest.ManifestFile() + m.load(io.StringIO(TEST_DEPRECATED_MANIFEST)) + + +class ManifestEntryTest(unittest.TestCase): + """ + Basic tests for Manifest entries. + """ + + file_vals = ('test', '0', 'MD5', 'd41d8cd98f00b204e9800998ecf8427e', + 'SHA1', 'da39a3ee5e6b4b0d3255bfef95601890afd80709') + exp_cksums = { + 'MD5': 'd41d8cd98f00b204e9800998ecf8427e', + 'SHA1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709', + } + + def test_TIMESTAMP(self): + self.assertEqual(gemato.manifest.ManifestEntryTIMESTAMP.from_list(('2010-01-01T11:12:13Z',)).ts, + datetime.datetime(2010, 1, 1, 11, 12, 13)) + + def test_MANIFEST(self): + m = gemato.manifest.ManifestEntryMANIFEST.from_list(self.file_vals) + self.assertEqual(m.path, 'test') + self.assertEqual(m.size, 0) + self.assertDictEqual(m.checksums, self.exp_cksums) + + def test_IGNORE(self): + self.assertEqual(gemato.manifest.ManifestEntryIGNORE.from_list(('test',)).path, + 'test') + + def test_DATA(self): + m = gemato.manifest.ManifestEntryDATA.from_list(self.file_vals) + self.assertEqual(m.path, 'test') + self.assertEqual(m.size, 0) + self.assertDictEqual(m.checksums, self.exp_cksums) + + def test_MISC(self): + m = gemato.manifest.ManifestEntryMISC.from_list(self.file_vals) + self.assertEqual(m.path, 'test') + self.assertEqual(m.size, 0) + self.assertDictEqual(m.checksums, self.exp_cksums) + + def test_OPTIONAL(self): + self.assertEqual(gemato.manifest.ManifestEntryOPTIONAL.from_list(('test',)).path, + 'test') + + def test_DIST(self): + m = gemato.manifest.ManifestEntryDIST.from_list(self.file_vals) + self.assertEqual(m.path, 'test') + self.assertEqual(m.size, 0) + self.assertDictEqual(m.checksums, self.exp_cksums) + + def test_EBUILD(self): + m = gemato.manifest.ManifestEntryEBUILD.from_list(self.file_vals) + self.assertEqual(m.path, 'test') + self.assertEqual(m.size, 0) + self.assertDictEqual(m.checksums, self.exp_cksums) + + def test_AUX(self): + m = gemato.manifest.ManifestEntryAUX.from_list(self.file_vals) + self.assertEqual(m.aux_path, 'test') + self.assertEqual(m.path, 'files/test') + self.assertEqual(m.size, 0) + self.assertDictEqual(m.checksums, self.exp_cksums) + + def test_timestamp_invalid(self): + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('2017-10-22T18:06:41+02:00',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('2017-10-22T18:06:41',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('2017-10-22 18:06:41Z',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryTIMESTAMP.from_list, ('20171022T180641Z',)) + + def test_path_invalid(self): + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMANIFEST.from_list, ('', '0')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMANIFEST.from_list, ('/foo', '0')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryIGNORE.from_list, ('',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryIGNORE.from_list, ('/foo',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryOPTIONAL.from_list, ('',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryOPTIONAL.from_list, ('/foo',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDATA.from_list, ('',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDATA.from_list, ('/foo',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMISC.from_list, ('',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMISC.from_list, ('/foo',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDIST.from_list, ('',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDIST.from_list, ('/foo',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDIST.from_list, ('foo/bar.gz',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryEBUILD.from_list, ('',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryEBUILD.from_list, ('/foo',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryAUX.from_list, ('',)) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryAUX.from_list, ('/foo',)) + + def test_size_invalid(self): + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMANIFEST.from_list, ('foo', 'asdf')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMANIFEST.from_list, ('foo', '5ds')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMANIFEST.from_list, ('foo', '-5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDATA.from_list, ('foo', 'asdf')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDATA.from_list, ('foo', '5ds')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDATA.from_list, ('foo', '-5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMISC.from_list, ('foo', 'asdf')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMISC.from_list, ('foo', '5ds')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMISC.from_list, ('foo', '-5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDIST.from_list, ('foo', 'asdf')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDIST.from_list, ('foo', '5ds')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDIST.from_list, ('foo', '-5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryEBUILD.from_list, ('foo', 'asdf')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryEBUILD.from_list, ('foo', '5ds')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryEBUILD.from_list, ('foo', '-5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryAUX.from_list, ('foo', 'asdf')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryAUX.from_list, ('foo', '5ds')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryAUX.from_list, ('foo', '-5')) + + def test_checksum_short(self): + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMANIFEST.from_list, ('foo', '0', 'md5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMANIFEST.from_list, + ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDATA.from_list, ('foo', '0', 'md5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDATA.from_list, + ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMISC.from_list, ('foo', '0', 'md5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryMISC.from_list, + ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDIST.from_list, ('foo', '0', 'md5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryDIST.from_list, + ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryEBUILD.from_list, ('foo', '0', 'md5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryEBUILD.from_list, + ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryAUX.from_list, ('foo', '0', 'md5')) + self.assertRaises(gemato.manifest.ManifestSyntaxError, + gemato.manifest.ManifestEntryAUX.from_list, + ('foo', '0', 'md5', 'd41d8cd98f00b204e9800998ecf8427e', 'sha1')) -- cgit v1.2.3