Include list of files in manifest file

1. Per SOL004 v2.6.1, the list of files should be in included in the
manifest file even without integrity.

2. Refactor the manifest file parsing code to reduce the complexity.

Change-Id: Iac06dff7f1ca2014e7c1a5b1546ecae1b02ad7c6
Issue-ID: VNFSDK-420
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>

4 files changed, 76 insertions, 59 deletions

diff --git a/tests/packager/test_csar.py b/tests/packager/test_csar.py
index 6fca020..b63a117 100644
--- a/tests/packager/test_csar.py
+++ b/tests/packager/test_csar.py
@@ -121,9 +121,6 @@ def csar_write_test(args):
         csar.write(args.source, args.entry, csar_target_dir + '/' + CSAR_OUTPUT_FILE, args)
         csar.read(csar_target_dir + '/' + CSAR_OUTPUT_FILE, csar_extract_dir, True)
         assert filecmp.cmp(args.source + '/' + args.entry, csar_extract_dir + '/' + args.entry)
-        if(args.manifest and not args.digest):
-            assert filecmp.cmp(args.source + '/' + args.manifest,
-                               csar_extract_dir + '/' + args.manifest)
         if(args.history):
             assert filecmp.cmp(args.source + '/' + args.history,
                                csar_extract_dir + '/' + args.history)
diff --git a/tests/packager/test_manifest.py b/tests/packager/test_manifest.py
index e8e1e17..556b87f 100644
--- a/tests/packager/test_manifest.py
+++ b/tests/packager/test_manifest.py
@@ -59,6 +59,10 @@ CMS = '\n'.join(['-----BEGIN CMS-----',
                  '-----END CMS-----',
                 ])
 
+FILE_SOURCE_ONLY = '\n'.join(['Source: source1',
+                              'Source: source2',
+                           ])
+
 def test_metadata(tmpdir):
     p = tmpdir.mkdir('csar').join('test.mf')
     p.write(METADATA)
@@ -85,7 +89,7 @@ def test_missing_metadata(tmpdir):
 
     with pytest.raises(manifest.ManifestException) as excinfo:
         manifest.Manifest(p.dirname, 'test.mf')
-    excinfo.match(r"Unknown key in line")
+    excinfo.match(r"Unrecognized file digest line vnf_product_name: test:")
 
 def test_digest(tmpdir):
     root = tmpdir.mkdir('csar')
@@ -152,3 +156,10 @@ def test_signature_strip(tmpdir):
     assert m1.metadata == m2.metadata
     assert m2.signature is None
     os.unlink(newfile)
+
+def test_source_only(tmpdir):
+    p = tmpdir.mkdir('csar').join('test.mf')
+    p.write(METADATA + "\n\n" + FILE_SOURCE_ONLY)
+    m = manifest.Manifest(p.dirname, 'test.mf')
+    assert 'source1' in m.digests.keys()
+    assert 'source2' in m.digests.keys()
diff --git a/vnfsdk_pkgtools/packager/csar.py b/vnfsdk_pkgtools/packager/csar.py
index 5fcbec7..1797644 100644
--- a/vnfsdk_pkgtools/packager/csar.py
+++ b/vnfsdk_pkgtools/packager/csar.py
@@ -89,7 +89,7 @@ def write(source, entry, destination, args):
                     file_relative_path = os.path.relpath(file_full_path, source)
                     LOG.debug('Writing to archive: {0}'.format(file_relative_path))
                     f.write(file_full_path, file_relative_path)
-                    if manifest_file and args.digest:
+                    if manifest_file:
                         LOG.debug('Update file digest: {0}'.format(file_relative_path))
                         manifest_file.add_file(file_relative_path, args.digest)
         if manifest_file:
diff --git a/vnfsdk_pkgtools/packager/manifest.py b/vnfsdk_pkgtools/packager/manifest.py
index 5066f45..e142c56 100644
--- a/vnfsdk_pkgtools/packager/manifest.py
+++ b/vnfsdk_pkgtools/packager/manifest.py
@@ -45,7 +45,7 @@ class Manifest(object):
         self.signature = None
         self.blocks = [ ]
         self._split_blocks()
-        self._parse_blocks()
+        self._parse_all_blocks()
 
     @staticmethod
     def __split_line(s):
@@ -77,70 +77,78 @@ class Manifest(object):
         if len(block_content):
             self.blocks.append(block_content)
 
-    def _parse_blocks(self):
+    def _parse_all_blocks(self):
         for block in self.blocks:
-            (key, value, remain) = self.__split_line(block.pop(0))
-            if key == 'metadata':
-                # metadata block
-                for line in block:
-                    (key, value, remain) = self.__split_line(line)
-                    if key in METADATA_KEYS:
-                        self.metadata[key] = value
-                    else:
-                        raise ManifestException("Unrecognized metadata %s:" % line)
-                #validate metadata keys
-                missing_keys = set(METADATA_KEYS) - set(self.metadata.keys())
-                if missing_keys:
-                    raise ManifestException("Missing metadata keys: %s" % ','.join(missing_keys))
-                # validate vnf_release_data_time
-                try:
-                    udatetime.from_string(self.metadata['vnf_release_data_time'])
-                except ValueError:
-                    raise ManifestException("Non IETF RFC 3339 vnf_release_data_time: %s"
-                                    % self.metadata['vnf_release_data_time'])
-            elif key in DIGEST_KEYS:
-                # file digest block
-                desc = {}
+            if block[0] == 'metadata:':
+                self.parse_metadata(block)
+            elif '--BEGIN CMS--' in block[0]:
+                self.parse_cms(block)
+            else:
+                self.parse_digest(block)
+
+        if not self.metadata:
+            raise ManifestException("No metadata")
+
+    def parse_metadata(self, lines):
+        # Skip the first line
+        for line in lines[1:]:
+            (key, value, remain) = self.__split_line(line)
+            if key in METADATA_KEYS:
+                self.metadata[key] = value
+            else:
+                raise ManifestException("Unrecognized metadata %s:" % line)
+        #validate metadata keys
+        missing_keys = set(METADATA_KEYS) - set(self.metadata.keys())
+        if missing_keys:
+            raise ManifestException("Missing metadata keys: %s" % ','.join(missing_keys))
+        # validate vnf_release_data_time
+        try:
+            udatetime.from_string(self.metadata['vnf_release_data_time'])
+        except ValueError:
+            raise ManifestException("Non IETF RFC 3339 vnf_release_data_time: %s"
+                            % self.metadata['vnf_release_data_time'])
+
+    def parse_cms(self, lines):
+        if '--END CMS--' not in lines[-1]:
+            raise ManifestException("Can NOT find end of sigature block")
+        self.signature = '\n'.join(lines)
+
+    def parse_digest(self, lines):
+        desc = {}
+        for line in lines:
+            (key, value, remain) = self.__split_line(line)
+            if key in DIGEST_KEYS:
                 desc[key] = value
-                for line in block:
-                    (key, value, remain) = self.__split_line(line)
-                    if key in DIGEST_KEYS:
-                        desc[key] = value
-                    else:
-                        raise ManifestException("Unrecognized file digest line %s:" % line)
-                # validate file digest keys
-                missing_keys = set(DIGEST_KEYS) - set(desc.keys())
-                if missing_keys:
-                    raise ManifestException("Missing file digest keys: %s" % ','.join(missing_keys))
-                # validate file digest algo
+            else:
+                raise ManifestException("Unrecognized file digest line %s:" % line)
+
+            if key == 'Source':
+                self.digests[value] = (None, None)
+            elif key == 'Algorithm':
+                #validate algorithm
                 desc['Algorithm'] = desc['Algorithm'].upper()
                 if desc['Algorithm'] not in SUPPORTED_HASH_ALGO:
                     raise ManifestException("Unsupported hash algorithm: %s" % desc['Algorithm'])
-                # validate file digest hash
+
+            #validate hash
+            if desc.get('Algorithm') and desc.get('Hash') and desc.get('Source'):
                 hash = utils.cal_file_hash(self.root, desc['Source'], desc['Algorithm'])
                 if hash != desc['Hash']:
                     raise ManifestException("Mismatched hash for file %s" % desc['Source'])
-                # nothing is wrong, let's store this
+                # nothing is wrong, let's store this and start a new round
                 self.digests[desc['Source']] = (desc['Algorithm'], desc['Hash'])
-            elif key:
-                raise ManifestException("Unknown key in line '%s:%s'" % (key, value))
-            elif '--BEGIN CMS--' in remain:
-                if '--END CMS--' not in block[-1]:
-                    raise ManifestException("Can NOT find end of sigature block")
-                self.signature = remain + '\n' + '\n'.join(block)
-            else:
-                raise ManifestException("Unknown content: '%s'" % remain)
-
-        if not self.metadata:
-            raise ManifestException("No metadata")
+                desc = {}
 
     def add_file(self, rel_path, algo='SHA256'):
         '''Add file to the manifest and calculate the digest
         '''
-        algo = algo.upper()
-        if algo not in SUPPORTED_HASH_ALGO:
-            raise ManifestException("Unsupported hash algorithm: %s" % algo)
-        hash = utils.cal_file_hash(self.root, rel_path, algo)
+        if algo:
+            algo = algo.upper()
+            if algo not in SUPPORTED_HASH_ALGO:
+                raise ManifestException("Unsupported hash algorithm: %s" % algo)
+            hash = utils.cal_file_hash(self.root, rel_path, algo)
+        else:
+            hash = None
         self.digests[rel_path] = (algo, hash)
 
     def return_as_string(self):
@@ -157,8 +165,9 @@ class Manifest(object):
         for (key, digest) in six.iteritems(self.digests):
             ret += "\n"
             ret += "Source: %s\n" % key
-            ret += "Algorithm: %s\n" % digest[0]
-            ret += "Hash: %s\n" % digest[1]
+            if digest[0]:
+                ret += "Algorithm: %s\n" % digest[0]
+                ret += "Hash: %s\n" % digest[1]
         if self.digests:
             # empty line between digest and signature section
             ret += "\n"