Support of file digest in manifest file

Per sol-004, section 4.3.2, the csar manifest file should include the
digest of individual files contained in the package. This patch lays the
foundation of that support.

- Added content check of manifest file
- Added support of generating local file digest in manifest file

Change-Id: If575012d319e6f6aa0e2259e7405d8a2b6f8f338
Issue-ID: VNFSDK-174
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>

7 files changed, 319 insertions, 35 deletions

diff --git a/requirements.txt b/requirements.txt
index 0153edc..f27a466 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ ruamel.yaml<0.12.0,>=0.11.12
 requests<2.14.0,>=2.3.0
 apache-ariatosca==0.1.1
 stevedore >= 1.9.0
+udatetime<1.0,>=0.0.16
diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py
deleted file mode 100644
index a9e8dd2..0000000
--- a/tests/cli/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Copyright (c) 2017 GigaSpaces Technologies Ltd. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-
diff --git a/tests/packager/__init__.py b/tests/packager/__init__.py
deleted file mode 100644
index a9e8dd2..0000000
--- a/tests/packager/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Copyright (c) 2017 GigaSpaces Technologies Ltd. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-
diff --git a/tests/packager/test_manifest.py b/tests/packager/test_manifest.py
new file mode 100644
index 0000000..b95d7c6
--- /dev/null
+++ b/tests/packager/test_manifest.py
@@ -0,0 +1,109 @@
+# Copyright (c) 2018 Intel Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import pytest
+
+from vnfsdk_pkgtools.packager import manifest
+
+METADATA = '\n'.join(["metadata:",
+                      "vnf_product_name: test",
+                      "vnf_provider_id: test",
+                      "vnf_package_version:1.0",
+                      "vnf_release_data_time: 2017-09-15T15:00:10+08:00",
+                      ])
+
+METADATA_MISSING_KEY = '\n'.join(["metadata:",
+                                   "vnf_product_name: test",
+                                   "vnf_provider_id: test",
+                                   "vnf_package_version:1.0",
+                                ])
+
+METADATA_MISSING = "vnf_product_name: test"
+
+FILE_CONTENT = "needToBeHashed"
+FILE_DIGEST = '\n'.join(['Source: digest',
+                         'Algorithm: SHA256',
+                         'Hash: 20a480339aa4371099f9503511dcc5a8051ce3884846678ced5611ec64bbfc9c',
+                       ])
+
+def test_metadata(tmpdir):
+    p = tmpdir.mkdir('csar').join('test.mf')
+    p.write(METADATA)
+
+    m = manifest.Manifest(p.dirname, 'test.mf')
+    assert m.metadata['vnf_product_name'] == 'test'
+    assert m.metadata['vnf_provider_id'] == 'test'
+    assert m.metadata['vnf_package_version'] == '1.0'
+    assert m.metadata['vnf_release_data_time'] == '2017-09-15T15:00:10+08:00'
+
+
+def test_metadata_missing_key(tmpdir):
+    p = tmpdir.mkdir('csar').join('test.mf')
+    p.write(METADATA_MISSING_KEY)
+
+    with pytest.raises(manifest.ManifestException) as excinfo:
+        manifest.Manifest(p.dirname, 'test.mf')
+    excinfo.match(r"Missing metadata keys:")
+
+
+def test_missing_metadata(tmpdir):
+    p = tmpdir.mkdir('csar').join('test.mf')
+    p.write(METADATA_MISSING)
+
+    with pytest.raises(manifest.ManifestException) as excinfo:
+        manifest.Manifest(p.dirname, 'test.mf')
+    excinfo.match(r"Unknown key in line")
+
+def test_digest(tmpdir):
+    root = tmpdir.mkdir('csar')
+    mf = root.join('test.mf')
+    digest = root.join('digest')
+    mf.write(METADATA + '\n\n' + FILE_DIGEST)
+    digest.write(FILE_CONTENT)
+
+    m = manifest.Manifest(mf.dirname, 'test.mf')
+    assert m.digests['digest'][0] == "SHA256"
+    assert m.digests['digest'][1] == "20a480339aa4371099f9503511dcc5a8051ce3884846678ced5611ec64bbfc9c"
+
+def test_add_file(tmpdir):
+    root = tmpdir.mkdir('csar')
+    mf = root.join('test.mf')
+    digest = root.join('digest')
+    mf.write(METADATA)
+    digest.write(FILE_CONTENT)
+
+    m = manifest.Manifest(mf.dirname, 'test.mf')
+    m.add_file('digest', 'SHA256')
+    assert m.digests['digest'][0] == "SHA256"
+    assert m.digests['digest'][1] == "20a480339aa4371099f9503511dcc5a8051ce3884846678ced5611ec64bbfc9c"
+
+def test_update_to_file(tmpdir):
+    root = tmpdir.mkdir('csar')
+    mf = root.join('test.mf')
+    digest = root.join('digest')
+    mf.write(METADATA + '\n\n' + FILE_DIGEST)
+    digest.write(FILE_CONTENT)
+    digest2 = root.join('digest2')
+    digest2.write(FILE_CONTENT)
+
+    m1 = manifest.Manifest(mf.dirname, 'test.mf')
+    m1.add_file('digest2', 'SHA256')
+    m1.update_to_file()
+    m2 = manifest.Manifest(mf.dirname, 'test.mf')
+    assert m1.metadata['vnf_provider_id'] == m2.metadata['vnf_provider_id']
+    assert m1.digests['digest'] == m2.digests['digest2']
+    assert len(m2.digests.keys()) == 2
+
+    
diff --git a/tests/packager/test_utils.py b/tests/packager/test_utils.py
new file mode 100644
index 0000000..03b3f24
--- /dev/null
+++ b/tests/packager/test_utils.py
@@ -0,0 +1,28 @@
+# Copyright (c) 2018 Intel Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+
+from vnfsdk_pkgtools.packager import utils
+
+CONTENT = "needToBeHashed"
+SHA256 = "20a480339aa4371099f9503511dcc5a8051ce3884846678ced5611ec64bbfc9c"
+SHA512 = "dbed8672e752d51d0c7ca42050f67faf1534e58470bba96e787df5c4cf6a4f8ecf7ad45fb9307adbc5b9dec8432627d86b3eb1d3d43ee9c5e93f754ff2825320"
+
+def test_cal_file_hash(tmpdir):
+    p = tmpdir.join("file_to_hash.txt")
+    p.write(CONTENT)
+    assert SHA512 == utils.cal_file_hash("", str(p), 'SHA512')
+    assert SHA256 == utils.cal_file_hash(p.dirname, p.basename, 'sha256')
diff --git a/vnfsdk_pkgtools/packager/manifest.py b/vnfsdk_pkgtools/packager/manifest.py
new file mode 100644
index 0000000..a2d9d70
--- /dev/null
+++ b/vnfsdk_pkgtools/packager/manifest.py
@@ -0,0 +1,164 @@
+# Copyright (c) 2018 Intel Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+from collections import namedtuple
+import os
+
+import udatetime
+
+from vnfsdk_pkgtools.packager import utils
+
+METADATA_KEYS = [ 'vnf_provider_id',
+                  'vnf_product_name',
+                  'vnf_release_data_time',
+                  'vnf_package_version']
+DIGEST_KEYS = [ 'Source', 'Algorithm', 'Hash' ]
+SUPPORTED_HASH_ALGO = ['SHA256', 'SHA512']
+
+class ManifestException(Exception):
+    pass
+
+class Manifest(object):
+    ' Manifest file in CSAR package'
+    def __init__(self, root_path, manifest_path):
+        self.path = manifest_path
+        self.root = root_path
+        self.metadata = {}
+        # digest dict
+        #   :key = source
+        #   :value = (algorithm, hash)
+        self.digests = {}
+        self.signature = None
+        self.blocks = [ ]
+        self._split_blocks()
+        self._parse_blocks()
+
+    @staticmethod
+    def __split_line(s):
+        remain=s
+        try:
+            (key, value)=s.split(':', 1)
+            value = value.strip()
+            remain = None
+        except ValueError:
+            key = None
+            value = None
+        return (key, value, remain)
+
+    def _split_blocks(self):
+        '''
+        Split manifest file into blocks, each block is seperated by a empty
+        line or a line with only spaces and tabs.
+        '''
+        block_content = [ ]
+        with open(os.path.join(self.root, self.path),'rU') as fp:
+            for line in fp:
+                line = line.strip(' \t\n')
+                if line:
+                    block_content.append(line)
+                else:
+                    if len(block_content):
+                        self.blocks.append(block_content)
+                    block_content = []
+        if len(block_content):
+            self.blocks.append(block_content)
+
+    def _parse_blocks(self):
+        for block in self.blocks:
+            (key, value, remain) = self.__split_line(block.pop(0))
+            if key == 'metadata':
+                # metadata block
+                for line in block:
+                    (key, value, remain) = self.__split_line(line)
+                    if key in METADATA_KEYS:
+                        self.metadata[key] = value
+                    else:
+                        raise ManifestException("Unrecognized metadata %s:" % line)
+                #validate metadata keys
+                missing_keys = set(METADATA_KEYS) - set(self.metadata.keys())
+                if missing_keys:
+                    raise ManifestException("Missing metadata keys: %s" % ','.join(missing_keys))
+                # validate vnf_release_data_time
+                try:
+                    udatetime.from_string(self.metadata['vnf_release_data_time'])
+                except ValueError:
+                    raise ManifestException("Non IETF RFC 3339 vnf_release_data_time: %s"
+                                    % self.metadata['vnf_release_data_time'])
+            elif key in DIGEST_KEYS:
+                # file digest block
+                desc = {}
+                desc[key] = value
+                for line in block:
+                    (key, value, remain) = self.__split_line(line)
+                    if key in DIGEST_KEYS:
+                        desc[key] = value
+                    else:
+                        raise ManifestException("Unrecognized file digest line %s:" % line)
+                # validate file digest keys
+                missing_keys = set(DIGEST_KEYS) - set(desc.keys())
+                if missing_keys:
+                    raise ManifestException("Missing file digest keys: %s" % ','.join(missing_keys))
+                # validate file digest algo
+                desc['Algorithm'] = desc['Algorithm'].upper()
+                if desc['Algorithm'] not in SUPPORTED_HASH_ALGO:
+                    raise ManifestException("Unsupported hash algorithm: %s" % desc['Algorithm'])
+                # validate file digest hash
+                # TODO need to support remote file
+                if "://" not in desc['Source']:
+                    hash = utils.cal_file_hash(self.root, desc['Source'], desc['Algorithm'])
+                    if hash != desc['Hash']:
+                        raise ManifestException("Mismatched hash for file %s" % desc['Source'])
+                # nothing is wrong, let's store this
+                self.digests[desc['Source']] = (desc['Algorithm'], desc['Hash'])
+            elif key:
+                raise ManifestException("Unknown key in line '%s:%s'" % (key, value))
+            else:
+                # TODO signature block
+                pass
+
+        if not self.metadata:
+            raise ManifestException("No metadata")
+
+    def add_file(self, rel_path, algo='SHA256'):
+        '''Add file to the manifest and calculate the digest
+        '''
+        algo = algo.upper()
+        if algo not in SUPPORTED_HASH_ALGO:
+            raise ManifestException("Unsupported hash algorithm: %s" % algo)
+        hash = utils.cal_file_hash(self.root, rel_path, algo)
+        self.digests[rel_path] = (algo, hash)
+
+    def return_as_string(self):
+        '''Return the manifest file content as a string
+        '''
+        ret = ""
+        # metadata
+        ret += "metadata:\n"
+        ret += "vnf_product_name: %s\n" % (self.metadata['vnf_product_name'])
+        ret += "vnf_provider_id: %s\n" % (self.metadata['vnf_provider_id'])
+        ret += "vnf_package_version: %s\n" % (self.metadata['vnf_package_version'])
+        ret += "vnf_release_data_time: %s\n" % (self.metadata['vnf_release_data_time'])
+        # degist
+        for (key, digest) in self.digests.iteritems():
+            ret += "\n"
+            ret += "Source: %s\n" % key
+            ret += "Algorithm: %s\n" % digest[0]
+            ret += "Hash: %s\n" % digest[1]
+        return ret
+
+    def update_to_file(self):
+        content = self.return_as_string()
+        with open(os.path.join(self.root, self.path), 'w') as fp:
+            fp.write(content)
diff --git a/tests/__init__.py b/vnfsdk_pkgtools/packager/utils.py
index d78727c..78c7b0f 100644
--- a/tests/__init__.py
+++ b/vnfsdk_pkgtools/packager/utils.py
@@ -1,5 +1,4 @@
-#
-# Copyright (c) 2017 GigaSpaces Technologies Ltd. All rights reserved.
+# Copyright (c) 2018 Intel Corp Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -14,6 +13,21 @@
 # under the License.
 #
 
+import hashlib
 import os
 
-ROOT_DIR = os.path.dirname(os.path.dirname(__file__))
+def _hash_value_for_file(f, hash_function, block_size=2**20):
+    while True:
+        data = f.read(block_size)
+        if not data:
+            break
+        hash_function.update(data)
+
+    return hash_function.hexdigest()
+
+
+def cal_file_hash(root, path, algo):
+    with open(os.path.join(root, path), 'rb') as fp:
+        h = hashlib.new(algo)
+        return _hash_value_for_file(fp, h)
+