diff options
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | tests/cli/__init__.py | 16 | ||||
| -rw-r--r-- | tests/packager/__init__.py | 16 | ||||
| -rw-r--r-- | tests/packager/test_manifest.py | 109 | ||||
| -rw-r--r-- | tests/packager/test_utils.py | 28 | ||||
| -rw-r--r-- | vnfsdk_pkgtools/packager/manifest.py | 164 | ||||
| -rw-r--r-- | vnfsdk_pkgtools/packager/utils.py (renamed from tests/__init__.py) | 20 |
7 files changed, 319 insertions, 35 deletions
diff --git a/requirements.txt b/requirements.txt index 0153edc..f27a466 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ ruamel.yaml<0.12.0,>=0.11.12 requests<2.14.0,>=2.3.0 apache-ariatosca==0.1.1 stevedore >= 1.9.0 +udatetime<1.0,>=0.0.16 diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py deleted file mode 100644 index a9e8dd2..0000000 --- a/tests/cli/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2017 GigaSpaces Technologies Ltd. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - diff --git a/tests/packager/__init__.py b/tests/packager/__init__.py deleted file mode 100644 index a9e8dd2..0000000 --- a/tests/packager/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2017 GigaSpaces Technologies Ltd. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - diff --git a/tests/packager/test_manifest.py b/tests/packager/test_manifest.py new file mode 100644 index 0000000..b95d7c6 --- /dev/null +++ b/tests/packager/test_manifest.py @@ -0,0 +1,109 @@ +# Copyright (c) 2018 Intel Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import pytest + +from vnfsdk_pkgtools.packager import manifest + +METADATA = '\n'.join(["metadata:", + "vnf_product_name: test", + "vnf_provider_id: test", + "vnf_package_version:1.0", + "vnf_release_data_time: 2017-09-15T15:00:10+08:00", + ]) + +METADATA_MISSING_KEY = '\n'.join(["metadata:", + "vnf_product_name: test", + "vnf_provider_id: test", + "vnf_package_version:1.0", + ]) + +METADATA_MISSING = "vnf_product_name: test" + +FILE_CONTENT = "needToBeHashed" +FILE_DIGEST = '\n'.join(['Source: digest', + 'Algorithm: SHA256', + 'Hash: 20a480339aa4371099f9503511dcc5a8051ce3884846678ced5611ec64bbfc9c', + ]) + +def test_metadata(tmpdir): + p = tmpdir.mkdir('csar').join('test.mf') + p.write(METADATA) + + m = manifest.Manifest(p.dirname, 'test.mf') + assert m.metadata['vnf_product_name'] == 'test' + assert m.metadata['vnf_provider_id'] == 'test' + assert m.metadata['vnf_package_version'] == '1.0' + assert m.metadata['vnf_release_data_time'] == '2017-09-15T15:00:10+08:00' + + +def test_metadata_missing_key(tmpdir): + p = tmpdir.mkdir('csar').join('test.mf') + p.write(METADATA_MISSING_KEY) + + with pytest.raises(manifest.ManifestException) as excinfo: + manifest.Manifest(p.dirname, 'test.mf') + excinfo.match(r"Missing metadata keys:") + + +def test_missing_metadata(tmpdir): + p = tmpdir.mkdir('csar').join('test.mf') + p.write(METADATA_MISSING) + + with pytest.raises(manifest.ManifestException) as excinfo: + manifest.Manifest(p.dirname, 'test.mf') + excinfo.match(r"Unknown key in line") + +def test_digest(tmpdir): + root = tmpdir.mkdir('csar') + mf = root.join('test.mf') + digest = root.join('digest') + mf.write(METADATA + '\n\n' + FILE_DIGEST) + digest.write(FILE_CONTENT) + + m = manifest.Manifest(mf.dirname, 'test.mf') + assert m.digests['digest'][0] == "SHA256" + assert m.digests['digest'][1] == "20a480339aa4371099f9503511dcc5a8051ce3884846678ced5611ec64bbfc9c" + +def test_add_file(tmpdir): + root = tmpdir.mkdir('csar') + mf = root.join('test.mf') + digest = root.join('digest') + mf.write(METADATA) + digest.write(FILE_CONTENT) + + m = manifest.Manifest(mf.dirname, 'test.mf') + m.add_file('digest', 'SHA256') + assert m.digests['digest'][0] == "SHA256" + assert m.digests['digest'][1] == "20a480339aa4371099f9503511dcc5a8051ce3884846678ced5611ec64bbfc9c" + +def test_update_to_file(tmpdir): + root = tmpdir.mkdir('csar') + mf = root.join('test.mf') + digest = root.join('digest') + mf.write(METADATA + '\n\n' + FILE_DIGEST) + digest.write(FILE_CONTENT) + digest2 = root.join('digest2') + digest2.write(FILE_CONTENT) + + m1 = manifest.Manifest(mf.dirname, 'test.mf') + m1.add_file('digest2', 'SHA256') + m1.update_to_file() + m2 = manifest.Manifest(mf.dirname, 'test.mf') + assert m1.metadata['vnf_provider_id'] == m2.metadata['vnf_provider_id'] + assert m1.digests['digest'] == m2.digests['digest2'] + assert len(m2.digests.keys()) == 2 + + diff --git a/tests/packager/test_utils.py b/tests/packager/test_utils.py new file mode 100644 index 0000000..03b3f24 --- /dev/null +++ b/tests/packager/test_utils.py @@ -0,0 +1,28 @@ +# Copyright (c) 2018 Intel Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import os + +from vnfsdk_pkgtools.packager import utils + +CONTENT = "needToBeHashed" +SHA256 = "20a480339aa4371099f9503511dcc5a8051ce3884846678ced5611ec64bbfc9c" +SHA512 = "dbed8672e752d51d0c7ca42050f67faf1534e58470bba96e787df5c4cf6a4f8ecf7ad45fb9307adbc5b9dec8432627d86b3eb1d3d43ee9c5e93f754ff2825320" + +def test_cal_file_hash(tmpdir): + p = tmpdir.join("file_to_hash.txt") + p.write(CONTENT) + assert SHA512 == utils.cal_file_hash("", str(p), 'SHA512') + assert SHA256 == utils.cal_file_hash(p.dirname, p.basename, 'sha256') diff --git a/vnfsdk_pkgtools/packager/manifest.py b/vnfsdk_pkgtools/packager/manifest.py new file mode 100644 index 0000000..a2d9d70 --- /dev/null +++ b/vnfsdk_pkgtools/packager/manifest.py @@ -0,0 +1,164 @@ +# Copyright (c) 2018 Intel Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +from collections import namedtuple +import os + +import udatetime + +from vnfsdk_pkgtools.packager import utils + +METADATA_KEYS = [ 'vnf_provider_id', + 'vnf_product_name', + 'vnf_release_data_time', + 'vnf_package_version'] +DIGEST_KEYS = [ 'Source', 'Algorithm', 'Hash' ] +SUPPORTED_HASH_ALGO = ['SHA256', 'SHA512'] + +class ManifestException(Exception): + pass + +class Manifest(object): + ' Manifest file in CSAR package' + def __init__(self, root_path, manifest_path): + self.path = manifest_path + self.root = root_path + self.metadata = {} + # digest dict + # :key = source + # :value = (algorithm, hash) + self.digests = {} + self.signature = None + self.blocks = [ ] + self._split_blocks() + self._parse_blocks() + + @staticmethod + def __split_line(s): + remain=s + try: + (key, value)=s.split(':', 1) + value = value.strip() + remain = None + except ValueError: + key = None + value = None + return (key, value, remain) + + def _split_blocks(self): + ''' + Split manifest file into blocks, each block is seperated by a empty + line or a line with only spaces and tabs. + ''' + block_content = [ ] + with open(os.path.join(self.root, self.path),'rU') as fp: + for line in fp: + line = line.strip(' \t\n') + if line: + block_content.append(line) + else: + if len(block_content): + self.blocks.append(block_content) + block_content = [] + if len(block_content): + self.blocks.append(block_content) + + def _parse_blocks(self): + for block in self.blocks: + (key, value, remain) = self.__split_line(block.pop(0)) + if key == 'metadata': + # metadata block + for line in block: + (key, value, remain) = self.__split_line(line) + if key in METADATA_KEYS: + self.metadata[key] = value + else: + raise ManifestException("Unrecognized metadata %s:" % line) + #validate metadata keys + missing_keys = set(METADATA_KEYS) - set(self.metadata.keys()) + if missing_keys: + raise ManifestException("Missing metadata keys: %s" % ','.join(missing_keys)) + # validate vnf_release_data_time + try: + udatetime.from_string(self.metadata['vnf_release_data_time']) + except ValueError: + raise ManifestException("Non IETF RFC 3339 vnf_release_data_time: %s" + % self.metadata['vnf_release_data_time']) + elif key in DIGEST_KEYS: + # file digest block + desc = {} + desc[key] = value + for line in block: + (key, value, remain) = self.__split_line(line) + if key in DIGEST_KEYS: + desc[key] = value + else: + raise ManifestException("Unrecognized file digest line %s:" % line) + # validate file digest keys + missing_keys = set(DIGEST_KEYS) - set(desc.keys()) + if missing_keys: + raise ManifestException("Missing file digest keys: %s" % ','.join(missing_keys)) + # validate file digest algo + desc['Algorithm'] = desc['Algorithm'].upper() + if desc['Algorithm'] not in SUPPORTED_HASH_ALGO: + raise ManifestException("Unsupported hash algorithm: %s" % desc['Algorithm']) + # validate file digest hash + # TODO need to support remote file + if "://" not in desc['Source']: + hash = utils.cal_file_hash(self.root, desc['Source'], desc['Algorithm']) + if hash != desc['Hash']: + raise ManifestException("Mismatched hash for file %s" % desc['Source']) + # nothing is wrong, let's store this + self.digests[desc['Source']] = (desc['Algorithm'], desc['Hash']) + elif key: + raise ManifestException("Unknown key in line '%s:%s'" % (key, value)) + else: + # TODO signature block + pass + + if not self.metadata: + raise ManifestException("No metadata") + + def add_file(self, rel_path, algo='SHA256'): + '''Add file to the manifest and calculate the digest + ''' + algo = algo.upper() + if algo not in SUPPORTED_HASH_ALGO: + raise ManifestException("Unsupported hash algorithm: %s" % algo) + hash = utils.cal_file_hash(self.root, rel_path, algo) + self.digests[rel_path] = (algo, hash) + + def return_as_string(self): + '''Return the manifest file content as a string + ''' + ret = "" + # metadata + ret += "metadata:\n" + ret += "vnf_product_name: %s\n" % (self.metadata['vnf_product_name']) + ret += "vnf_provider_id: %s\n" % (self.metadata['vnf_provider_id']) + ret += "vnf_package_version: %s\n" % (self.metadata['vnf_package_version']) + ret += "vnf_release_data_time: %s\n" % (self.metadata['vnf_release_data_time']) + # degist + for (key, digest) in self.digests.iteritems(): + ret += "\n" + ret += "Source: %s\n" % key + ret += "Algorithm: %s\n" % digest[0] + ret += "Hash: %s\n" % digest[1] + return ret + + def update_to_file(self): + content = self.return_as_string() + with open(os.path.join(self.root, self.path), 'w') as fp: + fp.write(content) diff --git a/tests/__init__.py b/vnfsdk_pkgtools/packager/utils.py index d78727c..78c7b0f 100644 --- a/tests/__init__.py +++ b/vnfsdk_pkgtools/packager/utils.py @@ -1,5 +1,4 @@ -# -# Copyright (c) 2017 GigaSpaces Technologies Ltd. All rights reserved. +# Copyright (c) 2018 Intel Corp Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,6 +13,21 @@ # under the License. # +import hashlib import os -ROOT_DIR = os.path.dirname(os.path.dirname(__file__)) +def _hash_value_for_file(f, hash_function, block_size=2**20): + while True: + data = f.read(block_size) + if not data: + break + hash_function.update(data) + + return hash_function.hexdigest() + + +def cal_file_hash(root, path, algo): + with open(os.path.join(root, path), 'rb') as fp: + h = hashlib.new(algo) + return _hash_value_for_file(fp, h) + |