From e5d51b57a83f17a471b69d83c39655c1523b66d6 Mon Sep 17 00:00:00 2001 From: shabs2020 Date: Fri, 4 Jun 2021 12:08:23 +0200 Subject: Adapt DCAE adapter to work with Acumos Demeter's metadata structure change Fix the dataformat generation failure due to Demeter's metadata change. At the same maintain its backward compatibility with Acumos Clio. Issue-ID: DCAEGEN2-2825 Signed-off-by: shabs2020 Change-Id: I0f665415d1b3c94e774eec5a56c8df598d222cd3 Signed-off-by: shabs2020 --- adapter/acumos/Changelog.md | 2 + adapter/acumos/aoconversion/dataformat_gen.py | 5 +- adapter/acumos/aoconversion/spec_gen.py | 5 +- adapter/acumos/aoconversion/utils.py | 15 ++++++ adapter/acumos/pom.xml | 2 +- adapter/acumos/setup.py | 2 +- .../models/example-model-demeter/metadata.json | 61 ++++++++++++++++++++++ adapter/acumos/tests/test_validatejson.py | 53 +++++++++++++++++++ 8 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 adapter/acumos/tests/fixtures/models/example-model-demeter/metadata.json create mode 100644 adapter/acumos/tests/test_validatejson.py diff --git a/adapter/acumos/Changelog.md b/adapter/acumos/Changelog.md index 736bcea..2f8d7a3 100644 --- a/adapter/acumos/Changelog.md +++ b/adapter/acumos/Changelog.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.0.6] - 6/04/2021 + * Adapt DCAE adapter to work with Acumos Demeter's metadata structure change ## [1.0.5] - 5/05/2021 * Enhance adapter to also work behind proxy. ## [1.0.4] - 1/07/2021 diff --git a/adapter/acumos/aoconversion/dataformat_gen.py b/adapter/acumos/aoconversion/dataformat_gen.py index 71b1617..6701c7c 100644 --- a/adapter/acumos/aoconversion/dataformat_gen.py +++ b/adapter/acumos/aoconversion/dataformat_gen.py @@ -2,6 +2,7 @@ # org.onap.dcae # ============================================================================= # Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2021 highstreet technologies GmbH. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -54,8 +55,8 @@ def _get_needed_formats(meta): # we use a dict because multiple methods may reuse names needed_formats = {} for method in meta["methods"]: - needed_formats[meta["methods"][method]["input"]] = 1 - needed_formats[meta["methods"][method]["output"]] = 1 + needed_formats[utils.validate_format(meta, method, "input")] = 1 + needed_formats[utils.validate_format(meta, method, "output")] = 1 return list(needed_formats.keys()) diff --git a/adapter/acumos/aoconversion/spec_gen.py b/adapter/acumos/aoconversion/spec_gen.py index 8564434..a08b720 100644 --- a/adapter/acumos/aoconversion/spec_gen.py +++ b/adapter/acumos/aoconversion/spec_gen.py @@ -2,6 +2,7 @@ # org.onap.dcae # ============================================================================= # Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2021 highstreet technologies GmbH. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -62,7 +63,7 @@ def _generate_spec(model_name, meta, dcae_cs_schema, data_formats, docker_uri): pstype = "message_router" for method in meta["methods"]: - df_in_name = meta["methods"][method]["input"] + df_in_name = utils.validate_format(meta, method, "input") subscriber = { "config_key": "{0}_subscriber".format(method), "format": df_in_name, @@ -72,7 +73,7 @@ def _generate_spec(model_name, meta, dcae_cs_schema, data_formats, docker_uri): spec["streams"]["subscribes"].append(subscriber) - df_out_name = meta["methods"][method]["output"] + df_out_name = utils.validate_format(meta, method, "output") publisher = { "config_key": "{0}_publisher".format(method), diff --git a/adapter/acumos/aoconversion/utils.py b/adapter/acumos/aoconversion/utils.py index 7403505..65c6b95 100644 --- a/adapter/acumos/aoconversion/utils.py +++ b/adapter/acumos/aoconversion/utils.py @@ -2,6 +2,7 @@ # org.onap.dcae # ============================================================================= # Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2021 highstreet technologies GmbH. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -46,3 +47,17 @@ dataformat_schema = _Schema('schemas/dataformat.json') def get_metadata(model_repo_path, model_name): # for now, assume it's called "metadata.json" return json.loads(open("{0}/{1}/metadata.json".format(model_repo_path, model_name), "r").read()) + + +def validate_format(meta, method, type): + """ + Method to check for the metadata structure of the Acumos Model + due to change in tree structure of the input and output with Acumos-Demeter's release + Solution for Issue id: DCAEGEN2-2825 + """ + try: + df_name = meta["methods"][method][type]["name"] + + except TypeError: + df_name = meta["methods"][method][type] + return df_name diff --git a/adapter/acumos/pom.xml b/adapter/acumos/pom.xml index f0a6be0..eec4777 100644 --- a/adapter/acumos/pom.xml +++ b/adapter/acumos/pom.xml @@ -25,7 +25,7 @@ limitations under the License. 4.0.0 org.onap.dcaegen2.platform.adapter dcaegen2-platform-adapter-acumos - 1.0.5-SNAPSHOT + 1.0.6-SNAPSHOT UTF-8 . diff --git a/adapter/acumos/setup.py b/adapter/acumos/setup.py index 1af4a3f..ae56ca1 100644 --- a/adapter/acumos/setup.py +++ b/adapter/acumos/setup.py @@ -22,7 +22,7 @@ from setuptools import setup, find_packages setup( name="aoconversion", - version="1.0.5-SNAPSHOT", + version="1.0.6-SNAPSHOT", packages=find_packages(exclude=["tests.*", "tests"]), author="Tommy Carpenter, Andrew Gauld", author_email="tommy@research.att.com, agauld@att.com", diff --git a/adapter/acumos/tests/fixtures/models/example-model-demeter/metadata.json b/adapter/acumos/tests/fixtures/models/example-model-demeter/metadata.json new file mode 100644 index 0000000..4fd4695 --- /dev/null +++ b/adapter/acumos/tests/fixtures/models/example-model-demeter/metadata.json @@ -0,0 +1,61 @@ +{ + "schema": "acumos.schema.model:0.6.0", + "runtime": { + "name": "python", + "version": "3.7.9", + "dependencies": { + "pip": { + "indexes": [ + ], + "requirements": [ + { + "name": "dill", + "version": "0.3.3" + }, + { + "name": "scikit-learn", + "version": "0.23.2" + }, + { + "name": "numpy", + "version": "1.19.2" + }, + { + "name": "acumos", + "version": "0.9.7" + } + ] + }, + "conda": { + "channels": [ + ], + "requirements": [ + ] + } + } + }, + "name": "example-model", + "methods": { + "Numbers": { + "input": { + "name": "NumbersIn", + "media_type": [ + "application/vnd.google.protobuf" + ], + "metadata": { + }, + "description": "" + }, + "output": { + "name": "NumbersOut", + "media_type": [ + "application/vnd.google.protobuf" + ], + "metadata": { + }, + "description": "" + }, + "description": "" + } + } +} diff --git a/adapter/acumos/tests/test_validatejson.py b/adapter/acumos/tests/test_validatejson.py new file mode 100644 index 0000000..82f69c3 --- /dev/null +++ b/adapter/acumos/tests/test_validatejson.py @@ -0,0 +1,53 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2021 highstreet technologies GmbH. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +""" +Test the compatibility of Adapter while reading metadata from both previous (upto Clio) Acumos releases +and the new Demeter release. +""" + +import aoconversion +import testing_helpers + + +def test_validate_format_Clio(): + """ + Given the metadata as per Acumos Clio's tree structure, check if the validate_json method + reads the correct input and output name + """ + model_repo_path = testing_helpers.get_fixture_path('models') + model_name = 'example-model' + meta = aoconversion.utils.get_metadata(model_repo_path, model_name) + + for method in meta["methods"]: + assert (aoconversion.utils.validate_format(meta, method, "input")) == "NumbersIn" + assert (aoconversion.utils.validate_format(meta, method, "output")) == "NumberOut" + + +def test_validate_format_Demeter(): + """ + Given the metadata as per Acumos Clio's tree structure, check if the validate_json method + can also read the correct input and output name without fail + """ + model_repo_path = testing_helpers.get_fixture_path('models') + model_name = 'example-model-demeter' + meta = aoconversion.utils.get_metadata(model_repo_path, model_name) + + for method in meta["methods"]: + assert (aoconversion.utils.validate_format(meta, method, "input")) == "NumbersIn" + assert (aoconversion.utils.validate_format(meta, method, "output")) == "NumbersOut" -- cgit 1.2.3-korg