From 849da15d5b7ddc68e4c2b90b603fc8948d4b5e6d Mon Sep 17 00:00:00 2001 From: Andrew Gauld Date: Wed, 13 Nov 2019 18:09:32 +0000 Subject: Add acumos adapter project Signed-off-by: Andrew Gauld Issue-ID: DCAEGEN2-1860 Change-Id: Ib22fd2aa61fe7761bacf85e69540d11803c7acee Signed-off-by: Andrew Gauld --- adapter/acumos/Changelog.md | 8 + adapter/acumos/Dockerfile | 34 + adapter/acumos/LICENSE.txt | 17 + adapter/acumos/README.md | 80 ++ adapter/acumos/aoconversion/__init__.py | 0 adapter/acumos/aoconversion/adapter.py | 50 ++ adapter/acumos/aoconversion/convert.py | 33 + adapter/acumos/aoconversion/dataformat_gen.py | 155 ++++ adapter/acumos/aoconversion/docker_gen.py | 110 +++ adapter/acumos/aoconversion/exceptions.py | 29 + adapter/acumos/aoconversion/index.html | 254 ++++++ adapter/acumos/aoconversion/scanner.py | 312 ++++++++ adapter/acumos/aoconversion/spec_gen.py | 112 +++ adapter/acumos/aoconversion/utils.py | 24 + adapter/acumos/pom.xml | 37 + adapter/acumos/setup.py | 36 + adapter/acumos/tests/fixtures/README.md | 2 + adapter/acumos/tests/fixtures/dataformat_101.json | 212 +++++ .../dcae-cli-v2_component-spec-schema.json | 860 +++++++++++++++++++++ adapter/acumos/tests/fixtures/jsdraft4schema.json | 149 ++++ .../models/ArgsList_1.0.0_dcae_data_format.json | 1 + .../models/SumOut_1.0.0_dcae_data_format.json | 1 + .../models/example-model-listofm/example_model.py | 41 + .../models/example-model-listofm/metadata.json | 1 + .../models/example-model-listofm/model.proto | 39 + .../fixtures/models/example-model/metadata.json | 35 + .../fixtures/models/example-model/model.proto | 35 + adapter/acumos/tests/test_df.py | 109 +++ adapter/acumos/tests/test_docker.py | 47 ++ adapter/acumos/tests/test_fed.py | 180 +++++ adapter/acumos/tests/test_spec.py | 55 ++ adapter/acumos/tests/testing_helpers.py | 31 + adapter/acumos/tox.ini | 44 ++ 33 files changed, 3133 insertions(+) create mode 100644 adapter/acumos/Changelog.md create mode 100644 adapter/acumos/Dockerfile create mode 100644 adapter/acumos/LICENSE.txt create mode 100644 adapter/acumos/README.md create mode 100644 adapter/acumos/aoconversion/__init__.py create mode 100644 adapter/acumos/aoconversion/adapter.py create mode 100644 adapter/acumos/aoconversion/convert.py create mode 100644 adapter/acumos/aoconversion/dataformat_gen.py create mode 100644 adapter/acumos/aoconversion/docker_gen.py create mode 100644 adapter/acumos/aoconversion/exceptions.py create mode 100644 adapter/acumos/aoconversion/index.html create mode 100644 adapter/acumos/aoconversion/scanner.py create mode 100644 adapter/acumos/aoconversion/spec_gen.py create mode 100644 adapter/acumos/aoconversion/utils.py create mode 100644 adapter/acumos/pom.xml create mode 100644 adapter/acumos/setup.py create mode 100644 adapter/acumos/tests/fixtures/README.md create mode 100644 adapter/acumos/tests/fixtures/dataformat_101.json create mode 100644 adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json create mode 100644 adapter/acumos/tests/fixtures/jsdraft4schema.json create mode 100644 adapter/acumos/tests/fixtures/models/ArgsList_1.0.0_dcae_data_format.json create mode 100644 adapter/acumos/tests/fixtures/models/SumOut_1.0.0_dcae_data_format.json create mode 100644 adapter/acumos/tests/fixtures/models/example-model-listofm/example_model.py create mode 100644 adapter/acumos/tests/fixtures/models/example-model-listofm/metadata.json create mode 100644 adapter/acumos/tests/fixtures/models/example-model-listofm/model.proto create mode 100644 adapter/acumos/tests/fixtures/models/example-model/metadata.json create mode 100644 adapter/acumos/tests/fixtures/models/example-model/model.proto create mode 100644 adapter/acumos/tests/test_df.py create mode 100644 adapter/acumos/tests/test_docker.py create mode 100644 adapter/acumos/tests/test_fed.py create mode 100644 adapter/acumos/tests/test_spec.py create mode 100644 adapter/acumos/tests/testing_helpers.py create mode 100644 adapter/acumos/tox.ini (limited to 'adapter') diff --git a/adapter/acumos/Changelog.md b/adapter/acumos/Changelog.md new file mode 100644 index 0000000..0823a8e --- /dev/null +++ b/adapter/acumos/Changelog.md @@ -0,0 +1,8 @@ +# Change Log +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [1.0.0] - 11/13/2019 + * Onboard models from Acumos - initial version diff --git a/adapter/acumos/Dockerfile b/adapter/acumos/Dockerfile new file mode 100644 index 0000000..7ec9656 --- /dev/null +++ b/adapter/acumos/Dockerfile @@ -0,0 +1,34 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +FROM python:3.7 +COPY setup.py /tmp/build/ +COPY aoconversion/ /tmp/build/aoconversion/ +RUN apt-get update && \ + apt-get install -y npm nodejs && \ + npm -v && \ + npm install -g protobuf-jsonschema && \ + cd /tmp/build/ && \ + python setup.py install && \ + cd / && \ + rm -rf /tmp/* + +EXPOSE 9000 +ENV PYTHONUNBUFFERED TRUE +ENTRYPOINT [ "/usr/local/bin/acumos-adapter" ] +CMD [ "/run/config/config.yaml" ] diff --git a/adapter/acumos/LICENSE.txt b/adapter/acumos/LICENSE.txt new file mode 100644 index 0000000..de62c61 --- /dev/null +++ b/adapter/acumos/LICENSE.txt @@ -0,0 +1,17 @@ +============LICENSE_START======================================================= +org.onap.dcae +================================================================================ +Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +================================================================================ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +============LICENSE_END========================================================= diff --git a/adapter/acumos/README.md b/adapter/acumos/README.md new file mode 100644 index 0000000..55490ba --- /dev/null +++ b/adapter/acumos/README.md @@ -0,0 +1,80 @@ +# Acumos Model DCAE Component + +This package is a service that converts Acumos models into ONAP DCAE components. + +This operates in 2 modes: + +- In command line mode, the catalogs and Acumos instance providing the models + to be converted are identified in the configuration file, and the service + exits once they have been processed. +- In web UI mode, a TCP/IP port to listen on is identified in the + configuration file, this port provides access to a web UI for identifying + the models to be converted, and the service continues to handle requests + after processing models. + +# Unit Testing + + tox + open htmlcov/index.html + +# Runtime Pre-Requisites + +- An ACUMOS instance from which to pull models. +- A PEM format file containing the unencrypted private key, certificate, and + any necessary intermediate certificate(s) used to identify this tool to + the above ACUMOS instance. +- Any required DNS/Firewall setup such that this tool will be able to connect + to the Federation Gateway of the ACUMOS instance. +- The tool must be able to access the Docker daemon +- The Docker daemon must have connectivity to the Docker registry used by ONAP. +- The credentials the Docker daemon will use to write the Docker registry must + be known. +- The tool must be able to access the DCAE onboarding service API +- The credentials the tool will use to invoke the DCAE onboarding service must + be known. +- The tool must be able to access web sites containing the DCAE data format and + component JSON schemas. +- "pip install nodeenv", "nodeenv -p", "npm install --global protobuf-jsonschema", + and "pip install aoconversion", or equivalent must be run. +- The DCAE user who will "own" the loaded data formats and components must be + known. +- If the certificate chain of the ACUMOS instance's server certificate does not + lead to a well known certificate authority, then a PEM format file containing + the appropriate CA certificate must be available. +- A configuration file, in YAML format, containing the following keys must be + available. + + dcaeurl - The base URL for the DCAE component and data format schemas. For + example, https://git.onap.org/dcaegen2/platform/cli/plain. + dcaeuser - The DCAE user who will "own" the loaded data. + onboardingurl - The URL for accessing the onboarding service. + onboardinguser - The user ID for accessing the onboarding service. + onboardingpass - The password for accessing the onboarding service. + port - (required in web UI mode, not allowed in command-line mode) The TCP/IP + port to listen on in web UI mode. + acumosurl - (required in command-line mode) The URL for the Federation + Gateway of the ACUMOS instance. + certfile - The file path for the PEM file containing the private key, etc. + dockerhost - (optional) The URL for the docker host. By default, + unix:///var/run/docker.sock. + dockerregistry - The host:port for the ONAP docker registry. + dockeruser - The user ID for uploading images to the docker registry. + dockerpass - The password for uploading images to the docker registry. + certverify - (optional) The PEM file containing the CA certificate. By + default, a standard set of certificate authorities are recognized. + tmpdir - (optional) A directory in which to work. By default, + temporary files are put in /var/tmp/aoadapter. + interval - (optional) The number of seconds between scans of the ACUMOS + instance. By default, 900 seconds (15 minutes). + catalogs - (optional) a list of catalog IDs or catalog names to load. By + default, all catalogs are loaded. + + Note: The values of onboardingpass and dockerpass can either be the literal + passwords to be used or, if they begin with "@" they can specify the paths + to files containing the passwords. + +# Running + +Assuming the name of the above configuration file is "config.yaml," + + acumos-adapter config.yaml diff --git a/adapter/acumos/aoconversion/__init__.py b/adapter/acumos/aoconversion/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/adapter/acumos/aoconversion/adapter.py b/adapter/acumos/aoconversion/adapter.py new file mode 100644 index 0000000..38b5a71 --- /dev/null +++ b/adapter/acumos/aoconversion/adapter.py @@ -0,0 +1,50 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +""" +Command line for Acumos -> ONAP adapter +""" + +import os +import shutil +import sys +import yaml + + +def adapter(): + """ + Run the adapter + """ + import warnings + warnings.simplefilter('ignore') + with open(sys.argv[1], 'r') as f: + ycfg = yaml.safe_load(f.read()) + from aoconversion import scanner + config = scanner.Config(**ycfg) + try: + shutil.rmtree(config.tmpdir) + except Exception: + pass + os.makedirs(config.tmpdir) + if config.port: + print('Starting web server') + scanner.serve(config) + else: + print('Starting scan') + scanner.scan(config) + print('Scan complete. Sleeping') diff --git a/adapter/acumos/aoconversion/convert.py b/adapter/acumos/aoconversion/convert.py new file mode 100644 index 0000000..e41820a --- /dev/null +++ b/adapter/acumos/aoconversion/convert.py @@ -0,0 +1,33 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +""" +Module that converts acumos models to dcae artifacts +""" +from aoconversion import docker_gen, dataformat_gen, spec_gen + + +def gen_dcae_artifacts_for_model(config, model_name, model_version="latest"): + """ + Generate all dcae artifacts given an acumos model + """ + model_repo_path = config.tmpdir + docker_uri = docker_gen.build_and_push_docker(config, model_name, model_version) + data_formats = dataformat_gen.generate_dcae_data_formats(model_repo_path, model_name) + spec = spec_gen.generate_spec(model_repo_path, model_name, data_formats, docker_uri) + return docker_uri, data_formats, spec diff --git a/adapter/acumos/aoconversion/dataformat_gen.py b/adapter/acumos/aoconversion/dataformat_gen.py new file mode 100644 index 0000000..a5ead96 --- /dev/null +++ b/adapter/acumos/aoconversion/dataformat_gen.py @@ -0,0 +1,155 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +from subprocess import PIPE, Popen +import json +from jsonschema import validate +import requests +from aoconversion import utils, exceptions + + +def _get_js_schema(): + res = requests.get("http://json-schema.org/draft-04/schema#") + return res.json() + + +def _get_dcae_df_schema(): + res = requests.get( + "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD" + ) + return res.json() + + +def _protobuf_to_js(proto_path): + """ + Converts a protobuf to jsonschema and returns the generated schema as a JSON object. + """ + cmd = ["protobuf-jsonschema", proto_path] + p = Popen(cmd, stderr=PIPE, stdout=PIPE) + out = p.stdout.read() + asjson = json.loads(out) + + # change the defintion names to remove the random package name that acumos generates + defs = asjson["definitions"] + defns = list(defs.keys()) + for defn in defns: + # https://stackoverflow.com/questions/16475384/rename-a-dictionary-key + defs[defn.split(".")[1]] = defs.pop(defn) + + # make sure what we got out is a valid jsonschema + draft4 = _get_js_schema() + validate(instance=asjson, schema=draft4) + + return asjson + + +def _get_needed_formats(meta): + """ + Read the metadata and figure out what the principle data formats are. + We cannot determine this from the proto because the proto may list "submessages" in a flat namespace; some of them may not coorespond to a data format but rather a referenced defintion in another. + We don't want to generate a data format for submessages though; instead they should be included in definitions as part of the relevent data format + """ + # we use a dict because multiple methods may reuse names + needed_formats = {} + for method in meta["methods"]: + needed_formats[meta["methods"][method]["input"]] = 1 + needed_formats[meta["methods"][method]["output"]] = 1 + return list(needed_formats.keys()) + + +def _generate_dcae_data_formats(proto_path, meta, dcae_df_schema, draft_4_schema): + """ + Generates a collection of data formats from the model .proto + This helper function is broken out for the ease of unit testing; this can be unit tested easily because all deps are parameters, + but generate_dcae_data_formats requires some mocking etc. + """ + js = _protobuf_to_js(proto_path) + needed_formats = _get_needed_formats(meta) + + data_formats = [] + + used_defns = [] + + # iterate over and convert + for nf in needed_formats: + defn = js["definitions"][nf] + + definitions = {} + + # check for the case where we have an array of other defns + for prop in defn["properties"]: + if defn["properties"][prop]["type"] == "array" and "$ref" in defn["properties"][prop]["items"]: + unclean_ref_name = defn["properties"][prop]["items"]["$ref"] + clean_ref_name = unclean_ref_name.split(".")[1] + if clean_ref_name in js["definitions"]: + defn["properties"][prop]["items"]["$ref"] = "#/definitions/{0}".format(clean_ref_name) + definitions[clean_ref_name] = js["definitions"][clean_ref_name] + used_defns.append(clean_ref_name) + else: # this is bad/unsupported, investigate + raise exceptions.UnsupportedFormatScenario() + + # the defns created by this tool do not include a schema field. + # I created an issue: https://github.com/devongovett/protobuf-jsonschema/issues/12 + defn["$schema"] = "http://json-schema.org/draft-04/schema#" + + # Include the definitions, which may be empty {} + defn["definitions"] = definitions + + # Validate that our resulting jsonschema is valid jsonschema + validate(instance=defn, schema=draft_4_schema) + + # we currently hardcode dataformatversion, since it is the latest and has been for years https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD + dcae_df = {"self": {"name": nf, "version": "1.0.0"}, "dataformatversion": "1.0.1", "jsonschema": defn} + + # make sure the schema validates against the DCAE data format schema + validate(instance=dcae_df, schema=dcae_df_schema) + + # if we've passed the validation and exc raising so far, we are good, append this to output list of dcae data formats + data_formats.append(dcae_df) + + # make sure every definitin we got out was used. Otherwise, this requires investigation!! + if sorted(needed_formats + used_defns) != sorted(list(js["definitions"].keys())): + raise exceptions.UnsupportedFormatScenario() + + return data_formats + + +# Public + + +def generate_dcae_data_formats(model_repo_path, model_name): + """ + Generates a collection of data formats from the model .proto + Writes them to disk + Returns them as the return of this call so this can be fed directly into spec gen + """ + data_formats = _generate_dcae_data_formats( + "{0}/{1}/model.proto".format(model_repo_path, model_name), + utils.get_metadata(model_repo_path, model_name), + _get_dcae_df_schema(), + _get_js_schema(), + ) + + # now we iterate over these and write a file to disk for each, since the dcae cli seems to want that + for df in data_formats: + # name_version seems like a reasonable filename + fname = "{0}_{1}_dcae_data_format.json".format(df["self"]["name"], df["self"]["version"]) + with open("{0}/{1}".format(model_repo_path, fname), "w") as f: + f.write(json.dumps(df)) + + return data_formats diff --git a/adapter/acumos/aoconversion/docker_gen.py b/adapter/acumos/aoconversion/docker_gen.py new file mode 100644 index 0000000..bf54cfc --- /dev/null +++ b/adapter/acumos/aoconversion/docker_gen.py @@ -0,0 +1,110 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import os +from docker import APIClient +from aoconversion import exceptions, utils + + +def _generate_dockerfile(meta, model_name): + """ + bind the templated docker string + """ + docker_template = """ + FROM python:{VERSION} + + ENV MODELNAME {MODELNAME} + RUN mkdir /app + WORKDIR /app + + ADD ./{MODELNAME} /app/{MODELNAME} + ADD ./requirements.txt /app + + RUN pip install -r /app/requirements.txt && \ + pip install acumos_dcae_model_runner + + ENV DCAEPORT=10000 + EXPOSE $DCAEPORT + + ENTRYPOINT ["acumos_dcae_model_runner"] + CMD ["/app/{MODELNAME}"] + """ + python_version = meta["runtime"]["version"] + return docker_template.format(VERSION=python_version, MODELNAME=model_name) + + +# Public + + +def build_and_push_docker(config, model_name, model_version="latest"): + """ + build and push the dcae docker container + Returns the docker uri so this can be pipelined into specgen + """ + model_repo_path = config.tmpdir + dockerfile_path = "{0}/Dockerfile".format(model_repo_path) + reqs_path = "{0}/requirements.txt".format(model_repo_path) + + # get the metadata + meta = utils.get_metadata(model_repo_path, model_name) + + # write the reqs file, will be removed later + reqs = meta["runtime"]["dependencies"]["pip"]["requirements"] + with open(reqs_path, "w") as f: + for r in reqs: + f.write("{0}=={1}\n".format(r["name"], r["version"])) + + # generate the dockerfile + dockerfile = _generate_dockerfile(meta, model_name) + + # write the dockerfile, will be removed later + with open("{0}/Dockerfile".format(model_repo_path), "w") as f: + f.write(dockerfile) + + docker_uri = "{0}/{1}:{2}".format(config.dockerregistry, model_name, model_version) + + # do the docker build + cli = APIClient(base_url=config.dockerhost, user_agent="Docker-Client-xx.yy") + response = [line.decode() for line in cli.build(path=model_repo_path, rm=True, tag=docker_uri)] + + # clean up the files + os.remove(dockerfile_path) + os.remove(reqs_path) + + # parse the Docker response to see whether we succeeded + for r in response: + # In some scenarios, for example a non-existing Dockerfile, docker build raises a native exception, see: + # https://docker-py.readthedocs.io/en/stable/api.html#module-docker.api.build + # However, if something fails such as a non-existing directory referenced in the dockerfile, NO exception is raised. + # In this case, one of the console output lines is "errorDetail".... + if "errorDetail" in r: + raise exceptions.DockerBuildFail(r) + + # if the above succeeded, we can push + # push. same problem as above; e.g., "no basic auth credentials" does not throw an exception! + response = [ + line.decode() + for line in cli.push( + repository=docker_uri, auth_config={"username": config.dockeruser, "password": config.dockerpass()}, stream=True + ) + ] + for r in response: + if "errorDetail" in r: + raise exceptions.DockerPushFail(r) + + return docker_uri diff --git a/adapter/acumos/aoconversion/exceptions.py b/adapter/acumos/aoconversion/exceptions.py new file mode 100644 index 0000000..8e87458 --- /dev/null +++ b/adapter/acumos/aoconversion/exceptions.py @@ -0,0 +1,29 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + + +class DockerBuildFail(BaseException): + pass + + +class DockerPushFail(BaseException): + pass + + +class UnsupportedFormatScenario(BaseException): + pass diff --git a/adapter/acumos/aoconversion/index.html b/adapter/acumos/aoconversion/index.html new file mode 100644 index 0000000..c94621a --- /dev/null +++ b/adapter/acumos/aoconversion/index.html @@ -0,0 +1,254 @@ + + + + +Acumos-ONAP Adapter Demo + + +
+ +
+
+

Import Model

+ +
+
+ + +
+
+
+ +
+
+
+ +
+
+
+ +
+
+ + + Onboarding - Please Wait ... +
+ + + diff --git a/adapter/acumos/aoconversion/scanner.py b/adapter/acumos/aoconversion/scanner.py new file mode 100644 index 0000000..22a5922 --- /dev/null +++ b/adapter/acumos/aoconversion/scanner.py @@ -0,0 +1,312 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import json +import os +from pkg_resources import resource_string +import shutil +import traceback +try: + from urllib.parse import unquote_plus + from socketserver import ThreadingMixIn + from http.server import BaseHTTPRequestHandler, HTTPServer +except ImportError: + from urllib import unquote_plus + from SocketServer import ThreadingMixIn + from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + +import requests + +from aoconversion import convert + + +def _derefconfig(value): + if value.startswith('@'): + with open(value[1:], 'r') as f: + return f.readline().strip() + return value + + +class Config(object): + """ + Configuration parameters as attributes, make sure the required ones are there, + populate defaults. + """ + def __init__(self, dcaeurl, dcaeuser, onboardingurl, onboardinguser, onboardingpass, certfile, dockerregistry, dockeruser, dockerpass, acumosurl=None, interval=900, dockerhost='unix:///var/run/docker.sock', tmpdir='/var/tmp/aoadapter', certverify=True, catalogs=None, port=None, **extras): + self.dcaeurl = dcaeurl + self.dcaeuser = dcaeuser + + def x(fmt, *args, **kwargs): + return onboardingurl + fmt.format(*args, **kwargs) + self.oburl = x + self._onboardingpass = onboardingpass + self._onboardinguser = onboardinguser + self.acumosurl = acumosurl + self.certfile = certfile + self.certverify = certverify + self.dockerhost = dockerhost + self.dockerregistry = dockerregistry + self.dockeruser = dockeruser + self._dockerpass = dockerpass + self.interval = interval + self.tmpdir = tmpdir + if catalogs is not None and type(catalogs) is not list: + catalogs = [catalogs] + self.catalogs = catalogs + self.port = port + + def obauth(self): + return (self._onboardinguser, _derefconfig(self._onboardingpass)) + + def dockerpass(self): + return _derefconfig(self._dockerpass) + + +class _AcumosAccess(object): + def __init__(self, config, url): + self.cert = config.certfile + self.verify = config.certverify + self.url = url.strip().rstrip('/') + + def artgetter(self, xrev, matcher): + for art in xrev['artifacts']: + if matcher(art): + def ret(): + nurl = '{}/artifacts/{}/content'.format(self.url, art['artifactId']) + resp = requests.get(nurl, stream=True, cert=self.cert, verify=self.verify) + if resp.status_code == 500: + resp = requests.get(nurl, stream=True, cert=self.cert, verify=self.verify) + resp.raise_for_status() + return resp + return ret + return None + + def jsonget(self, format, *args, **kwargs): + nurl = self.url + format.format(*args, **kwargs) + resp = requests.get(nurl, cert=self.cert, verify=self.verify) + if resp.status_code == 500: + resp = requests.get(nurl, cert=self.cert, verify=self.verify) + resp.raise_for_status() + return resp.json()['content'] + + +def _x_proto_matcher(art): + """ Is this artifact the x.proto file? """ + return art['name'].endswith('.proto') + + +def _x_zip_matcher(art): + """ Is this artifact the x.zip file? """ + return art['name'].endswith('.zip') + + +def _md_json_matcher(art): + """ Is this artifact the metadata.json file? """ + return art['name'].endswith('.json') + + +def _walk(config): + """ + Walk the Federation E5 interface of an Acumos instance + """ + url = config.acumosurl + callback = _makecallback(config) + catalogs = config.catalogs + aa = _AcumosAccess(config, url) + for catalog in aa.jsonget('/catalogs'): + if catalogs is not None and catalog['catalogId'] not in catalogs and catalog['name'] not in catalogs: + continue + for solution in aa.jsonget('/solutions?catalogId={}', catalog['catalogId']): + for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']): + onboard(aa, callback, solution, revision['revisionId']) + + +def onboard(aa, callback, solution, revid): + xrev = aa.jsonget('/solutions/{}/revisions/{}', solution['solutionId'], revid) + callback(model_name=solution['name'], model_version=xrev['version'], model_last_updated=xrev['modified'], rating=solution['ratingAverageTenths'] / 10.0, proto_getter=aa.artgetter(xrev, _x_proto_matcher), zip_getter=aa.artgetter(xrev, _x_zip_matcher), metadata_getter=aa.artgetter(xrev, _md_json_matcher)) + + +def _pullfile(source, dest): + with open(dest, 'wb') as f: + for chunk in source().iter_content(65536): + f.write(chunk) + + +_loadedformats = set() +_loadedcomponents = set() + + +def scan(config): + _walk(config) + + +def _makecallback(config): + workdir = config.tmpdir + obauth = config.obauth() + oburl = config.oburl + + def callback(model_name, model_version, model_last_updated, rating, proto_getter, zip_getter, metadata_getter): + model_name = model_name.lower() + model_version = model_version.lower() + compid = (model_name, model_version) + if compid in _loadedcomponents: + print('Skipping component {}: already analyzed'.format(compid)) + return + if proto_getter is None or zip_getter is None or metadata_getter is None: + print('Skipping component {}: does not have required artifacts'.format(compid)) + _loadedcomponents.add(compid) + return + modeldir = '{}/{}'.format(workdir, model_name) + shutil.rmtree(modeldir, True) + os.makedirs(modeldir) + try: + _pullfile(proto_getter, '{}/model.proto'.format(modeldir)) + _pullfile(zip_getter, '{}/model.zip'.format(modeldir)) + _pullfile(metadata_getter, '{}/metadata.json'.format(modeldir)) + except Exception: + print('Skipping component {}: artifact access error'.format(compid)) + _loadedcomponents.add(compid) + return + try: + docker_uri, data_formats, spec = convert.gen_dcae_artifacts_for_model(config, model_name, model_version) + shutil.rmtree(modeldir) + except Exception: + print('Error analyzing artifacts for {}'.format(compid)) + traceback.print_exc() + return + for data_format in data_formats: + fmtid = (data_format['self']['name'], data_format['self']['version']) + if fmtid in _loadedformats: + print('Skipping data format {}: already analyzed'.format(fmtid)) + continue + try: + resp = requests.post(oburl('/dataformats'), json={'owner': config.dcaeuser, 'spec': data_format}, auth=obauth) + if resp.status_code == 409: + print('Skipping data format {}: previously loaded'.format(fmtid)) + _loadedformats.add(fmtid) + continue + resp.raise_for_status() + requests.patch(resp.json()['dataFormatUrl'], json={'owner': config.dcaeuser, 'status': 'published'}, auth=obauth).raise_for_status() + print('Loaded data format {}'.format(fmtid)) + _loadedformats.add(fmtid) + except Exception: + print('Error loading data format {}'.format(fmtid)) + traceback.print_exc() + raise + try: + resp = requests.post(oburl('/components'), json={'owner': config.dcaeuser, 'spec': spec}, auth=obauth) + if resp.status_code == 409: + print('Skipping component {}: previously loaded'.format(compid)) + _loadedcomponents.add(compid) + return + resp.raise_for_status() + requests.patch(resp.json()['componentUrl'], json={'owner': config.dcaeuser, 'status': 'published'}, auth=obauth).raise_for_status() + print('Loaded component {}'.format(compid)) + _loadedcomponents.add(compid) + except Exception: + print('Error loading component {}'.format(compid)) + traceback.print_exc() + raise + return callback + + +class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): + protocol_version = "HTTP/1.1" + + +class Apihandler(BaseHTTPRequestHandler): + def doqp(self): + self.qparams = {} + if not self.path or '?' not in self.path: + return + self.path, qp = self.path.split('?', 1) + for x in qp.split('&'): + k, v = x.split('=', 1) + self.qparams[unquote_plus(k)] = unquote_plus(v) + + def replyjson(self, body, ctype='application/json'): + self.replyraw(json.dumps(body).encode('utf-8'), ctype) + + def replyraw(self, data, ctype): + self.send_response(200) + self.send_header('Content-Type', ctype) + self.send_header('Content-Length', len(data)) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + self.doqp() + if self.path == '/' or self.path == '/index.html': + self.replyraw(self.server.index, 'text/html') + return + if 'acumos' not in self.qparams: + self.send_error(400) + return + aa = _AcumosAccess(self.server.config, self.qparams['acumos']) + if self.path == '/listCatalogs.js': + self.replyjson([{'name': x['name'], 'id': x['catalogId']} for x in aa.jsonget('/catalogs')]) + return + if self.path == '/listSolutions.js': + if 'catalogId' not in self.qparams: + self.send_error(400) + return + self.replyjson([{'name': x['name'], 'id': x['solutionId']} for x in aa.jsonget('/solutions?catalogId={}', self.qparams['catalogId'])]) + return + if self.path == '/listRevisions.js': + if 'solutionId' not in self.qparams: + self.send_error(400) + return + self.replyjson([{'name': x['version'], 'id': x['revisionId']} for x in aa.jsonget('/solutions/{}/revisions', self.qparams['solutionId'])]) + return + self.send_error(404) + + def do_POST(self): + self.doqp() + if self.path == '/onboard.js': + if 'acumos' not in self.qparams: + self.send_error(400) + return + aa = _AcumosAccess(self.server.config, self.qparams['acumos']) + callback = self.server.callback + if 'catalogId' not in self.qparams: + for catalog in aa.jsonget('/catalogs'): + for solution in aa.jsonget('/solutions?catalogId={}', catalog['catalogId']): + for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']): + onboard(aa, callback, solution, revision['revisionId']) + elif 'solutionId' not in self.qparams: + for solution in aa.jsonget('/solutions?catalogId={}', self.qparams['catalogId']): + for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']): + onboard(aa, callback, solution, revision['revisionId']) + elif 'revisionId' not in self.qparams: + solution = aa.jsonget('/solutions/{}', self.qparams['solutionId']) + for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']): + onboard(aa, callback, solution, revision['revisionId']) + else: + solution = aa.jsonget('/solutions/{}', self.qparams['solutionId']) + onboard(aa, callback, solution, self.qparams['revisionId']) + self.replyraw('OK', 'text/plain') + return + self.send_error(400) + + +def serve(config): + server = ThreadedHTTPServer(('', config.port), Apihandler) + server.config = config + server.callback = _makecallback(config) + server.index = resource_string(__name__, 'index.html') + server.serve_forever() diff --git a/adapter/acumos/aoconversion/spec_gen.py b/adapter/acumos/aoconversion/spec_gen.py new file mode 100644 index 0000000..1662872 --- /dev/null +++ b/adapter/acumos/aoconversion/spec_gen.py @@ -0,0 +1,112 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +""" +Generates DCAE component specs +""" + + +import json +from jsonschema import validate +import requests +from aoconversion import utils + + +def _get_dcae_cs_schema(): + res = requests.get( + "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json;hb=HEAD" + ) + return res.json() + + +def _get_format_version(target_name, data_formats): + """ + search through the data formats for name, make sure we have it, and retrieve the version + """ + # the df must exist, since the data formats were generated from the same metadata, or dataformats call blew up. + # So we don't do error checking here + for df in data_formats: + if df["self"]["name"] == target_name: + return df["self"]["version"] + + +def _generate_spec(model_name, meta, dcae_cs_schema, data_formats, docker_uri): + """ + Function that generates the component spec from the model metadata and docker info + Broken out to be unit-testable. + """ + + spec = { + "self": { + "version": "1.0.0", # hopefully we get this from somewhere and not hardcode this + "name": model_name, + "description": "Automatically generated from Acumos model", + "component_type": "docker", + }, + "services": {"calls": [], "provides": []}, + "streams": {"subscribes": [], "publishes": []}, + "parameters": [], + "auxilary": {"healthcheck": {"type": "http", "endpoint": "/healthcheck"}}, + "artifacts": [{"type": "docker image", "uri": docker_uri}], + } + + # from https://pypi.org/project/acumos-dcae-model-runner/ + # each model method gets a subscruber and a publisher, using the methood name + pstype = "message_router" + for method in meta["methods"]: + + df_in_name = meta["methods"][method]["input"] + subscriber = { + "config_key": "{0}_subscriber".format(method), + "format": df_in_name, + "version": _get_format_version(df_in_name, data_formats), + "type": pstype, + } + + spec["streams"]["subscribes"].append(subscriber) + + df_out_name = meta["methods"][method]["output"] + + publisher = { + "config_key": "{0}_publisher".format(method), + "format": df_out_name, + "version": _get_format_version(df_out_name, data_formats), + "type": pstype, + } + + spec["streams"]["publishes"].append(publisher) + + # Validate that we have a valid spec + validate(instance=spec, schema=dcae_cs_schema) + + return spec + + +def generate_spec(model_repo_path, model_name, data_formats, docker_uri): + """ + Generate and write the component spec to disk + Returns the spec + """ + spec = _generate_spec( + model_name, utils.get_metadata(model_repo_path, model_name), _get_dcae_cs_schema(), data_formats, docker_uri + ) + fname = "{0}_dcae_component_specification.json".format(model_name) + with open("{0}/{1}".format(model_repo_path, fname), "w") as f: + f.write(json.dumps(spec)) + + return spec diff --git a/adapter/acumos/aoconversion/utils.py b/adapter/acumos/aoconversion/utils.py new file mode 100644 index 0000000..a5aae75 --- /dev/null +++ b/adapter/acumos/aoconversion/utils.py @@ -0,0 +1,24 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import json + + +def get_metadata(model_repo_path, model_name): + # for now, assume it's called "metadata.json" + return json.loads(open("{0}/{1}/metadata.json".format(model_repo_path, model_name), "r").read()) diff --git a/adapter/acumos/pom.xml b/adapter/acumos/pom.xml new file mode 100644 index 0000000..8495ee6 --- /dev/null +++ b/adapter/acumos/pom.xml @@ -0,0 +1,37 @@ + + + + + 4.0.0 + org.onap.dcaegen2.platform.adapter + dcaegen2-platform-adapter-acumos + 1.0.0 + + UTF-8 + . + xunit-results.xml + coverage.xml + py + python + **/*.py + **/tests/**,**/setup.py + + diff --git a/adapter/acumos/setup.py b/adapter/acumos/setup.py new file mode 100644 index 0000000..1845b9e --- /dev/null +++ b/adapter/acumos/setup.py @@ -0,0 +1,36 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +from setuptools import setup, find_packages + +setup( + name="aoconversion", + version="1.0.0", + packages=find_packages(exclude=["tests.*", "tests"]), + author="Tommy Carpenter, Andrew Gauld", + author_email="tommy@research.att.com, agauld@att.com", + description="Service to create DCAE artifacts from acumos models", + url="", + install_requires=["docker>=4.0.0,<5.0.0", "jsonschema", "PyYAML", "requests"], + package_data={'aoconversion': ['index.html']}, + entry_points={ + "console_scripts": [ + "acumos-adapter=aoconversion.adapter:adapter" + ] + } +) diff --git a/adapter/acumos/tests/fixtures/README.md b/adapter/acumos/tests/fixtures/README.md new file mode 100644 index 0000000..1ec5b68 --- /dev/null +++ b/adapter/acumos/tests/fixtures/README.md @@ -0,0 +1,2 @@ +# Fixtures +The test fixtures in here came from https://gerrit.acumos.org/r/admin/repos/python-dcae-model-runner in the example folder. diff --git a/adapter/acumos/tests/fixtures/dataformat_101.json b/adapter/acumos/tests/fixtures/dataformat_101.json new file mode 100644 index 0000000..66aa2ab --- /dev/null +++ b/adapter/acumos/tests/fixtures/dataformat_101.json @@ -0,0 +1,212 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Data format specification schema Version 1.0.1", + "type": "object", + "oneOf": [{ + "properties": { + "self": { + "$ref": "#/definitions/self" + }, + "dataformatversion": { + "$ref": "#/definitions/dataformatversion" + }, + "reference": { + + "type": "object", + "description": "A reference to an external schema - name/version or url, if specified, is used to access the artifact", + "properties": { + "name": { + "$ref": "#/definitions/name" + }, + "url": { + "$ref": "#/definitions/url" + }, + "version": { + "$ref": "#/definitions/version" + }, + "format": { + "$ref": "#/definitions/format" + } + }, + "required": [ + "name", + "version", + "format" + ], + "additionalProperties": false + } + }, + "required": ["self", "dataformatversion", "reference"], + "additionalProperties": false + }, { + "properties": { + "self": { + "$ref": "#/definitions/self" + }, + "dataformatversion": { + "$ref": "#/definitions/dataformatversion" + }, + "jsonschema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "The actual JSON schema for this data format" + } + + }, + "required": ["self", "dataformatversion", "jsonschema"], + "additionalProperties": false + }, { + "properties": { + "self": { + "$ref": "#/definitions/self" + }, + "dataformatversion": { + "$ref": "#/definitions/dataformatversion" + }, + "delimitedschema": { + "type": "object", + "description": "A JSON schema for delimited files", + "properties": { + "delimiter": { + "enum": [",", "|", "\t"] + }, + "fields": { + "type": "array", + "description": "Array of field descriptions", + "items": { + "$ref": "#/definitions/field" + } + } + }, + "additionalProperties": false + } + }, + "required": ["self", "dataformatversion", "delimitedschema"], + "additionalProperties": false + }, { + "properties": { + "self": { + "$ref": "#/definitions/self" + }, + "dataformatversion": { + "$ref": "#/definitions/dataformatversion" + }, + "unstructured": { + "type": "object", + "description": "A JSON schema for unstructured text", + "properties": { + "encoding": { + "type": "string", + "enum": ["ASCII", "UTF-8", "UTF-16", "UTF-32"] + } + }, + "additionalProperties": false + + } + }, + "required": ["self", "dataformatversion", "unstructured"], + "additionalProperties": false + }], + "definitions": { + "url": { + "format": "uri" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string", + "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" + }, + "self": { + "description": "Identifying Information for the Data Format - name/version can be used to access the artifact", + "type": "object", + "properties": { + "name": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "description": { + "type": "string" + } + }, + "required": [ + "name", + "version" + ], + "additionalProperties": false + }, + "format": { + "description": "Reference schema type", + "type": "string", + "enum": [ + "JSON", + "Delimited Format", + "XML", + "Protocol Buffer", + "Unstructured" + ] + }, + "field": { + "description": "A field definition for the delimited schema", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "fieldtype": { + "description": "the field type - from the XML schema types", + "type": "string", + "enum": ["string", "boolean", + "decimal", "float", "double", + "duration", "dateTime", "time", + "date", "gYearMonth", "gYear", + "gMonthDay", "gDay", "gMonth", + "hexBinary", "base64Binary", + "anyURI", "QName", "NOTATION", + "normalizedString", "token", + "language", "IDREFS", "ENTITIES", + "NMTOKEN", "NMTOKENS", "Name", + "NCName", "ID", "IDREF", "ENTITY", + "integer", "nonPositiveInteger", + "negativeInteger", "long", "int", + "short", "byte", + "nonNegativeInteger", "unsignedLong", + "unsignedInt", "unsignedShort", + "unsignedByte", "positiveInteger" + + ] + }, + "fieldPattern": { + "description": "Regular expression that defines the field format", + "type": "integer" + }, + "fieldMaxLength": { + "description": "The maximum length of the field", + "type": "integer" + }, + "fieldMinLength": { + "description": "The minimum length of the field", + "type": "integer" + }, + "fieldMinimum": { + "description": "The minimum numeric value of the field", + "type": "integer" + }, + "fieldMaximum": { + "description": "The maximum numeric value of the field", + "type": "integer" + } + }, + "additionalProperties": false + }, + "dataformatversion": { + "type": "string", + "enum": ["1.0.0", "1.0.1"] + } + } +} diff --git a/adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json b/adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json new file mode 100644 index 0000000..1f1f75e --- /dev/null +++ b/adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json @@ -0,0 +1,860 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Component specification schema", + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "version": { + "$ref": "#/definitions/version" + }, + "description": { + "type": "string" + }, + "component_type": { + "type": "string", + "enum": [ + "docker", + "cdap" + ] + }, + "name": { + "$ref": "#/definitions/name" + } + }, + "required": [ + "version", + "name", + "description", + "component_type" + ] + }, + "streams": { + "type": "object", + "properties": { + "publishes": { + "type": "array", + "uniqueItems": true, + "items": { + "oneOf": [ + { "$ref": "#/definitions/publisher_http" }, + { "$ref": "#/definitions/publisher_message_router" }, + { "$ref": "#/definitions/publisher_data_router" } + ] + } + }, + "subscribes": { + "type": "array", + "uniqueItems": true, + "items": { + "oneOf": [ + { "$ref": "#/definitions/subscriber_http" }, + { "$ref": "#/definitions/subscriber_message_router" }, + { "$ref": "#/definitions/subscriber_data_router" } + ] + } + } + }, + "required": [ + "publishes", + "subscribes" + ] + }, + "services": { + "type": "object", + "properties": { + "calls": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/caller" + } + }, + "provides": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/provider" + } + } + }, + "required": [ + "calls", + "provides" + ] + }, + "parameters" : { + "anyOf" : [ + {"$ref": "#/definitions/docker-parameters"}, + {"$ref": "#/definitions/cdap-parameters"} + ] + }, + "auxilary": { + "oneOf" : [ + {"$ref": "#/definitions/auxilary_cdap"}, + {"$ref": "#/definitions/auxilary_docker"} + ] + }, + "artifacts": { + "type": "array", + "description": "List of component artifacts", + "items": { + "$ref": "#/definitions/artifact" + } + } + }, + "required": [ + "self", + "streams", + "services", + "parameters", + "auxilary", + "artifacts" + ], + "additionalProperties": false, + "definitions": { + "cdap-parameters": { + "description" : "There are three seperate ways to pass parameters to CDAP: app config, app preferences, program preferences. These are all treated as optional.", + "type": "object", + "properties" : { + "program_preferences": { + "description" : "A list of {program_id, program_type, program_preference} objects where program_preference is an object passed into program_id of type program_type", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/program_preference" + } + }, + "app_preferences" : { + "description" : "Parameters Passed down to the CDAP preference API", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + }, + "app_config" : { + "description" : "Parameters Passed down to the CDAP App Config", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + } + } + }, + "program_preference": { + "type": "object", + "properties": { + "program_type": { + "$ref": "#/definitions/program_type" + }, + "program_id": { + "type": "string" + }, + "program_pref":{ + "description" : "Parameters that the CDAP developer wants pushed to this program's preferences API. Optional", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + } + }, + "required": ["program_type", "program_id", "program_pref"] + }, + "program_type": { + "type": "string", + "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] + }, + "docker-parameters": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + }, + "parameter": { + "oneOf": [ + {"$ref": "#/definitions/parameter-list"}, + {"$ref": "#/definitions/parameter-other"} + ] + }, + "parameter-list": { + "properties": { + "name": { + "type": "string" + }, + "value": { + "description": "Default value for the parameter" + }, + "description": { + "description": "Description for the parameter.", + "type": "string" + }, + "type": { + "description": "Only valid type is list, the entry_schema is required - which contains the type of the list element. All properties set for the parameter apply to all elements in the list at this time", + "type": "string", + "enum": ["list"] + }, + "required": { + "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", + "type": "boolean", + "default": true + }, + "constraints": { + "description": "The optional list of sequenced constraint clauses for the parameter.", + "type": "array", + "items": { + "$ref": "#/definitions/parameter-constraints" + } + }, + "entry_schema": { + "description": "The optional property used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is list. This is the only type it is currently supported for.", + "type": "object", + "uniqueItems": true, + "items": {"$ref": "#/definitions/list-parameter"} + }, + "designer_editable": { + "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", + "type": "boolean" + }, + "sourced_at_deployment": { + "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", + "type": "boolean" + }, + "policy_editable": { + "description": "A required property that declares a parameter as editable by DevOps in Policy UI (true) or not (false).", + "type": "boolean" + }, + "policy_group": { + "description": "An optional property used to group policy_editable parameters into groups. Each group will become it's own policy model. Any parameters without this property will be grouped together to form their own policy model", + "type": "string" + }, + "policy_schema" :{ + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/policy_schema_parameter"} + } + }, + "required": [ + "name", + "value", + "description", + "designer_editable", + "policy_editable", + "sourced_at_deployment", + "entry_schema" + ], + "additionalProperties": false, + "dependencies": { + "policy_schema": ["policy_editable"] + } + }, + "parameter-other": { + "properties": { + "name": { + "type": "string" + }, + "value": { + "description": "Default value for the parameter" + }, + "description": { + "description": "Description for the parameter.", + "type": "string" + }, + "type": { + "description": "The required data type for the parameter.", + "type": "string", + "enum": [ "string", "number", "boolean", "datetime" ] + }, + "required": { + "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", + "type": "boolean", + "default": true + }, + "constraints": { + "description": "The optional list of sequenced constraint clauses for the parameter.", + "type": "array", + "items": { + "$ref": "#/definitions/parameter-constraints" + } + }, + "designer_editable": { + "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", + "type": "boolean" + }, + "sourced_at_deployment": { + "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", + "type": "boolean" + }, + "policy_editable": { + "description": "A required property that declares a parameter as editable in Policy UI (true) or not (false).", + "type": "boolean" + }, + "policy_group": { + "description": "An optional property used to group policy_editable parameters into groups. Each group will become it's own policy model. Any parameters without this property will be grouped together to form their own policy model", + "type": "string" + }, + "policy_schema" :{ + "description": "An optional property used to define policy_editable parameters as lists or maps", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/policy_schema_parameter"} + } + }, + "required": [ + "name", + "value", + "description", + "designer_editable", + "sourced_at_deployment", + "policy_editable" + ], + "additionalProperties": false, + "dependencies": { + "policy_schema": ["policy_editable"] + } + }, + "list-parameter": { + "type": "object", + "properties": { + "type": { + "description": "The required data type for each parameter in the list.", + "type": "string", + "enum": ["string", "number"] + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "policy_schema_parameter": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "description": "Default value for the parameter" + }, + "description": { + "description": "Description for the parameter.", + "type": "string" + }, + "type": { + "description": "The required data type for the parameter.", + "type": "string", + "enum": [ "string", "number", "boolean", "datetime", "list", "map" ] + }, + "required": { + "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", + "type": "boolean", + "default": true + }, + "constraints": { + "description": "The optional list of sequenced constraint clauses for the parameter.", + "type": "array", + "items": { + "$ref": "#/definitions/parameter-constraints" + } + }, + "entry_schema": { + "description": "The optional key that is used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is either list or map. If the type is list and the entry type is a simple type (string, number, boolean, datetime), follow with a simple string to describe the entry type. If the type is list and the entry type is a map, follow with an array to describe the keys for the entry map. If the type is list and the entry type is also list, this is not currently supported here. If the type is map, then follow with an array to describe the keys for this map. ", + "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/policy_schema_parameter"} + } + }, + "required": [ + "name", + "type" + ], + "additionalProperties": false + }, + "parameter-constraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "equal": { + "description": "Constrains a property or parameter to a value equal to (‘=’) the value declared." + }, + "greater_than": { + "description": "Constrains a property or parameter to a value greater than (‘>’) the value declared.", + "type": "number" + }, + "greater_or_equal": { + "description": "Constrains a property or parameter to a value greater than or equal to (‘>=’) the value declared.", + "type": "number" + }, + "less_than": { + "description": "Constrains a property or parameter to a value less than (‘<’) the value declared.", + "type": "number" + }, + "less_or_equal": { + "description": "Constrains a property or parameter to a value less than or equal to (‘<=’) the value declared.", + "type": "number" + }, + "valid_values": { + "description": "Constrains a property or parameter to a value that is in the list of declared values.", + "type": "array" + }, + "length": { + "description": "Constrains the property or parameter to a value of a given length.", + "type": "number" + }, + "min_length": { + "description": "Constrains the property or parameter to a value to a minimum length.", + "type": "number" + }, + "max_length": { + "description": "Constrains the property or parameter to a value to a maximum length.", + "type": "number" + } + } + }, + "stream_message_router": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "config_key": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "message router", "message_router" + ] + } + }, + "required": [ + "format", + "version", + "config_key", + "type" + ] + }, + "publisher_http": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "config_key": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "http", + "https" + ] + } + }, + "required": [ + "format", + "version", + "config_key", + "type" + ] + }, + "publisher_message_router": { + "$ref": "#/definitions/stream_message_router" + }, + "publisher_data_router": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "config_key": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "data router", "data_router" + ] + } + }, + "required": [ + "format", + "version", + "config_key", + "type" + ] + }, + "subscriber_http": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "route": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "http", + "https" + ] + } + }, + "required": [ + "format", + "version", + "route", + "type" + ] + }, + "subscriber_message_router": { + "$ref": "#/definitions/stream_message_router" + }, + "subscriber_data_router": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "route": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "data router", "data_router" + ] + }, + "config_key": { + "description": "Data router subscribers require config info to setup their endpoints to handle requests. For example, needs username and password", + "type": "string" + } + }, + "required": [ + "format", + "version", + "route", + "type", + "config_key" + ] + }, + "provider" : { + "oneOf" : [ + {"$ref": "#/definitions/docker-provider"}, + {"$ref": "#/definitions/cdap-provider"} + ] + }, + "cdap-provider" : { + "type": "object", + "properties" : { + "request": { + "$ref": "#/definitions/formatPair" + }, + "response": { + "$ref": "#/definitions/formatPair" + }, + "service_name" : { + "type" : "string" + }, + "service_endpoint" : { + "type" : "string" + }, + "verb" : { + "type": "string", + "enum": ["GET", "PUT", "POST", "DELETE"] + } + }, + "required" : [ + "request", + "response", + "service_name", + "service_endpoint", + "verb" + ] + }, + "docker-provider": { + "type": "object", + "properties": { + "request": { + "$ref": "#/definitions/formatPair" + }, + "response": { + "$ref": "#/definitions/formatPair" + }, + "route": { + "type": "string" + }, + "verb": { + "type": "string", + "enum": ["GET", "PUT", "POST", "DELETE"] + } + }, + "required": [ + "request", + "response", + "route" + ] + }, + "caller": { + "type": "object", + "properties": { + "request": { + "$ref": "#/definitions/formatPair" + }, + "response": { + "$ref": "#/definitions/formatPair" + }, + "config_key": { + "type": "string" + } + }, + "required": [ + "request", + "response", + "config_key" + ] + }, + "formatPair": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + } + } + }, + "name": { + "type": "string" + }, + "version": { + "type": "string", + "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" + }, + "artifact": { + "type": "object", + "description": "Component artifact object", + "properties": { + "uri": { + "type": "string", + "description": "Uri to artifact" + }, + "type": { + "type": "string", + "enum": ["jar", "docker image"] + } + }, + "required": ["uri", "type"] + }, + + "auxilary_cdap": { + "title": "cdap component specification schema", + "type": "object", + "properties": { + "streamname": { + "type": "string" + }, + "artifact_name" : { + "type": "string" + }, + "artifact_version" : { + "type": "string", + "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" + }, + "namespace":{ + "type": "string", + "description" : "optional" + }, + "programs": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/cdap_program" + } + } + }, + "required": [ + "streamname", + "programs", + "artifact_name", + "artifact_version" + ] + }, + "cdap_program_type": { + "type": "string", + "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] + }, + "cdap_program": { + "type": "object", + "properties": { + "program_type": { + "$ref": "#/definitions/cdap_program_type" + }, + "program_id": { + "type": "string" + } + }, + "required": ["program_type", "program_id"] + }, + + "auxilary_docker": { + "title": "Docker component specification schema", + "type": "object", + "properties": { + "healthcheck": { + "description": "Define the health check that Consul should perfom for this component", + "type": "object", + "oneOf": [ + { "$ref": "#/definitions/docker_healthcheck_http" }, + { "$ref": "#/definitions/docker_healthcheck_script" } + ] + }, + "ports": { + "description": "Port mapping to be used for Docker containers. Each entry is of the format :.", + "type": "array", + "items": { + "type": "string" + } + }, + "logging": { + "description": "Component specific details for logging", + "type": "object", + "properties": { + "log_directory": { + "description": "The path in the container where the component writes its logs. If the component is following the EELF requirements, this would be the directory where the four EELF files are being written. (Other logs can be placed in the directory--if their names in '.log', they'll also be sent into ELK.)", + "type": "string" + }, + "alternate_fb_path": { + "description": "By default, the log volume is mounted at /var/log/onap/ in the sidecar container's file system. 'alternate_fb_path' allows overriding the default. Will affect how the log data can be found in the ELK system.", + "type": "string" + } + }, + "additionalProperties": false + }, + "policy": { + "properties": { + "trigger_type": { + "description": "Only value of docker is supported at this time.", + "type": "string", + "enum": ["docker"] + }, + "script_path": { + "description": "Script command that will be executed for policy reconfiguration", + "type": "string" + } + }, + "required": [ + "trigger_type","script_path" + ], + "additionalProperties": false + }, + "volumes": { + "description": "Volume mapping to be used for Docker containers. Each entry is of the format below", + "type": "array", + "items": { + "type": "object", + "properties": { + "host":{ + "type":"object", + "path": {"type": "string"} + }, + "container":{ + "type":"object", + "bind": { "type": "string"}, + "mode": { "type": "string"} + } + } + } + } + }, + "required": [ + "healthcheck" + ], + "additionalProperties": false + }, + "docker_healthcheck_http": { + "properties": { + "type": { + "description": "Consul health check type", + "type": "string", + "enum": [ + "http", + "https" + ] + }, + "interval": { + "description": "Interval duration in seconds i.e. 10s", + "default": "15s", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds i.e. 10s", + "default": "1s", + "type": "string" + }, + "endpoint": { + "description": "Relative endpoint used by Consul to check health by making periodic HTTP GET calls", + "type": "string" + } + }, + "required": [ + "type", + "endpoint" + ] + }, + "docker_healthcheck_script": { + "properties": { + "type": { + "description": "Consul health check type", + "type": "string", + "enum": [ + "script", + "docker" + ] + }, + "interval": { + "description": "Interval duration in seconds i.e. 10s", + "default": "15s", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds i.e. 10s", + "default": "1s", + "type": "string" + }, + "script": { + "description": "Script command that will be executed by Consul to check health", + "type": "string" + } + }, + "required": [ + "type", + "script" + ] + } + } +} diff --git a/adapter/acumos/tests/fixtures/jsdraft4schema.json b/adapter/acumos/tests/fixtures/jsdraft4schema.json new file mode 100644 index 0000000..bcbb847 --- /dev/null +++ b/adapter/acumos/tests/fixtures/jsdraft4schema.json @@ -0,0 +1,149 @@ +{ + "id": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "positiveInteger": { + "type": "integer", + "minimum": 0 + }, + "positiveIntegerDefault0": { + "allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ] + }, + "simpleTypes": { + "enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "uniqueItems": true + } + }, + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "$schema": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": {}, + "multipleOf": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "boolean", + "default": false + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "boolean", + "default": false + }, + "maxLength": { "$ref": "#/definitions/positiveInteger" }, + "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { + "anyOf": [ + { "type": "boolean" }, + { "$ref": "#" } + ], + "default": {} + }, + "items": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/schemaArray" } + ], + "default": {} + }, + "maxItems": { "$ref": "#/definitions/positiveInteger" }, + "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "maxProperties": { "$ref": "#/definitions/positiveInteger" }, + "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { + "anyOf": [ + { "type": "boolean" }, + { "$ref": "#" } + ], + "default": {} + }, + "definitions": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/stringArray" } + ] + } + }, + "enum": { + "type": "array", + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { "type": "string" }, + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "dependencies": { + "exclusiveMaximum": [ "maximum" ], + "exclusiveMinimum": [ "minimum" ] + }, + "default": {} +} diff --git a/adapter/acumos/tests/fixtures/models/ArgsList_1.0.0_dcae_data_format.json b/adapter/acumos/tests/fixtures/models/ArgsList_1.0.0_dcae_data_format.json new file mode 100644 index 0000000..56b6e6e --- /dev/null +++ b/adapter/acumos/tests/fixtures/models/ArgsList_1.0.0_dcae_data_format.json @@ -0,0 +1 @@ +{"self": {"name": "ArgsList", "version": "1.0.0"}, "dataformatversion": "1.0.1", "jsonschema": {"title": "ArgsList", "type": "object", "properties": {"args": {"type": "array", "items": {"$ref": "#/definitions/Args"}}}, "$schema": "http://json-schema.org/draft-04/schema#", "definitions": {"Args": {"title": "Args", "type": "object", "properties": {"x": {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}, "y": {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}}}}}} \ No newline at end of file diff --git a/adapter/acumos/tests/fixtures/models/SumOut_1.0.0_dcae_data_format.json b/adapter/acumos/tests/fixtures/models/SumOut_1.0.0_dcae_data_format.json new file mode 100644 index 0000000..3f7cdca --- /dev/null +++ b/adapter/acumos/tests/fixtures/models/SumOut_1.0.0_dcae_data_format.json @@ -0,0 +1 @@ +{"self": {"name": "SumOut", "version": "1.0.0"}, "dataformatversion": "1.0.1", "jsonschema": {"title": "SumOut", "type": "object", "properties": {"value": {"type": "array", "items": {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}}}, "$schema": "http://json-schema.org/draft-04/schema#", "definitions": {}}} \ No newline at end of file diff --git a/adapter/acumos/tests/fixtures/models/example-model-listofm/example_model.py b/adapter/acumos/tests/fixtures/models/example-model-listofm/example_model.py new file mode 100644 index 0000000..cff1acb --- /dev/null +++ b/adapter/acumos/tests/fixtures/models/example-model-listofm/example_model.py @@ -0,0 +1,41 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +from acumos.session import AcumosSession +from acumos.modeling import Model, List, NamedTuple + + +class Args(NamedTuple): + x: int + y: int + + +class ArgsList(NamedTuple): + args: List[Args] + + +def sum(args: ArgsList) -> List[int]: + return [arg.x + arg.y for arg in args] + + +if __name__ == '__main__': + '''Main''' + model = Model(sum=sum) + + session = AcumosSession() + session.dump(model, 'example-model', '.') diff --git a/adapter/acumos/tests/fixtures/models/example-model-listofm/metadata.json b/adapter/acumos/tests/fixtures/models/example-model-listofm/metadata.json new file mode 100644 index 0000000..e9df2bd --- /dev/null +++ b/adapter/acumos/tests/fixtures/models/example-model-listofm/metadata.json @@ -0,0 +1 @@ +{"schema": "acumos.schema.model:0.4.0", "runtime": {"name": "python", "encoding": "protobuf", "version": "3.6.8", "dependencies": {"pip": {"indexes": [], "requirements": [{"name": "dill", "version": "0.3.0"}, {"name": "acumos", "version": "0.8.0"}]}, "conda": {"channels": [], "requirements": []}}}, "name": "example-model", "methods": {"sum": {"input": "ArgsList", "output": "SumOut", "description": ""}}} \ No newline at end of file diff --git a/adapter/acumos/tests/fixtures/models/example-model-listofm/model.proto b/adapter/acumos/tests/fixtures/models/example-model-listofm/model.proto new file mode 100644 index 0000000..ba40893 --- /dev/null +++ b/adapter/acumos/tests/fixtures/models/example-model-listofm/model.proto @@ -0,0 +1,39 @@ +/*- + * ===============LICENSE_START================================================ + * org.onap.dcae + * ============================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================ + * This Acumos software file is distributed by AT&T and Tech Mahindra + * under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===============LICENSE_END================================================== + */ + +syntax = "proto3"; +package QrPbKNxHnOMiIoWCyFplggLzzNUBisBm; + +service Model { + rpc sum (ArgsList) returns (SumOut); +} + +message ArgsList { + repeated Args args = 1; +} + +message Args { + int64 x = 1; + int64 y = 2; +} + +message SumOut { + repeated int64 value = 1; +} diff --git a/adapter/acumos/tests/fixtures/models/example-model/metadata.json b/adapter/acumos/tests/fixtures/models/example-model/metadata.json new file mode 100644 index 0000000..302b5f7 --- /dev/null +++ b/adapter/acumos/tests/fixtures/models/example-model/metadata.json @@ -0,0 +1,35 @@ +{ + "schema": "acumos.schema.model:0.4.0", + "runtime": { + "name": "python", + "encoding": "protobuf", + "version": "3.6.8", + "dependencies": { + "pip": { + "indexes": [], + "requirements": [ + { + "name": "dill", + "version": "0.3.0" + }, + { + "name": "acumos", + "version": "0.8.0" + } + ] + }, + "conda": { + "channels": [], + "requirements": [] + } + } + }, + "name": "example-model", + "methods": { + "add": { + "input": "NumbersIn", + "output": "NumberOut", + "description": "Adds two integers" + } + } +} diff --git a/adapter/acumos/tests/fixtures/models/example-model/model.proto b/adapter/acumos/tests/fixtures/models/example-model/model.proto new file mode 100644 index 0000000..da03383 --- /dev/null +++ b/adapter/acumos/tests/fixtures/models/example-model/model.proto @@ -0,0 +1,35 @@ +/*- + * ===============LICENSE_START================================================ + * org.onap.dcae + * ============================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================ + * This Acumos software file is distributed by AT&T and Tech Mahindra + * under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===============LICENSE_END================================================== + */ + +syntax = "proto3"; +package geYBmFkDJtafXLpLFWhXwFWZekRqgaGs; + +service Model { + rpc add (NumbersIn) returns (NumberOut); +} + +message NumbersIn { + int64 x = 1; + int64 y = 2; +} + +message NumberOut { + int64 result = 1; +} diff --git a/adapter/acumos/tests/test_df.py b/adapter/acumos/tests/test_df.py new file mode 100644 index 0000000..cdf41c4 --- /dev/null +++ b/adapter/acumos/tests/test_df.py @@ -0,0 +1,109 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +from testing_helpers import get_json_fixture, get_fixture_path +from aoconversion import dataformat_gen + +TEST_META = get_json_fixture("models/example-model/metadata.json") +DRAFT_4_SCHEMA = get_json_fixture("jsdraft4schema.json") +DF_101 = get_json_fixture("dataformat_101.json") + + +def test_get_needed_formats(): + assert dataformat_gen._get_needed_formats(TEST_META) == ["NumbersIn", "NumberOut"] + + +def test_generate_dcae_data_formats(): + """ + Test generating data formats from the protobuf + """ + test_proto_path = get_fixture_path("models/example-model/model.proto") + assert dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, DF_101, DRAFT_4_SCHEMA) == [ + { + "self": {"name": "NumbersIn", "version": "1.0.0"}, + "dataformatversion": "1.0.1", + "jsonschema": { + "title": "NumbersIn", + "type": "object", + "properties": { + "x": {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}, + "y": {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}, + }, + "$schema": "http://json-schema.org/draft-04/schema#", + "definitions": {}, + }, + }, + { + "self": {"name": "NumberOut", "version": "1.0.0"}, + "dataformatversion": "1.0.1", + "jsonschema": { + "title": "NumberOut", + "type": "object", + "properties": {"result": {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}}, + "$schema": "http://json-schema.org/draft-04/schema#", + "definitions": {}, + }, + }, + ] + + +def test_generate_dcae_data_formats_listofm(): + """ + Test generating data formats from the protobuf + This one tests the case where definitions needs to be populated in one of the data formats because it's referenced in a "top level" message + """ + test_meta = get_json_fixture("models/example-model-listofm/metadata.json") + test_proto_path = get_fixture_path("models/example-model-listofm/model.proto") + assert dataformat_gen._generate_dcae_data_formats(test_proto_path, test_meta, DF_101, DRAFT_4_SCHEMA) == [ + { + "self": {"name": "ArgsList", "version": "1.0.0"}, + "dataformatversion": "1.0.1", + "jsonschema": { + "title": "ArgsList", + "type": "object", + "properties": {"args": {"type": "array", "items": {"$ref": "#/definitions/Args"}}}, + "$schema": "http://json-schema.org/draft-04/schema#", + "definitions": { + "Args": { + "title": "Args", + "type": "object", + "properties": { + "x": {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}, + "y": {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}, + }, + } + }, + }, + }, + { + "self": {"name": "SumOut", "version": "1.0.0"}, + "dataformatversion": "1.0.1", + "jsonschema": { + "title": "SumOut", + "type": "object", + "properties": { + "value": { + "type": "array", + "items": {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}, + } + }, + "$schema": "http://json-schema.org/draft-04/schema#", + "definitions": {}, + }, + }, + ] diff --git a/adapter/acumos/tests/test_docker.py b/adapter/acumos/tests/test_docker.py new file mode 100644 index 0000000..0d84038 --- /dev/null +++ b/adapter/acumos/tests/test_docker.py @@ -0,0 +1,47 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +from testing_helpers import get_json_fixture +from aoconversion import docker_gen + +TEST_META = get_json_fixture("models/example-model/metadata.json") + + +def test_generate_dockerfile(): + assert ( + docker_gen._generate_dockerfile(TEST_META, "example-model") + == """ + FROM python:3.6.8 + + ENV MODELNAME example-model + RUN mkdir /app + WORKDIR /app + + ADD ./example-model /app/example-model + ADD ./requirements.txt /app + + RUN pip install -r /app/requirements.txt && \ + pip install acumos_dcae_model_runner + + ENV DCAEPORT=10000 + EXPOSE $DCAEPORT + + ENTRYPOINT ["acumos_dcae_model_runner"] + CMD ["/app/example-model"] + """ + ) diff --git a/adapter/acumos/tests/test_fed.py b/adapter/acumos/tests/test_fed.py new file mode 100644 index 0000000..4d3636f --- /dev/null +++ b/adapter/acumos/tests/test_fed.py @@ -0,0 +1,180 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import json +import requests + +from testing_helpers import get_json_fixture as get_test_json +from testing_helpers import get_fixture_path as get_test_file + +from aoconversion import docker_gen as aoc_docker_gen +from aoconversion import scanner as aoc_scanner + +# +# General mocking +# + + +class _MockModule: + """ + Class to mock packages. + Just a quick way to create an object with specified attributes + """ + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + +# +# Mocking for package "docker" +# + + +class _MockAPIClient: + """ + Class to mock docker.APIClient class. + """ + def __init__(self, base_url, version=None, user_agent='xxx'): + pass + + def build(self, path, rm, tag): + return [b'some message', b'another message'] + + def push(self, repository, auth_config, stream): + return [b'some message', b'another message'] + + def images(self, x): + return True + + +def _mock_kwargs_from_env(**kwargs): + """ + Method to mock docker.utils.kwargs_from_env method. + """ + return {'base_url': None} + + +_mockdocker = _MockModule( + APIClient=_MockAPIClient, + utils=_MockModule(kwargs_from_env=_mock_kwargs_from_env)) + + +# +# Mocking for requests.get +# + + +class _r: + """ + Fake responses for mocking requests.get + """ + def __init__(self, json=None, file=None, data=None): + self.jx = json + self.fx = file + self.dx = data + self.status_code = 200 + + @property + def text(self): + return self._raw().decode() + + def raise_for_status(self): + pass + + def _raw(self): + if self.dx is None: + if self.fx is not None: + with open(self.fx, 'rb') as f: + self.dx = f.read() + elif self.jx is not None: + self.dx = json.dumps(self.jx, sort_keys=True).encode() + else: + self.dx = b'' + return self.dx + + def iter_content(self, bsize=-1): + buf = self._raw() + pos = 0 + lim = len(buf) + if bsize <= 0: + bsize = lim + while pos + bsize < lim: + yield buf[pos:pos + bsize] + pos = pos + bsize + yield buf[pos:] + + def json(self): + if self.jx is None: + self.jx = json.loads(self._raw().decode()) + return self.jx + + +def _mockwww(responses): + def _op(path, json=None, auth=None, cert=None, verify=None, stream=False): + return responses[path] + return _op + + +_mockpostdata = { + 'https://onboarding/dataformats': _r({'dataFormatUrl': 'https://onboarding/dataformats/somedfid'}), + 'https://onboarding/components': _r({'componentUrl': 'https://onboarding/components/somedxid'}), +} + +_mockpatchdata = { + 'https://onboarding/dataformats/somedfid': _r({}), + 'https://onboarding/components/somedxid': _r({}), +} + +_mockwebdata = { + 'https://acumos/catalogs': _r({'content': [{'catalogId': 'c1'}]}), + 'https://acumos/solutions?catalogId=c1': _r({'content': [{'solutionId': 's1', 'name': 'example-model', 'ratingAverageTenths': 17}]}), + 'https://acumos/solutions/s1/revisions': _r({'content': [{'revisionId': 'r1'}]}), + 'https://acumos/solutions/s1/revisions/r1': _r({'content': { + 'version': 'v1', + 'modified': '2019-01-01T00:00:00Z', + 'artifacts': [ + {'artifactId': 'a1', 'name': 'xxx.other'}, + {'artifactId': 'a2', 'name': 'xxx.proto'}, + {'artifactId': 'a3', 'name': 'xxx.zip'}, + {'artifactId': 'a4', 'name': 'xxx.json'}, + ]} + }), + 'https://acumos/artifacts/a2/content': _r(file=get_test_file('models/example-model/model.proto')), + 'https://acumos/artifacts/a3/content': _r(data=b'dummy zip archive data'), + 'https://acumos/artifacts/a4/content': _r(file=get_test_file('models/example-model/metadata.json')), + 'http://json-schema.org/draft-04/schema#': _r(get_test_json('jsdraft4schema.json')), + 'https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD': _r(get_test_json('dataformat_101.json')), + 'http://dcaeurl//component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json': _r(get_test_json('dataformat_101.json')), + 'https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json;hb=HEAD': _r(get_test_json('dcae-cli-v2_component-spec-schema.json')), + 'http://dcaeurl//component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json': _r(get_test_json('dcae-cli-v2_component-spec-schema.json')), +} + + +# +# End mocking tools +# + + +def test_aoconversion(tmpdir, monkeypatch): + config = aoc_scanner.Config(dcaeurl='http://dcaeurl', dcaeuser='dcaeuser', onboardingurl='https://onboarding', onboardinguser='obuser', onboardingpass='obpass', acumosurl='https://acumos', certfile=None, dockerregistry='dockerregistry', dockeruser='registryuser', dockerpass='registrypassword') + monkeypatch.setattr(aoc_docker_gen, 'APIClient', _mockdocker.APIClient) + monkeypatch.setattr(requests, 'get', _mockwww(_mockwebdata)) + monkeypatch.setattr(requests, 'post', _mockwww(_mockpostdata)) + monkeypatch.setattr(requests, 'patch', _mockwww(_mockpatchdata)) + aoc_scanner.scan(config) + aoc_scanner.scan(config) diff --git a/adapter/acumos/tests/test_spec.py b/adapter/acumos/tests/test_spec.py new file mode 100644 index 0000000..7f6ec41 --- /dev/null +++ b/adapter/acumos/tests/test_spec.py @@ -0,0 +1,55 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +from testing_helpers import get_json_fixture, get_fixture_path +from aoconversion import dataformat_gen, spec_gen + +TEST_META = get_json_fixture("models/example-model/metadata.json") +DRAFT_4_SCHEMA = get_json_fixture("jsdraft4schema.json") +DF_101 = get_json_fixture("dataformat_101.json") +CS_SCHEMA = get_json_fixture("dcae-cli-v2_component-spec-schema.json") + + +def test_generate_spec(): + """ + Test generating data formats from the protobuf + """ + test_proto_path = get_fixture_path("models/example-model/model.proto") + data_formats = dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, DF_101, DRAFT_4_SCHEMA) + assert spec_gen._generate_spec( + "example-model", TEST_META, CS_SCHEMA, data_formats, "nexus01.fake.com:18443/example-model:latest" + ) == { + "self": { + "version": "1.0.0", + "name": "example-model", + "description": "Automatically generated from Acumos model", + "component_type": "docker", + }, + "services": {"calls": [], "provides": []}, + "streams": { + "subscribes": [ + {"config_key": "add_subscriber", "format": "NumbersIn", "version": "1.0.0", "type": "message_router"} + ], + "publishes": [ + {"config_key": "add_publisher", "format": "NumberOut", "version": "1.0.0", "type": "message_router"} + ], + }, + "parameters": [], + "auxilary": {"healthcheck": {"type": "http", "endpoint": "/healthcheck"}}, + "artifacts": [{"type": "docker image", "uri": "nexus01.fake.com:18443/example-model:latest"}], + } diff --git a/adapter/acumos/tests/testing_helpers.py b/adapter/acumos/tests/testing_helpers.py new file mode 100644 index 0000000..26457bc --- /dev/null +++ b/adapter/acumos/tests/testing_helpers.py @@ -0,0 +1,31 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import json +import os + + +def get_fixture_path(name): + cur_dir = os.path.dirname(os.path.realpath(__file__)) + return "{0}/fixtures/{1}".format(cur_dir, name) + + +def get_json_fixture(name): + path = get_fixture_path(name) + with open(path, "r") as f: + return json.loads(f.read()) diff --git a/adapter/acumos/tox.ini b/adapter/acumos/tox.ini new file mode 100644 index 0000000..bc94f22 --- /dev/null +++ b/adapter/acumos/tox.ini @@ -0,0 +1,44 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +[tox] +envlist = py37,flake8 + +[testenv] +whitelist_externals = + npm +deps= + pytest + coverage + pytest-cov + nodeenv +setenv = + PYTHONPATH={toxinidir} +commands= + nodeenv -p + npm install --global protobuf-jsonschema + pytest --verbose --junitxml xunit-results.xml --cov aoconversion --cov-report xml --cov-report html + +[testenv:flake8] +basepython = python3.7 +skip_install = true +deps = flake8 +commands = flake8 setup.py aoconversion tests + +[flake8] +extend-ignore = E501 -- cgit 1.2.3-korg