diff options
Diffstat (limited to 'adapter')
| -rw-r--r-- | adapter/acumos/Changelog.md | 2 | ||||
| -rw-r--r-- | adapter/acumos/Dockerfile | 4 | ||||
| -rw-r--r-- | adapter/acumos/README.md | 2 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/dataformat_gen.py | 21 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/scanner.py | 5 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/spec_gen.py | 12 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/utils.py | 26 | ||||
| -rw-r--r-- | adapter/acumos/pom.xml | 4 | ||||
| -rw-r--r-- | adapter/acumos/setup.py | 2 | ||||
| -rw-r--r-- | adapter/acumos/tests/conftest.py | 31 | ||||
| -rw-r--r-- | adapter/acumos/tests/fixtures/dataformat_101.json | 212 | ||||
| -rw-r--r-- | adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json | 918 | ||||
| -rw-r--r-- | adapter/acumos/tests/fixtures/jsdraft4schema.json | 149 | ||||
| -rw-r--r-- | adapter/acumos/tests/test_df.py | 14 | ||||
| -rw-r--r-- | adapter/acumos/tests/test_fed.py | 10 | ||||
| -rw-r--r-- | adapter/acumos/tests/test_spec.py | 13 |
16 files changed, 86 insertions, 1339 deletions
diff --git a/adapter/acumos/Changelog.md b/adapter/acumos/Changelog.md index 55bafa8..71e5520 100644 --- a/adapter/acumos/Changelog.md +++ b/adapter/acumos/Changelog.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.0.3] - 7/28/2020 + * Fetch json schemas during image build ## [1.0.2] - 3/26/2020 * Run as non-root ## [1.0.1] - 3/20/2020 diff --git a/adapter/acumos/Dockerfile b/adapter/acumos/Dockerfile index d1a0984..4f1bea2 100644 --- a/adapter/acumos/Dockerfile +++ b/adapter/acumos/Dockerfile @@ -25,6 +25,10 @@ RUN apt-get update && \ apt-get install -y npm nodejs && \ npm -v && \ npm install -g protobuf-jsonschema && \ + mkdir schemas && \ + wget -O schemas/schema.json 'https://json-schema.org/draft-04/schema#' && \ + wget -O schemas/compspec.json https://git.onap.org/dcaegen2/platform/plain/mod/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json && \ + wget -O schemas/dataformat.json https://git.onap.org/dcaegen2/platform/plain/mod/component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json && \ cd /tmp/build/ && \ python setup.py install && \ cd / && \ diff --git a/adapter/acumos/README.md b/adapter/acumos/README.md index 2de3845..3ebce54 100644 --- a/adapter/acumos/README.md +++ b/adapter/acumos/README.md @@ -44,8 +44,6 @@ This operates in 2 modes: - A configuration file, in YAML format, containing the following keys must be available. - dcaeurl - The base URL for the DCAE component and data format schemas. For - example, https://git.onap.org/dcaegen2/platform/cli/plain. dcaeuser - The DCAE user who will "own" the loaded data. onboardingurl - The URL for accessing the onboarding service. onboardinguser - The user ID for accessing the onboarding service. diff --git a/adapter/acumos/aoconversion/dataformat_gen.py b/adapter/acumos/aoconversion/dataformat_gen.py index a5ead96..71b1617 100644 --- a/adapter/acumos/aoconversion/dataformat_gen.py +++ b/adapter/acumos/aoconversion/dataformat_gen.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,22 +19,9 @@ from subprocess import PIPE, Popen import json from jsonschema import validate -import requests from aoconversion import utils, exceptions -def _get_js_schema(): - res = requests.get("http://json-schema.org/draft-04/schema#") - return res.json() - - -def _get_dcae_df_schema(): - res = requests.get( - "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD" - ) - return res.json() - - def _protobuf_to_js(proto_path): """ Converts a protobuf to jsonschema and returns the generated schema as a JSON object. @@ -52,7 +39,7 @@ def _protobuf_to_js(proto_path): defs[defn.split(".")[1]] = defs.pop(defn) # make sure what we got out is a valid jsonschema - draft4 = _get_js_schema() + draft4 = utils.schema_schema.get() validate(instance=asjson, schema=draft4) return asjson @@ -141,8 +128,8 @@ def generate_dcae_data_formats(model_repo_path, model_name): data_formats = _generate_dcae_data_formats( "{0}/{1}/model.proto".format(model_repo_path, model_name), utils.get_metadata(model_repo_path, model_name), - _get_dcae_df_schema(), - _get_js_schema(), + utils.dataformat_schema.get(), + utils.schema_schema.get() ) # now we iterate over these and write a file to disk for each, since the dcae cli seems to want that diff --git a/adapter/acumos/aoconversion/scanner.py b/adapter/acumos/aoconversion/scanner.py index cf3ac79..aecf760 100644 --- a/adapter/acumos/aoconversion/scanner.py +++ b/adapter/acumos/aoconversion/scanner.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -47,8 +47,7 @@ class Config(object): Configuration parameters as attributes, make sure the required ones are there, populate defaults. """ - def __init__(self, dcaeurl, dcaeuser, onboardingurl, onboardinguser, onboardingpass, certfile, dockerregistry, dockeruser, dockerpass, acumosurl=None, interval=900, dockerhost='tcp://localhost:2375', tmpdir='/var/tmp/aoadapter', certverify=True, catalogs=None, port=None, **extras): - self.dcaeurl = dcaeurl + def __init__(self, dcaeuser, onboardingurl, onboardinguser, onboardingpass, certfile, dockerregistry, dockeruser, dockerpass, acumosurl=None, interval=900, dockerhost='tcp://localhost:2375', tmpdir='/var/tmp/aoadapter', certverify=True, catalogs=None, port=None, **extras): self.dcaeuser = dcaeuser def x(fmt, *args, **kwargs): diff --git a/adapter/acumos/aoconversion/spec_gen.py b/adapter/acumos/aoconversion/spec_gen.py index 4230f6d..8564434 100644 --- a/adapter/acumos/aoconversion/spec_gen.py +++ b/adapter/acumos/aoconversion/spec_gen.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,17 +23,9 @@ Generates DCAE component specs import json from jsonschema import validate -import requests from aoconversion import utils -def _get_dcae_cs_schema(): - res = requests.get( - "https://git.onap.org/dcaegen2/platform/plain/mod/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json" - ) - return res.json() - - def _get_format_version(target_name, data_formats): """ search through the data formats for name, make sure we have it, and retrieve the version @@ -103,7 +95,7 @@ def generate_spec(model_repo_path, model_name, data_formats, docker_uri): Returns the spec """ spec = _generate_spec( - model_name, utils.get_metadata(model_repo_path, model_name), _get_dcae_cs_schema(), data_formats, docker_uri + model_name, utils.get_metadata(model_repo_path, model_name), utils.component_schema.get(), data_formats, docker_uri ) fname = "{0}_dcae_component_specification.json".format(model_name) with open("{0}/{1}".format(model_repo_path, fname), "w") as f: diff --git a/adapter/acumos/aoconversion/utils.py b/adapter/acumos/aoconversion/utils.py index a5aae75..7403505 100644 --- a/adapter/acumos/aoconversion/utils.py +++ b/adapter/acumos/aoconversion/utils.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +19,30 @@ import json +class FetchSchemaError(RuntimeError): + pass + + +class _Schema: + def __init__(self, path): + self.ret = None + self.path = path + + def get(self): + try: + if self.ret is None: + with open(self.path, 'r') as f: + self.ret = json.loads(f.read()) + return self.ret + except Exception as e: + raise FetchSchemaError("Unexpected error from fetching schema", e) + + +schema_schema = _Schema('schemas/schema.json') +component_schema = _Schema('schemas/compspec.json') +dataformat_schema = _Schema('schemas/dataformat.json') + + def get_metadata(model_repo_path, model_name): # for now, assume it's called "metadata.json" return json.loads(open("{0}/{1}/metadata.json".format(model_repo_path, model_name), "r").read()) diff --git a/adapter/acumos/pom.xml b/adapter/acumos/pom.xml index 1ea941d..20883cf 100644 --- a/adapter/acumos/pom.xml +++ b/adapter/acumos/pom.xml @@ -3,7 +3,7 @@ ============LICENSE_START======================================================= org.onap.dcae ================================================================================ -Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. ================================================================================ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ limitations under the License. <modelVersion>4.0.0</modelVersion> <groupId>org.onap.dcaegen2.platform.adapter</groupId> <artifactId>dcaegen2-platform-adapter-acumos</artifactId> - <version>1.0.2-SNAPSHOT</version> + <version>1.0.3-SNAPSHOT</version> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <sonar.sources>.</sonar.sources> diff --git a/adapter/acumos/setup.py b/adapter/acumos/setup.py index e833307..35c7273 100644 --- a/adapter/acumos/setup.py +++ b/adapter/acumos/setup.py @@ -20,7 +20,7 @@ from setuptools import setup, find_packages setup( name="aoconversion", - version="1.0.2", + version="1.0.3", packages=find_packages(exclude=["tests.*", "tests"]), author="Tommy Carpenter, Andrew Gauld", author_email="tommy@research.att.com, agauld@att.com", diff --git a/adapter/acumos/tests/conftest.py b/adapter/acumos/tests/conftest.py new file mode 100644 index 0000000..80351c8 --- /dev/null +++ b/adapter/acumos/tests/conftest.py @@ -0,0 +1,31 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2020 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import os +import pytest +import requests +import aoconversion + + +@pytest.fixture +def mock_schemas(monkeypatch): + cwd = os.getcwd() + schemadir = cwd[:cwd.find('/adapter/acumos')] + '/mod/component-json-schemas' + monkeypatch.setattr(aoconversion.utils.component_schema, 'path', schemadir + '/component-specification/dcae-cli-v2/component-spec-schema.json') + monkeypatch.setattr(aoconversion.utils.dataformat_schema, 'path', schemadir + '/data-format/dcae-cli-v1/data-format-schema.json') + monkeypatch.setattr(aoconversion.utils.schema_schema, 'ret', requests.get('https://json-schema.org/draft-04/schema#').json()) diff --git a/adapter/acumos/tests/fixtures/dataformat_101.json b/adapter/acumos/tests/fixtures/dataformat_101.json deleted file mode 100644 index 66aa2ab..0000000 --- a/adapter/acumos/tests/fixtures/dataformat_101.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Data format specification schema Version 1.0.1", - "type": "object", - "oneOf": [{ - "properties": { - "self": { - "$ref": "#/definitions/self" - }, - "dataformatversion": { - "$ref": "#/definitions/dataformatversion" - }, - "reference": { - - "type": "object", - "description": "A reference to an external schema - name/version or url, if specified, is used to access the artifact", - "properties": { - "name": { - "$ref": "#/definitions/name" - }, - "url": { - "$ref": "#/definitions/url" - }, - "version": { - "$ref": "#/definitions/version" - }, - "format": { - "$ref": "#/definitions/format" - } - }, - "required": [ - "name", - "version", - "format" - ], - "additionalProperties": false - } - }, - "required": ["self", "dataformatversion", "reference"], - "additionalProperties": false - }, { - "properties": { - "self": { - "$ref": "#/definitions/self" - }, - "dataformatversion": { - "$ref": "#/definitions/dataformatversion" - }, - "jsonschema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "The actual JSON schema for this data format" - } - - }, - "required": ["self", "dataformatversion", "jsonschema"], - "additionalProperties": false - }, { - "properties": { - "self": { - "$ref": "#/definitions/self" - }, - "dataformatversion": { - "$ref": "#/definitions/dataformatversion" - }, - "delimitedschema": { - "type": "object", - "description": "A JSON schema for delimited files", - "properties": { - "delimiter": { - "enum": [",", "|", "\t"] - }, - "fields": { - "type": "array", - "description": "Array of field descriptions", - "items": { - "$ref": "#/definitions/field" - } - } - }, - "additionalProperties": false - } - }, - "required": ["self", "dataformatversion", "delimitedschema"], - "additionalProperties": false - }, { - "properties": { - "self": { - "$ref": "#/definitions/self" - }, - "dataformatversion": { - "$ref": "#/definitions/dataformatversion" - }, - "unstructured": { - "type": "object", - "description": "A JSON schema for unstructured text", - "properties": { - "encoding": { - "type": "string", - "enum": ["ASCII", "UTF-8", "UTF-16", "UTF-32"] - } - }, - "additionalProperties": false - - } - }, - "required": ["self", "dataformatversion", "unstructured"], - "additionalProperties": false - }], - "definitions": { - "url": { - "format": "uri" - }, - "name": { - "type": "string" - }, - "version": { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" - }, - "self": { - "description": "Identifying Information for the Data Format - name/version can be used to access the artifact", - "type": "object", - "properties": { - "name": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "description": { - "type": "string" - } - }, - "required": [ - "name", - "version" - ], - "additionalProperties": false - }, - "format": { - "description": "Reference schema type", - "type": "string", - "enum": [ - "JSON", - "Delimited Format", - "XML", - "Protocol Buffer", - "Unstructured" - ] - }, - "field": { - "description": "A field definition for the delimited schema", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "fieldtype": { - "description": "the field type - from the XML schema types", - "type": "string", - "enum": ["string", "boolean", - "decimal", "float", "double", - "duration", "dateTime", "time", - "date", "gYearMonth", "gYear", - "gMonthDay", "gDay", "gMonth", - "hexBinary", "base64Binary", - "anyURI", "QName", "NOTATION", - "normalizedString", "token", - "language", "IDREFS", "ENTITIES", - "NMTOKEN", "NMTOKENS", "Name", - "NCName", "ID", "IDREF", "ENTITY", - "integer", "nonPositiveInteger", - "negativeInteger", "long", "int", - "short", "byte", - "nonNegativeInteger", "unsignedLong", - "unsignedInt", "unsignedShort", - "unsignedByte", "positiveInteger" - - ] - }, - "fieldPattern": { - "description": "Regular expression that defines the field format", - "type": "integer" - }, - "fieldMaxLength": { - "description": "The maximum length of the field", - "type": "integer" - }, - "fieldMinLength": { - "description": "The minimum length of the field", - "type": "integer" - }, - "fieldMinimum": { - "description": "The minimum numeric value of the field", - "type": "integer" - }, - "fieldMaximum": { - "description": "The maximum numeric value of the field", - "type": "integer" - } - }, - "additionalProperties": false - }, - "dataformatversion": { - "type": "string", - "enum": ["1.0.0", "1.0.1"] - } - } -} diff --git a/adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json b/adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json deleted file mode 100644 index 0fd68ec..0000000 --- a/adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json +++ /dev/null @@ -1,918 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Component specification schema", - "type": "object", - "properties": { - "self": { - "type": "object", - "properties": { - "version": { - "$ref": "#/definitions/version" - }, - "description": { - "type": "string" - }, - "component_type": { - "type": "string", - "enum": [ - "docker", - "cdap" - ] - }, - "name": { - "$ref": "#/definitions/name" - } - }, - "required": [ - "version", - "name", - "description", - "component_type" - ] - }, - "streams": { - "type": "object", - "properties": { - "publishes": { - "type": "array", - "uniqueItems": true, - "items": { - "oneOf": [ - { "$ref": "#/definitions/publisher_http" }, - { "$ref": "#/definitions/publisher_message_router" }, - { "$ref": "#/definitions/publisher_data_router" } - ] - } - }, - "subscribes": { - "type": "array", - "uniqueItems": true, - "items": { - "oneOf": [ - { "$ref": "#/definitions/subscriber_http" }, - { "$ref": "#/definitions/subscriber_message_router" }, - { "$ref": "#/definitions/subscriber_data_router" } - ] - } - } - }, - "required": [ - "publishes", - "subscribes" - ] - }, - "services": { - "type": "object", - "properties": { - "calls": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/caller" - } - }, - "provides": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/provider" - } - } - }, - "required": [ - "calls", - "provides" - ] - }, - "parameters" : { - "anyOf" : [ - {"$ref": "#/definitions/docker-parameters"}, - {"$ref": "#/definitions/cdap-parameters"} - ] - }, - "auxilary": { - "oneOf" : [ - {"$ref": "#/definitions/auxilary_cdap"}, - {"$ref": "#/definitions/auxilary_docker"} - ] - }, - "artifacts": { - "type": "array", - "description": "List of component artifacts", - "items": { - "$ref": "#/definitions/artifact" - } - }, - "policy_info": { - "type": "object", - "properties": { - "policy": - { - "type": "array", - "items": - { - "type": "object", - "properties": - { - "node_label": - { - "type": "string" - }, - "policy_id": - { - "type": "string" - }, - "policy_model_id": - { - "type": "string" - } - }, - "required": ["node_label", "policy_model_id"] - } - } - }, - "additionalProperties": false - } - }, - "required": [ - "self", - "streams", - "services", - "parameters", - "auxilary", - "artifacts" - ], - "additionalProperties": false, - "definitions": { - "cdap-parameters": { - "description" : "There are three seperate ways to pass parameters to CDAP: app config, app preferences, program preferences. These are all treated as optional.", - "type": "object", - "properties" : { - "program_preferences": { - "description" : "A list of {program_id, program_type, program_preference} objects where program_preference is an object passed into program_id of type program_type", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/program_preference" - } - }, - "app_preferences" : { - "description" : "Parameters Passed down to the CDAP preference API", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - }, - "app_config" : { - "description" : "Parameters Passed down to the CDAP App Config", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - } - } - }, - "program_preference": { - "type": "object", - "properties": { - "program_type": { - "$ref": "#/definitions/program_type" - }, - "program_id": { - "type": "string" - }, - "program_pref":{ - "description" : "Parameters that the CDAP developer wants pushed to this program's preferences API. Optional", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - } - }, - "required": ["program_type", "program_id", "program_pref"] - }, - "program_type": { - "type": "string", - "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] - }, - "docker-parameters": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - }, - "parameter": { - "oneOf": [ - {"$ref": "#/definitions/parameter-list"}, - {"$ref": "#/definitions/parameter-other"} - ] - }, - "parameter-list": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "Only valid type is list, the entry_schema is required - which contains the type of the list element. All properties set for the parameter apply to all elements in the list at this time", - "type": "string", - "enum": ["list"] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "entry_schema": { - "description": "The optional property used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is list. This is the only type it is currently supported for.", - "type": "object", - "uniqueItems": true, - "items": {"$ref": "#/definitions/list-parameter"} - }, - "designer_editable": { - "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", - "type": "boolean" - }, - "sourced_at_deployment": { - "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", - "type": "boolean" - }, - "policy_editable": { - "description": "A required property that declares a parameter as editable by DevOps in Policy UI (true) or not (false).", - "type": "boolean" - }, - "policy_group": { - "description": "An optional property used to group policy_editable parameters into groups. Each group will become it's own policy model. Any parameters without this property will be grouped together to form their own policy model", - "type": "string" - }, - "policy_schema" :{ - "type": "array", - "uniqueItems": true, - "items": {"$ref": "#/definitions/policy_schema_parameter"} - } - }, - "required": [ - "name", - "value", - "description", - "designer_editable", - "policy_editable", - "sourced_at_deployment", - "entry_schema" - ], - "additionalProperties": false, - "dependencies": { - "policy_schema": ["policy_editable"] - } - }, - "parameter-other": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "The required data type for the parameter.", - "type": "string", - "enum": [ "string", "number", "boolean", "datetime" ] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "designer_editable": { - "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", - "type": "boolean" - }, - "sourced_at_deployment": { - "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", - "type": "boolean" - }, - "policy_editable": { - "description": "A required property that declares a parameter as editable in Policy UI (true) or not (false).", - "type": "boolean" - }, - "policy_group": { - "description": "An optional property used to group policy_editable parameters into groups. Each group will become it's own policy model. Any parameters without this property will be grouped together to form their own policy model", - "type": "string" - }, - "policy_schema" :{ - "description": "An optional property used to define policy_editable parameters as lists or maps", - "type": "array", - "uniqueItems": true, - "items": {"$ref": "#/definitions/policy_schema_parameter"} - } - }, - "required": [ - "name", - "value", - "description", - "designer_editable", - "sourced_at_deployment", - "policy_editable" - ], - "additionalProperties": false, - "dependencies": { - "policy_schema": ["policy_editable"] - } - }, - "list-parameter": { - "type": "object", - "properties": { - "type": { - "description": "The required data type for each parameter in the list.", - "type": "string", - "enum": ["string", "number"] - } - }, - "required": [ - "type" - ], - "additionalProperties": false - }, - "policy_schema_parameter": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "The required data type for the parameter.", - "type": "string", - "enum": [ "string", "number", "boolean", "datetime", "list", "map" ] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "entry_schema": { - "description": "The optional key that is used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is either list or map. If the type is list and the entry type is a simple type (string, number, boolean, datetime), follow with a simple string to describe the entry type. If the type is list and the entry type is a map, follow with an array to describe the keys for the entry map. If the type is list and the entry type is also list, this is not currently supported here. If the type is map, then follow with an array to describe the keys for this map. ", - "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/policy_schema_parameter"} - } - }, - "required": [ - "name", - "type" - ], - "additionalProperties": false - }, - "parameter-constraints": { - "type": "object", - "additionalProperties": false, - "properties": { - "equal": { - "description": "Constrains a property or parameter to a value equal to (‘=’) the value declared." - }, - "greater_than": { - "description": "Constrains a property or parameter to a value greater than (‘>’) the value declared.", - "type": "number" - }, - "greater_or_equal": { - "description": "Constrains a property or parameter to a value greater than or equal to (‘>=’) the value declared.", - "type": "number" - }, - "less_than": { - "description": "Constrains a property or parameter to a value less than (‘<’) the value declared.", - "type": "number" - }, - "less_or_equal": { - "description": "Constrains a property or parameter to a value less than or equal to (‘<=’) the value declared.", - "type": "number" - }, - "valid_values": { - "description": "Constrains a property or parameter to a value that is in the list of declared values.", - "type": "array" - }, - "length": { - "description": "Constrains the property or parameter to a value of a given length.", - "type": "number" - }, - "min_length": { - "description": "Constrains the property or parameter to a value to a minimum length.", - "type": "number" - }, - "max_length": { - "description": "Constrains the property or parameter to a value to a maximum length.", - "type": "number" - } - } - }, - "stream_message_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "message router", "message_router" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "publisher_http": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "http", - "https" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "publisher_message_router": { - "$ref": "#/definitions/stream_message_router" - }, - "publisher_data_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "data router", "data_router" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "subscriber_http": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "route": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "http", - "https" - ] - } - }, - "required": [ - "format", - "version", - "route", - "type" - ] - }, - "subscriber_message_router": { - "$ref": "#/definitions/stream_message_router" - }, - "subscriber_data_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "route": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "data router", "data_router" - ] - }, - "config_key": { - "description": "Data router subscribers require config info to setup their endpoints to handle requests. For example, needs username and password", - "type": "string" - } - }, - "required": [ - "format", - "version", - "route", - "type", - "config_key" - ] - }, - "provider" : { - "oneOf" : [ - {"$ref": "#/definitions/docker-provider"}, - {"$ref": "#/definitions/cdap-provider"} - ] - }, - "cdap-provider" : { - "type": "object", - "properties" : { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "service_name" : { - "type" : "string" - }, - "service_endpoint" : { - "type" : "string" - }, - "verb" : { - "type": "string", - "enum": ["GET", "PUT", "POST", "DELETE"] - } - }, - "required" : [ - "request", - "response", - "service_name", - "service_endpoint", - "verb" - ] - }, - "docker-provider": { - "type": "object", - "properties": { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "route": { - "type": "string" - }, - "verb": { - "type": "string", - "enum": ["GET", "PUT", "POST", "DELETE"] - } - }, - "required": [ - "request", - "response", - "route" - ] - }, - "caller": { - "type": "object", - "properties": { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "config_key": { - "type": "string" - } - }, - "required": [ - "request", - "response", - "config_key" - ] - }, - "formatPair": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - } - } - }, - "name": { - "type": "string" - }, - "version": { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" - }, - "artifact": { - "type": "object", - "description": "Component artifact object", - "properties": { - "uri": { - "type": "string", - "description": "Uri to artifact" - }, - "type": { - "type": "string", - "enum": ["jar", "docker image"] - } - }, - "required": ["uri", "type"] - }, - - "auxilary_cdap": { - "title": "cdap component specification schema", - "type": "object", - "properties": { - "streamname": { - "type": "string" - }, - "artifact_name" : { - "type": "string" - }, - "artifact_version" : { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" - }, - "namespace":{ - "type": "string", - "description" : "optional" - }, - "programs": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/cdap_program" - } - } - }, - "required": [ - "streamname", - "programs", - "artifact_name", - "artifact_version" - ] - }, - "cdap_program_type": { - "type": "string", - "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] - }, - "cdap_program": { - "type": "object", - "properties": { - "program_type": { - "$ref": "#/definitions/cdap_program_type" - }, - "program_id": { - "type": "string" - } - }, - "required": ["program_type", "program_id"] - }, - - "auxilary_docker": { - "title": "Docker component specification schema", - "type": "object", - "properties": { - "healthcheck": { - "description": "Define the health check that Consul should perfom for this component", - "type": "object", - "oneOf": [ - { "$ref": "#/definitions/docker_healthcheck_http" }, - { "$ref": "#/definitions/docker_healthcheck_script" } - ] - }, - "ports": { - "description": "Port mapping to be used for Docker containers. Each entry is of the format <container port>:<host port>.", - "type": "array", - "items": { - "type": "string" - } - }, - "log_info": { - "description": "Component specific details for logging", - "type": "object", - "properties": { - "log_directory": { - "description": "The path in the container where the component writes its logs. If the component is following the EELF requirements, this would be the directory where the four EELF files are being written. (Other logs can be placed in the directory--if their names in '.log', they'll also be sent into ELK.)", - "type": "string" - }, - "alternate_fb_path": { - "description": "By default, the log volume is mounted at /var/log/onap/<component_type> in the sidecar container's file system. 'alternate_fb_path' allows overriding the default. Will affect how the log data can be found in the ELK system.", - "type": "string" - } - }, - "additionalProperties": false - }, - "tls_info": { - "description": "Component information to use tls certificates", - "type": "object", - "properties": { - "cert_directory": { - "description": "The path in the container where the component certificates will be placed by the init container", - "type": "string" - }, - "use_tls": { - "description": "Boolean flag to determine if the application is using tls certificates", - "type": "boolean" - } - }, - "required": [ - "cert_directory","use_tls" - ], - "additionalProperties": false - }, - "databases": { - "description": "The databases the application is connecting to using the pgaas", - "type": "object", - "additionalProperties": { - "type": "string", - "enum": [ - "postgres" - ] - } - }, - "policy": { - "properties": { - "trigger_type": { - "description": "Only value of docker is supported at this time.", - "type": "string", - "enum": ["docker"] - }, - "script_path": { - "description": "Script command that will be executed for policy reconfiguration", - "type": "string" - } - }, - "required": [ - "trigger_type","script_path" - ], - "additionalProperties": false - }, - "volumes": { - "description": "Volume mapping to be used for Docker containers. Each entry is of the format below", - "type": "array", - "items": { - "type": "object", - "properties": { - "host":{ - "type":"object", - "path": {"type": "string"} - }, - "container":{ - "type":"object", - "bind": { "type": "string"}, - "mode": { "type": "string"} - } - } - } - } - }, - "required": [ - "healthcheck" - ], - "additionalProperties": false - }, - "docker_healthcheck_http": { - "properties": { - "type": { - "description": "Consul health check type", - "type": "string", - "enum": [ - "http", - "https" - ] - }, - "interval": { - "description": "Interval duration in seconds i.e. 10s", - "default": "15s", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds i.e. 10s", - "default": "1s", - "type": "string" - }, - "endpoint": { - "description": "Relative endpoint used by Consul to check health by making periodic HTTP GET calls", - "type": "string" - } - }, - "required": [ - "type", - "endpoint" - ] - }, - "docker_healthcheck_script": { - "properties": { - "type": { - "description": "Consul health check type", - "type": "string", - "enum": [ - "script", - "docker" - ] - }, - "interval": { - "description": "Interval duration in seconds i.e. 10s", - "default": "15s", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds i.e. 10s", - "default": "1s", - "type": "string" - }, - "script": { - "description": "Script command that will be executed by Consul to check health", - "type": "string" - } - }, - "required": [ - "type", - "script" - ] - } - } -} diff --git a/adapter/acumos/tests/fixtures/jsdraft4schema.json b/adapter/acumos/tests/fixtures/jsdraft4schema.json deleted file mode 100644 index bcbb847..0000000 --- a/adapter/acumos/tests/fixtures/jsdraft4schema.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "id": "http://json-schema.org/draft-04/schema#", - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "Core schema meta-schema", - "definitions": { - "schemaArray": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#" } - }, - "positiveInteger": { - "type": "integer", - "minimum": 0 - }, - "positiveIntegerDefault0": { - "allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ] - }, - "simpleTypes": { - "enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ] - }, - "stringArray": { - "type": "array", - "items": { "type": "string" }, - "minItems": 1, - "uniqueItems": true - } - }, - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "$schema": { - "type": "string" - }, - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "default": {}, - "multipleOf": { - "type": "number", - "minimum": 0, - "exclusiveMinimum": true - }, - "maximum": { - "type": "number" - }, - "exclusiveMaximum": { - "type": "boolean", - "default": false - }, - "minimum": { - "type": "number" - }, - "exclusiveMinimum": { - "type": "boolean", - "default": false - }, - "maxLength": { "$ref": "#/definitions/positiveInteger" }, - "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "pattern": { - "type": "string", - "format": "regex" - }, - "additionalItems": { - "anyOf": [ - { "type": "boolean" }, - { "$ref": "#" } - ], - "default": {} - }, - "items": { - "anyOf": [ - { "$ref": "#" }, - { "$ref": "#/definitions/schemaArray" } - ], - "default": {} - }, - "maxItems": { "$ref": "#/definitions/positiveInteger" }, - "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "uniqueItems": { - "type": "boolean", - "default": false - }, - "maxProperties": { "$ref": "#/definitions/positiveInteger" }, - "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "required": { "$ref": "#/definitions/stringArray" }, - "additionalProperties": { - "anyOf": [ - { "type": "boolean" }, - { "$ref": "#" } - ], - "default": {} - }, - "definitions": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "properties": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "patternProperties": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "dependencies": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { "$ref": "#" }, - { "$ref": "#/definitions/stringArray" } - ] - } - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true - }, - "type": { - "anyOf": [ - { "$ref": "#/definitions/simpleTypes" }, - { - "type": "array", - "items": { "$ref": "#/definitions/simpleTypes" }, - "minItems": 1, - "uniqueItems": true - } - ] - }, - "format": { "type": "string" }, - "allOf": { "$ref": "#/definitions/schemaArray" }, - "anyOf": { "$ref": "#/definitions/schemaArray" }, - "oneOf": { "$ref": "#/definitions/schemaArray" }, - "not": { "$ref": "#" } - }, - "dependencies": { - "exclusiveMaximum": [ "maximum" ], - "exclusiveMinimum": [ "minimum" ] - }, - "default": {} -} diff --git a/adapter/acumos/tests/test_df.py b/adapter/acumos/tests/test_df.py index cdf41c4..3872f35 100644 --- a/adapter/acumos/tests/test_df.py +++ b/adapter/acumos/tests/test_df.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,23 +17,21 @@ # ============LICENSE_END====================================================== from testing_helpers import get_json_fixture, get_fixture_path -from aoconversion import dataformat_gen +from aoconversion import dataformat_gen, utils TEST_META = get_json_fixture("models/example-model/metadata.json") -DRAFT_4_SCHEMA = get_json_fixture("jsdraft4schema.json") -DF_101 = get_json_fixture("dataformat_101.json") def test_get_needed_formats(): assert dataformat_gen._get_needed_formats(TEST_META) == ["NumbersIn", "NumberOut"] -def test_generate_dcae_data_formats(): +def test_generate_dcae_data_formats(mock_schemas): """ Test generating data formats from the protobuf """ test_proto_path = get_fixture_path("models/example-model/model.proto") - assert dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, DF_101, DRAFT_4_SCHEMA) == [ + assert dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, utils.dataformat_schema.get(), utils.schema_schema.get()) == [ { "self": {"name": "NumbersIn", "version": "1.0.0"}, "dataformatversion": "1.0.1", @@ -62,14 +60,14 @@ def test_generate_dcae_data_formats(): ] -def test_generate_dcae_data_formats_listofm(): +def test_generate_dcae_data_formats_listofm(mock_schemas): """ Test generating data formats from the protobuf This one tests the case where definitions needs to be populated in one of the data formats because it's referenced in a "top level" message """ test_meta = get_json_fixture("models/example-model-listofm/metadata.json") test_proto_path = get_fixture_path("models/example-model-listofm/model.proto") - assert dataformat_gen._generate_dcae_data_formats(test_proto_path, test_meta, DF_101, DRAFT_4_SCHEMA) == [ + assert dataformat_gen._generate_dcae_data_formats(test_proto_path, test_meta, utils.dataformat_schema.get(), utils.schema_schema.get()) == [ { "self": {"name": "ArgsList", "version": "1.0.0"}, "dataformatversion": "1.0.1", diff --git a/adapter/acumos/tests/test_fed.py b/adapter/acumos/tests/test_fed.py index 4d3636f..6119976 100644 --- a/adapter/acumos/tests/test_fed.py +++ b/adapter/acumos/tests/test_fed.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ import json import requests -from testing_helpers import get_json_fixture as get_test_json from testing_helpers import get_fixture_path as get_test_file from aoconversion import docker_gen as aoc_docker_gen @@ -157,11 +156,6 @@ _mockwebdata = { 'https://acumos/artifacts/a2/content': _r(file=get_test_file('models/example-model/model.proto')), 'https://acumos/artifacts/a3/content': _r(data=b'dummy zip archive data'), 'https://acumos/artifacts/a4/content': _r(file=get_test_file('models/example-model/metadata.json')), - 'http://json-schema.org/draft-04/schema#': _r(get_test_json('jsdraft4schema.json')), - 'https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD': _r(get_test_json('dataformat_101.json')), - 'http://dcaeurl//component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json': _r(get_test_json('dataformat_101.json')), - 'https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json;hb=HEAD': _r(get_test_json('dcae-cli-v2_component-spec-schema.json')), - 'http://dcaeurl//component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json': _r(get_test_json('dcae-cli-v2_component-spec-schema.json')), } @@ -170,7 +164,7 @@ _mockwebdata = { # -def test_aoconversion(tmpdir, monkeypatch): +def test_aoconversion(mock_schemas, tmpdir, monkeypatch): config = aoc_scanner.Config(dcaeurl='http://dcaeurl', dcaeuser='dcaeuser', onboardingurl='https://onboarding', onboardinguser='obuser', onboardingpass='obpass', acumosurl='https://acumos', certfile=None, dockerregistry='dockerregistry', dockeruser='registryuser', dockerpass='registrypassword') monkeypatch.setattr(aoc_docker_gen, 'APIClient', _mockdocker.APIClient) monkeypatch.setattr(requests, 'get', _mockwww(_mockwebdata)) diff --git a/adapter/acumos/tests/test_spec.py b/adapter/acumos/tests/test_spec.py index 7f6ec41..4771a1c 100644 --- a/adapter/acumos/tests/test_spec.py +++ b/adapter/acumos/tests/test_spec.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,22 +17,19 @@ # ============LICENSE_END====================================================== from testing_helpers import get_json_fixture, get_fixture_path -from aoconversion import dataformat_gen, spec_gen +from aoconversion import dataformat_gen, spec_gen, utils TEST_META = get_json_fixture("models/example-model/metadata.json") -DRAFT_4_SCHEMA = get_json_fixture("jsdraft4schema.json") -DF_101 = get_json_fixture("dataformat_101.json") -CS_SCHEMA = get_json_fixture("dcae-cli-v2_component-spec-schema.json") -def test_generate_spec(): +def test_generate_spec(mock_schemas): """ Test generating data formats from the protobuf """ test_proto_path = get_fixture_path("models/example-model/model.proto") - data_formats = dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, DF_101, DRAFT_4_SCHEMA) + data_formats = dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, utils.dataformat_schema.get(), utils.schema_schema.get()) assert spec_gen._generate_spec( - "example-model", TEST_META, CS_SCHEMA, data_formats, "nexus01.fake.com:18443/example-model:latest" + "example-model", TEST_META, utils.component_schema.get(), data_formats, "nexus01.fake.com:18443/example-model:latest" ) == { "self": { "version": "1.0.0", |