diff options
| author |   Andrew Gauld <agauld@att.com> | 2020-07-28 16:41:43 +0000 |
|---|---|---|
| committer | Andrew Gauld <agauld@att.com> | 2020-07-28 16:43:02 +0000 |
| commit | 711d4eae913f34e78aa10e8cd511f3b8e43cd54f (patch) | |
| tree | e0bf966addd527a3dd9812c6ed54e2ca0a97a10a | |
| parent | a1a37bde3365ebf0aaa543b2f10ebdb13bd4af40 (diff) | |
Pull JSON schemas at build/test not run time
Change-Id: I4f095a8a80f1c2868319c58bb0e9db916044d067
Issue-ID: DCAEGEN2-2221
Signed-off-by: Andrew Gauld <agauld@att.com>
27 files changed, 151 insertions, 1428 deletions
diff --git a/adapter/acumos/Changelog.md b/adapter/acumos/Changelog.md index 55bafa8..71e5520 100644 --- a/adapter/acumos/Changelog.md +++ b/adapter/acumos/Changelog.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.0.3] - 7/28/2020 + * Fetch json schemas during image build ## [1.0.2] - 3/26/2020 * Run as non-root ## [1.0.1] - 3/20/2020 diff --git a/adapter/acumos/Dockerfile b/adapter/acumos/Dockerfile index d1a0984..4f1bea2 100644 --- a/adapter/acumos/Dockerfile +++ b/adapter/acumos/Dockerfile @@ -25,6 +25,10 @@ RUN apt-get update && \ apt-get install -y npm nodejs && \ npm -v && \ npm install -g protobuf-jsonschema && \ + mkdir schemas && \ + wget -O schemas/schema.json 'https://json-schema.org/draft-04/schema#' && \ + wget -O schemas/compspec.json https://git.onap.org/dcaegen2/platform/plain/mod/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json && \ + wget -O schemas/dataformat.json https://git.onap.org/dcaegen2/platform/plain/mod/component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json && \ cd /tmp/build/ && \ python setup.py install && \ cd / && \ diff --git a/adapter/acumos/README.md b/adapter/acumos/README.md index 2de3845..3ebce54 100644 --- a/adapter/acumos/README.md +++ b/adapter/acumos/README.md @@ -44,8 +44,6 @@ This operates in 2 modes: - A configuration file, in YAML format, containing the following keys must be available. - dcaeurl - The base URL for the DCAE component and data format schemas. For - example, https://git.onap.org/dcaegen2/platform/cli/plain. dcaeuser - The DCAE user who will "own" the loaded data. onboardingurl - The URL for accessing the onboarding service. onboardinguser - The user ID for accessing the onboarding service. diff --git a/adapter/acumos/aoconversion/dataformat_gen.py b/adapter/acumos/aoconversion/dataformat_gen.py index a5ead96..71b1617 100644 --- a/adapter/acumos/aoconversion/dataformat_gen.py +++ b/adapter/acumos/aoconversion/dataformat_gen.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,22 +19,9 @@ from subprocess import PIPE, Popen import json from jsonschema import validate -import requests from aoconversion import utils, exceptions -def _get_js_schema(): - res = requests.get("http://json-schema.org/draft-04/schema#") - return res.json() - - -def _get_dcae_df_schema(): - res = requests.get( - "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD" - ) - return res.json() - - def _protobuf_to_js(proto_path): """ Converts a protobuf to jsonschema and returns the generated schema as a JSON object. @@ -52,7 +39,7 @@ def _protobuf_to_js(proto_path): defs[defn.split(".")[1]] = defs.pop(defn) # make sure what we got out is a valid jsonschema - draft4 = _get_js_schema() + draft4 = utils.schema_schema.get() validate(instance=asjson, schema=draft4) return asjson @@ -141,8 +128,8 @@ def generate_dcae_data_formats(model_repo_path, model_name): data_formats = _generate_dcae_data_formats( "{0}/{1}/model.proto".format(model_repo_path, model_name), utils.get_metadata(model_repo_path, model_name), - _get_dcae_df_schema(), - _get_js_schema(), + utils.dataformat_schema.get(), + utils.schema_schema.get() ) # now we iterate over these and write a file to disk for each, since the dcae cli seems to want that diff --git a/adapter/acumos/aoconversion/scanner.py b/adapter/acumos/aoconversion/scanner.py index cf3ac79..aecf760 100644 --- a/adapter/acumos/aoconversion/scanner.py +++ b/adapter/acumos/aoconversion/scanner.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -47,8 +47,7 @@ class Config(object): Configuration parameters as attributes, make sure the required ones are there, populate defaults. """ - def __init__(self, dcaeurl, dcaeuser, onboardingurl, onboardinguser, onboardingpass, certfile, dockerregistry, dockeruser, dockerpass, acumosurl=None, interval=900, dockerhost='tcp://localhost:2375', tmpdir='/var/tmp/aoadapter', certverify=True, catalogs=None, port=None, **extras): - self.dcaeurl = dcaeurl + def __init__(self, dcaeuser, onboardingurl, onboardinguser, onboardingpass, certfile, dockerregistry, dockeruser, dockerpass, acumosurl=None, interval=900, dockerhost='tcp://localhost:2375', tmpdir='/var/tmp/aoadapter', certverify=True, catalogs=None, port=None, **extras): self.dcaeuser = dcaeuser def x(fmt, *args, **kwargs): diff --git a/adapter/acumos/aoconversion/spec_gen.py b/adapter/acumos/aoconversion/spec_gen.py index 4230f6d..8564434 100644 --- a/adapter/acumos/aoconversion/spec_gen.py +++ b/adapter/acumos/aoconversion/spec_gen.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,17 +23,9 @@ Generates DCAE component specs import json from jsonschema import validate -import requests from aoconversion import utils -def _get_dcae_cs_schema(): - res = requests.get( - "https://git.onap.org/dcaegen2/platform/plain/mod/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json" - ) - return res.json() - - def _get_format_version(target_name, data_formats): """ search through the data formats for name, make sure we have it, and retrieve the version @@ -103,7 +95,7 @@ def generate_spec(model_repo_path, model_name, data_formats, docker_uri): Returns the spec """ spec = _generate_spec( - model_name, utils.get_metadata(model_repo_path, model_name), _get_dcae_cs_schema(), data_formats, docker_uri + model_name, utils.get_metadata(model_repo_path, model_name), utils.component_schema.get(), data_formats, docker_uri ) fname = "{0}_dcae_component_specification.json".format(model_name) with open("{0}/{1}".format(model_repo_path, fname), "w") as f: diff --git a/adapter/acumos/aoconversion/utils.py b/adapter/acumos/aoconversion/utils.py index a5aae75..7403505 100644 --- a/adapter/acumos/aoconversion/utils.py +++ b/adapter/acumos/aoconversion/utils.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +19,30 @@ import json +class FetchSchemaError(RuntimeError): + pass + + +class _Schema: + def __init__(self, path): + self.ret = None + self.path = path + + def get(self): + try: + if self.ret is None: + with open(self.path, 'r') as f: + self.ret = json.loads(f.read()) + return self.ret + except Exception as e: + raise FetchSchemaError("Unexpected error from fetching schema", e) + + +schema_schema = _Schema('schemas/schema.json') +component_schema = _Schema('schemas/compspec.json') +dataformat_schema = _Schema('schemas/dataformat.json') + + def get_metadata(model_repo_path, model_name): # for now, assume it's called "metadata.json" return json.loads(open("{0}/{1}/metadata.json".format(model_repo_path, model_name), "r").read()) diff --git a/adapter/acumos/pom.xml b/adapter/acumos/pom.xml index 1ea941d..20883cf 100644 --- a/adapter/acumos/pom.xml +++ b/adapter/acumos/pom.xml @@ -3,7 +3,7 @@ ============LICENSE_START======================================================= org.onap.dcae ================================================================================ -Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. ================================================================================ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ limitations under the License. <modelVersion>4.0.0</modelVersion> <groupId>org.onap.dcaegen2.platform.adapter</groupId> <artifactId>dcaegen2-platform-adapter-acumos</artifactId> - <version>1.0.2-SNAPSHOT</version> + <version>1.0.3-SNAPSHOT</version> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <sonar.sources>.</sonar.sources> diff --git a/adapter/acumos/setup.py b/adapter/acumos/setup.py index e833307..35c7273 100644 --- a/adapter/acumos/setup.py +++ b/adapter/acumos/setup.py @@ -20,7 +20,7 @@ from setuptools import setup, find_packages setup( name="aoconversion", - version="1.0.2", + version="1.0.3", packages=find_packages(exclude=["tests.*", "tests"]), author="Tommy Carpenter, Andrew Gauld", author_email="tommy@research.att.com, agauld@att.com", diff --git a/adapter/acumos/tests/conftest.py b/adapter/acumos/tests/conftest.py new file mode 100644 index 0000000..80351c8 --- /dev/null +++ b/adapter/acumos/tests/conftest.py @@ -0,0 +1,31 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2020 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import os +import pytest +import requests +import aoconversion + + +@pytest.fixture +def mock_schemas(monkeypatch): + cwd = os.getcwd() + schemadir = cwd[:cwd.find('/adapter/acumos')] + '/mod/component-json-schemas' + monkeypatch.setattr(aoconversion.utils.component_schema, 'path', schemadir + '/component-specification/dcae-cli-v2/component-spec-schema.json') + monkeypatch.setattr(aoconversion.utils.dataformat_schema, 'path', schemadir + '/data-format/dcae-cli-v1/data-format-schema.json') + monkeypatch.setattr(aoconversion.utils.schema_schema, 'ret', requests.get('https://json-schema.org/draft-04/schema#').json()) diff --git a/adapter/acumos/tests/fixtures/dataformat_101.json b/adapter/acumos/tests/fixtures/dataformat_101.json deleted file mode 100644 index 66aa2ab..0000000 --- a/adapter/acumos/tests/fixtures/dataformat_101.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Data format specification schema Version 1.0.1", - "type": "object", - "oneOf": [{ - "properties": { - "self": { - "$ref": "#/definitions/self" - }, - "dataformatversion": { - "$ref": "#/definitions/dataformatversion" - }, - "reference": { - - "type": "object", - "description": "A reference to an external schema - name/version or url, if specified, is used to access the artifact", - "properties": { - "name": { - "$ref": "#/definitions/name" - }, - "url": { - "$ref": "#/definitions/url" - }, - "version": { - "$ref": "#/definitions/version" - }, - "format": { - "$ref": "#/definitions/format" - } - }, - "required": [ - "name", - "version", - "format" - ], - "additionalProperties": false - } - }, - "required": ["self", "dataformatversion", "reference"], - "additionalProperties": false - }, { - "properties": { - "self": { - "$ref": "#/definitions/self" - }, - "dataformatversion": { - "$ref": "#/definitions/dataformatversion" - }, - "jsonschema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "The actual JSON schema for this data format" - } - - }, - "required": ["self", "dataformatversion", "jsonschema"], - "additionalProperties": false - }, { - "properties": { - "self": { - "$ref": "#/definitions/self" - }, - "dataformatversion": { - "$ref": "#/definitions/dataformatversion" - }, - "delimitedschema": { - "type": "object", - "description": "A JSON schema for delimited files", - "properties": { - "delimiter": { - "enum": [",", "|", "\t"] - }, - "fields": { - "type": "array", - "description": "Array of field descriptions", - "items": { - "$ref": "#/definitions/field" - } - } - }, - "additionalProperties": false - } - }, - "required": ["self", "dataformatversion", "delimitedschema"], - "additionalProperties": false - }, { - "properties": { - "self": { - "$ref": "#/definitions/self" - }, - "dataformatversion": { - "$ref": "#/definitions/dataformatversion" - }, - "unstructured": { - "type": "object", - "description": "A JSON schema for unstructured text", - "properties": { - "encoding": { - "type": "string", - "enum": ["ASCII", "UTF-8", "UTF-16", "UTF-32"] - } - }, - "additionalProperties": false - - } - }, - "required": ["self", "dataformatversion", "unstructured"], - "additionalProperties": false - }], - "definitions": { - "url": { - "format": "uri" - }, - "name": { - "type": "string" - }, - "version": { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" - }, - "self": { - "description": "Identifying Information for the Data Format - name/version can be used to access the artifact", - "type": "object", - "properties": { - "name": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "description": { - "type": "string" - } - }, - "required": [ - "name", - "version" - ], - "additionalProperties": false - }, - "format": { - "description": "Reference schema type", - "type": "string", - "enum": [ - "JSON", - "Delimited Format", - "XML", - "Protocol Buffer", - "Unstructured" - ] - }, - "field": { - "description": "A field definition for the delimited schema", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "fieldtype": { - "description": "the field type - from the XML schema types", - "type": "string", - "enum": ["string", "boolean", - "decimal", "float", "double", - "duration", "dateTime", "time", - "date", "gYearMonth", "gYear", - "gMonthDay", "gDay", "gMonth", - "hexBinary", "base64Binary", - "anyURI", "QName", "NOTATION", - "normalizedString", "token", - "language", "IDREFS", "ENTITIES", - "NMTOKEN", "NMTOKENS", "Name", - "NCName", "ID", "IDREF", "ENTITY", - "integer", "nonPositiveInteger", - "negativeInteger", "long", "int", - "short", "byte", - "nonNegativeInteger", "unsignedLong", - "unsignedInt", "unsignedShort", - "unsignedByte", "positiveInteger" - - ] - }, - "fieldPattern": { - "description": "Regular expression that defines the field format", - "type": "integer" - }, - "fieldMaxLength": { - "description": "The maximum length of the field", - "type": "integer" - }, - "fieldMinLength": { - "description": "The minimum length of the field", - "type": "integer" - }, - "fieldMinimum": { - "description": "The minimum numeric value of the field", - "type": "integer" - }, - "fieldMaximum": { - "description": "The maximum numeric value of the field", - "type": "integer" - } - }, - "additionalProperties": false - }, - "dataformatversion": { - "type": "string", - "enum": ["1.0.0", "1.0.1"] - } - } -} diff --git a/adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json b/adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json deleted file mode 100644 index 0fd68ec..0000000 --- a/adapter/acumos/tests/fixtures/dcae-cli-v2_component-spec-schema.json +++ /dev/null @@ -1,918 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Component specification schema", - "type": "object", - "properties": { - "self": { - "type": "object", - "properties": { - "version": { - "$ref": "#/definitions/version" - }, - "description": { - "type": "string" - }, - "component_type": { - "type": "string", - "enum": [ - "docker", - "cdap" - ] - }, - "name": { - "$ref": "#/definitions/name" - } - }, - "required": [ - "version", - "name", - "description", - "component_type" - ] - }, - "streams": { - "type": "object", - "properties": { - "publishes": { - "type": "array", - "uniqueItems": true, - "items": { - "oneOf": [ - { "$ref": "#/definitions/publisher_http" }, - { "$ref": "#/definitions/publisher_message_router" }, - { "$ref": "#/definitions/publisher_data_router" } - ] - } - }, - "subscribes": { - "type": "array", - "uniqueItems": true, - "items": { - "oneOf": [ - { "$ref": "#/definitions/subscriber_http" }, - { "$ref": "#/definitions/subscriber_message_router" }, - { "$ref": "#/definitions/subscriber_data_router" } - ] - } - } - }, - "required": [ - "publishes", - "subscribes" - ] - }, - "services": { - "type": "object", - "properties": { - "calls": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/caller" - } - }, - "provides": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/provider" - } - } - }, - "required": [ - "calls", - "provides" - ] - }, - "parameters" : { - "anyOf" : [ - {"$ref": "#/definitions/docker-parameters"}, - {"$ref": "#/definitions/cdap-parameters"} - ] - }, - "auxilary": { - "oneOf" : [ - {"$ref": "#/definitions/auxilary_cdap"}, - {"$ref": "#/definitions/auxilary_docker"} - ] - }, - "artifacts": { - "type": "array", - "description": "List of component artifacts", - "items": { - "$ref": "#/definitions/artifact" - } - }, - "policy_info": { - "type": "object", - "properties": { - "policy": - { - "type": "array", - "items": - { - "type": "object", - "properties": - { - "node_label": - { - "type": "string" - }, - "policy_id": - { - "type": "string" - }, - "policy_model_id": - { - "type": "string" - } - }, - "required": ["node_label", "policy_model_id"] - } - } - }, - "additionalProperties": false - } - }, - "required": [ - "self", - "streams", - "services", - "parameters", - "auxilary", - "artifacts" - ], - "additionalProperties": false, - "definitions": { - "cdap-parameters": { - "description" : "There are three seperate ways to pass parameters to CDAP: app config, app preferences, program preferences. These are all treated as optional.", - "type": "object", - "properties" : { - "program_preferences": { - "description" : "A list of {program_id, program_type, program_preference} objects where program_preference is an object passed into program_id of type program_type", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/program_preference" - } - }, - "app_preferences" : { - "description" : "Parameters Passed down to the CDAP preference API", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - }, - "app_config" : { - "description" : "Parameters Passed down to the CDAP App Config", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - } - } - }, - "program_preference": { - "type": "object", - "properties": { - "program_type": { - "$ref": "#/definitions/program_type" - }, - "program_id": { - "type": "string" - }, - "program_pref":{ - "description" : "Parameters that the CDAP developer wants pushed to this program's preferences API. Optional", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - } - }, - "required": ["program_type", "program_id", "program_pref"] - }, - "program_type": { - "type": "string", - "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] - }, - "docker-parameters": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - }, - "parameter": { - "oneOf": [ - {"$ref": "#/definitions/parameter-list"}, - {"$ref": "#/definitions/parameter-other"} - ] - }, - "parameter-list": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "Only valid type is list, the entry_schema is required - which contains the type of the list element. All properties set for the parameter apply to all elements in the list at this time", - "type": "string", - "enum": ["list"] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "entry_schema": { - "description": "The optional property used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is list. This is the only type it is currently supported for.", - "type": "object", - "uniqueItems": true, - "items": {"$ref": "#/definitions/list-parameter"} - }, - "designer_editable": { - "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", - "type": "boolean" - }, - "sourced_at_deployment": { - "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", - "type": "boolean" - }, - "policy_editable": { - "description": "A required property that declares a parameter as editable by DevOps in Policy UI (true) or not (false).", - "type": "boolean" - }, - "policy_group": { - "description": "An optional property used to group policy_editable parameters into groups. Each group will become it's own policy model. Any parameters without this property will be grouped together to form their own policy model", - "type": "string" - }, - "policy_schema" :{ - "type": "array", - "uniqueItems": true, - "items": {"$ref": "#/definitions/policy_schema_parameter"} - } - }, - "required": [ - "name", - "value", - "description", - "designer_editable", - "policy_editable", - "sourced_at_deployment", - "entry_schema" - ], - "additionalProperties": false, - "dependencies": { - "policy_schema": ["policy_editable"] - } - }, - "parameter-other": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "The required data type for the parameter.", - "type": "string", - "enum": [ "string", "number", "boolean", "datetime" ] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "designer_editable": { - "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", - "type": "boolean" - }, - "sourced_at_deployment": { - "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", - "type": "boolean" - }, - "policy_editable": { - "description": "A required property that declares a parameter as editable in Policy UI (true) or not (false).", - "type": "boolean" - }, - "policy_group": { - "description": "An optional property used to group policy_editable parameters into groups. Each group will become it's own policy model. Any parameters without this property will be grouped together to form their own policy model", - "type": "string" - }, - "policy_schema" :{ - "description": "An optional property used to define policy_editable parameters as lists or maps", - "type": "array", - "uniqueItems": true, - "items": {"$ref": "#/definitions/policy_schema_parameter"} - } - }, - "required": [ - "name", - "value", - "description", - "designer_editable", - "sourced_at_deployment", - "policy_editable" - ], - "additionalProperties": false, - "dependencies": { - "policy_schema": ["policy_editable"] - } - }, - "list-parameter": { - "type": "object", - "properties": { - "type": { - "description": "The required data type for each parameter in the list.", - "type": "string", - "enum": ["string", "number"] - } - }, - "required": [ - "type" - ], - "additionalProperties": false - }, - "policy_schema_parameter": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "The required data type for the parameter.", - "type": "string", - "enum": [ "string", "number", "boolean", "datetime", "list", "map" ] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "entry_schema": { - "description": "The optional key that is used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is either list or map. If the type is list and the entry type is a simple type (string, number, boolean, datetime), follow with a simple string to describe the entry type. If the type is list and the entry type is a map, follow with an array to describe the keys for the entry map. If the type is list and the entry type is also list, this is not currently supported here. If the type is map, then follow with an array to describe the keys for this map. ", - "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/policy_schema_parameter"} - } - }, - "required": [ - "name", - "type" - ], - "additionalProperties": false - }, - "parameter-constraints": { - "type": "object", - "additionalProperties": false, - "properties": { - "equal": { - "description": "Constrains a property or parameter to a value equal to (‘=’) the value declared." - }, - "greater_than": { - "description": "Constrains a property or parameter to a value greater than (‘>’) the value declared.", - "type": "number" - }, - "greater_or_equal": { - "description": "Constrains a property or parameter to a value greater than or equal to (‘>=’) the value declared.", - "type": "number" - }, - "less_than": { - "description": "Constrains a property or parameter to a value less than (‘<’) the value declared.", - "type": "number" - }, - "less_or_equal": { - "description": "Constrains a property or parameter to a value less than or equal to (‘<=’) the value declared.", - "type": "number" - }, - "valid_values": { - "description": "Constrains a property or parameter to a value that is in the list of declared values.", - "type": "array" - }, - "length": { - "description": "Constrains the property or parameter to a value of a given length.", - "type": "number" - }, - "min_length": { - "description": "Constrains the property or parameter to a value to a minimum length.", - "type": "number" - }, - "max_length": { - "description": "Constrains the property or parameter to a value to a maximum length.", - "type": "number" - } - } - }, - "stream_message_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "message router", "message_router" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "publisher_http": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "http", - "https" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "publisher_message_router": { - "$ref": "#/definitions/stream_message_router" - }, - "publisher_data_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "data router", "data_router" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "subscriber_http": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "route": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "http", - "https" - ] - } - }, - "required": [ - "format", - "version", - "route", - "type" - ] - }, - "subscriber_message_router": { - "$ref": "#/definitions/stream_message_router" - }, - "subscriber_data_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "route": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "data router", "data_router" - ] - }, - "config_key": { - "description": "Data router subscribers require config info to setup their endpoints to handle requests. For example, needs username and password", - "type": "string" - } - }, - "required": [ - "format", - "version", - "route", - "type", - "config_key" - ] - }, - "provider" : { - "oneOf" : [ - {"$ref": "#/definitions/docker-provider"}, - {"$ref": "#/definitions/cdap-provider"} - ] - }, - "cdap-provider" : { - "type": "object", - "properties" : { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "service_name" : { - "type" : "string" - }, - "service_endpoint" : { - "type" : "string" - }, - "verb" : { - "type": "string", - "enum": ["GET", "PUT", "POST", "DELETE"] - } - }, - "required" : [ - "request", - "response", - "service_name", - "service_endpoint", - "verb" - ] - }, - "docker-provider": { - "type": "object", - "properties": { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "route": { - "type": "string" - }, - "verb": { - "type": "string", - "enum": ["GET", "PUT", "POST", "DELETE"] - } - }, - "required": [ - "request", - "response", - "route" - ] - }, - "caller": { - "type": "object", - "properties": { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "config_key": { - "type": "string" - } - }, - "required": [ - "request", - "response", - "config_key" - ] - }, - "formatPair": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - } - } - }, - "name": { - "type": "string" - }, - "version": { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" - }, - "artifact": { - "type": "object", - "description": "Component artifact object", - "properties": { - "uri": { - "type": "string", - "description": "Uri to artifact" - }, - "type": { - "type": "string", - "enum": ["jar", "docker image"] - } - }, - "required": ["uri", "type"] - }, - - "auxilary_cdap": { - "title": "cdap component specification schema", - "type": "object", - "properties": { - "streamname": { - "type": "string" - }, - "artifact_name" : { - "type": "string" - }, - "artifact_version" : { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" - }, - "namespace":{ - "type": "string", - "description" : "optional" - }, - "programs": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/cdap_program" - } - } - }, - "required": [ - "streamname", - "programs", - "artifact_name", - "artifact_version" - ] - }, - "cdap_program_type": { - "type": "string", - "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] - }, - "cdap_program": { - "type": "object", - "properties": { - "program_type": { - "$ref": "#/definitions/cdap_program_type" - }, - "program_id": { - "type": "string" - } - }, - "required": ["program_type", "program_id"] - }, - - "auxilary_docker": { - "title": "Docker component specification schema", - "type": "object", - "properties": { - "healthcheck": { - "description": "Define the health check that Consul should perfom for this component", - "type": "object", - "oneOf": [ - { "$ref": "#/definitions/docker_healthcheck_http" }, - { "$ref": "#/definitions/docker_healthcheck_script" } - ] - }, - "ports": { - "description": "Port mapping to be used for Docker containers. Each entry is of the format <container port>:<host port>.", - "type": "array", - "items": { - "type": "string" - } - }, - "log_info": { - "description": "Component specific details for logging", - "type": "object", - "properties": { - "log_directory": { - "description": "The path in the container where the component writes its logs. If the component is following the EELF requirements, this would be the directory where the four EELF files are being written. (Other logs can be placed in the directory--if their names in '.log', they'll also be sent into ELK.)", - "type": "string" - }, - "alternate_fb_path": { - "description": "By default, the log volume is mounted at /var/log/onap/<component_type> in the sidecar container's file system. 'alternate_fb_path' allows overriding the default. Will affect how the log data can be found in the ELK system.", - "type": "string" - } - }, - "additionalProperties": false - }, - "tls_info": { - "description": "Component information to use tls certificates", - "type": "object", - "properties": { - "cert_directory": { - "description": "The path in the container where the component certificates will be placed by the init container", - "type": "string" - }, - "use_tls": { - "description": "Boolean flag to determine if the application is using tls certificates", - "type": "boolean" - } - }, - "required": [ - "cert_directory","use_tls" - ], - "additionalProperties": false - }, - "databases": { - "description": "The databases the application is connecting to using the pgaas", - "type": "object", - "additionalProperties": { - "type": "string", - "enum": [ - "postgres" - ] - } - }, - "policy": { - "properties": { - "trigger_type": { - "description": "Only value of docker is supported at this time.", - "type": "string", - "enum": ["docker"] - }, - "script_path": { - "description": "Script command that will be executed for policy reconfiguration", - "type": "string" - } - }, - "required": [ - "trigger_type","script_path" - ], - "additionalProperties": false - }, - "volumes": { - "description": "Volume mapping to be used for Docker containers. Each entry is of the format below", - "type": "array", - "items": { - "type": "object", - "properties": { - "host":{ - "type":"object", - "path": {"type": "string"} - }, - "container":{ - "type":"object", - "bind": { "type": "string"}, - "mode": { "type": "string"} - } - } - } - } - }, - "required": [ - "healthcheck" - ], - "additionalProperties": false - }, - "docker_healthcheck_http": { - "properties": { - "type": { - "description": "Consul health check type", - "type": "string", - "enum": [ - "http", - "https" - ] - }, - "interval": { - "description": "Interval duration in seconds i.e. 10s", - "default": "15s", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds i.e. 10s", - "default": "1s", - "type": "string" - }, - "endpoint": { - "description": "Relative endpoint used by Consul to check health by making periodic HTTP GET calls", - "type": "string" - } - }, - "required": [ - "type", - "endpoint" - ] - }, - "docker_healthcheck_script": { - "properties": { - "type": { - "description": "Consul health check type", - "type": "string", - "enum": [ - "script", - "docker" - ] - }, - "interval": { - "description": "Interval duration in seconds i.e. 10s", - "default": "15s", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds i.e. 10s", - "default": "1s", - "type": "string" - }, - "script": { - "description": "Script command that will be executed by Consul to check health", - "type": "string" - } - }, - "required": [ - "type", - "script" - ] - } - } -} diff --git a/adapter/acumos/tests/fixtures/jsdraft4schema.json b/adapter/acumos/tests/fixtures/jsdraft4schema.json deleted file mode 100644 index bcbb847..0000000 --- a/adapter/acumos/tests/fixtures/jsdraft4schema.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "id": "http://json-schema.org/draft-04/schema#", - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "Core schema meta-schema", - "definitions": { - "schemaArray": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#" } - }, - "positiveInteger": { - "type": "integer", - "minimum": 0 - }, - "positiveIntegerDefault0": { - "allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ] - }, - "simpleTypes": { - "enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ] - }, - "stringArray": { - "type": "array", - "items": { "type": "string" }, - "minItems": 1, - "uniqueItems": true - } - }, - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "$schema": { - "type": "string" - }, - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "default": {}, - "multipleOf": { - "type": "number", - "minimum": 0, - "exclusiveMinimum": true - }, - "maximum": { - "type": "number" - }, - "exclusiveMaximum": { - "type": "boolean", - "default": false - }, - "minimum": { - "type": "number" - }, - "exclusiveMinimum": { - "type": "boolean", - "default": false - }, - "maxLength": { "$ref": "#/definitions/positiveInteger" }, - "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "pattern": { - "type": "string", - "format": "regex" - }, - "additionalItems": { - "anyOf": [ - { "type": "boolean" }, - { "$ref": "#" } - ], - "default": {} - }, - "items": { - "anyOf": [ - { "$ref": "#" }, - { "$ref": "#/definitions/schemaArray" } - ], - "default": {} - }, - "maxItems": { "$ref": "#/definitions/positiveInteger" }, - "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "uniqueItems": { - "type": "boolean", - "default": false - }, - "maxProperties": { "$ref": "#/definitions/positiveInteger" }, - "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "required": { "$ref": "#/definitions/stringArray" }, - "additionalProperties": { - "anyOf": [ - { "type": "boolean" }, - { "$ref": "#" } - ], - "default": {} - }, - "definitions": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "properties": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "patternProperties": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "dependencies": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { "$ref": "#" }, - { "$ref": "#/definitions/stringArray" } - ] - } - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true - }, - "type": { - "anyOf": [ - { "$ref": "#/definitions/simpleTypes" }, - { - "type": "array", - "items": { "$ref": "#/definitions/simpleTypes" }, - "minItems": 1, - "uniqueItems": true - } - ] - }, - "format": { "type": "string" }, - "allOf": { "$ref": "#/definitions/schemaArray" }, - "anyOf": { "$ref": "#/definitions/schemaArray" }, - "oneOf": { "$ref": "#/definitions/schemaArray" }, - "not": { "$ref": "#" } - }, - "dependencies": { - "exclusiveMaximum": [ "maximum" ], - "exclusiveMinimum": [ "minimum" ] - }, - "default": {} -} diff --git a/adapter/acumos/tests/test_df.py b/adapter/acumos/tests/test_df.py index cdf41c4..3872f35 100644 --- a/adapter/acumos/tests/test_df.py +++ b/adapter/acumos/tests/test_df.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,23 +17,21 @@ # ============LICENSE_END====================================================== from testing_helpers import get_json_fixture, get_fixture_path -from aoconversion import dataformat_gen +from aoconversion import dataformat_gen, utils TEST_META = get_json_fixture("models/example-model/metadata.json") -DRAFT_4_SCHEMA = get_json_fixture("jsdraft4schema.json") -DF_101 = get_json_fixture("dataformat_101.json") def test_get_needed_formats(): assert dataformat_gen._get_needed_formats(TEST_META) == ["NumbersIn", "NumberOut"] -def test_generate_dcae_data_formats(): +def test_generate_dcae_data_formats(mock_schemas): """ Test generating data formats from the protobuf """ test_proto_path = get_fixture_path("models/example-model/model.proto") - assert dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, DF_101, DRAFT_4_SCHEMA) == [ + assert dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, utils.dataformat_schema.get(), utils.schema_schema.get()) == [ { "self": {"name": "NumbersIn", "version": "1.0.0"}, "dataformatversion": "1.0.1", @@ -62,14 +60,14 @@ def test_generate_dcae_data_formats(): ] -def test_generate_dcae_data_formats_listofm(): +def test_generate_dcae_data_formats_listofm(mock_schemas): """ Test generating data formats from the protobuf This one tests the case where definitions needs to be populated in one of the data formats because it's referenced in a "top level" message """ test_meta = get_json_fixture("models/example-model-listofm/metadata.json") test_proto_path = get_fixture_path("models/example-model-listofm/model.proto") - assert dataformat_gen._generate_dcae_data_formats(test_proto_path, test_meta, DF_101, DRAFT_4_SCHEMA) == [ + assert dataformat_gen._generate_dcae_data_formats(test_proto_path, test_meta, utils.dataformat_schema.get(), utils.schema_schema.get()) == [ { "self": {"name": "ArgsList", "version": "1.0.0"}, "dataformatversion": "1.0.1", diff --git a/adapter/acumos/tests/test_fed.py b/adapter/acumos/tests/test_fed.py index 4d3636f..6119976 100644 --- a/adapter/acumos/tests/test_fed.py +++ b/adapter/acumos/tests/test_fed.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ import json import requests -from testing_helpers import get_json_fixture as get_test_json from testing_helpers import get_fixture_path as get_test_file from aoconversion import docker_gen as aoc_docker_gen @@ -157,11 +156,6 @@ _mockwebdata = { 'https://acumos/artifacts/a2/content': _r(file=get_test_file('models/example-model/model.proto')), 'https://acumos/artifacts/a3/content': _r(data=b'dummy zip archive data'), 'https://acumos/artifacts/a4/content': _r(file=get_test_file('models/example-model/metadata.json')), - 'http://json-schema.org/draft-04/schema#': _r(get_test_json('jsdraft4schema.json')), - 'https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD': _r(get_test_json('dataformat_101.json')), - 'http://dcaeurl//component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json': _r(get_test_json('dataformat_101.json')), - 'https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json;hb=HEAD': _r(get_test_json('dcae-cli-v2_component-spec-schema.json')), - 'http://dcaeurl//component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json': _r(get_test_json('dcae-cli-v2_component-spec-schema.json')), } @@ -170,7 +164,7 @@ _mockwebdata = { # -def test_aoconversion(tmpdir, monkeypatch): +def test_aoconversion(mock_schemas, tmpdir, monkeypatch): config = aoc_scanner.Config(dcaeurl='http://dcaeurl', dcaeuser='dcaeuser', onboardingurl='https://onboarding', onboardinguser='obuser', onboardingpass='obpass', acumosurl='https://acumos', certfile=None, dockerregistry='dockerregistry', dockeruser='registryuser', dockerpass='registrypassword') monkeypatch.setattr(aoc_docker_gen, 'APIClient', _mockdocker.APIClient) monkeypatch.setattr(requests, 'get', _mockwww(_mockwebdata)) diff --git a/adapter/acumos/tests/test_spec.py b/adapter/acumos/tests/test_spec.py index 7f6ec41..4771a1c 100644 --- a/adapter/acumos/tests/test_spec.py +++ b/adapter/acumos/tests/test_spec.py @@ -1,7 +1,7 @@ # ============LICENSE_START==================================================== # org.onap.dcae # ============================================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,22 +17,19 @@ # ============LICENSE_END====================================================== from testing_helpers import get_json_fixture, get_fixture_path -from aoconversion import dataformat_gen, spec_gen +from aoconversion import dataformat_gen, spec_gen, utils TEST_META = get_json_fixture("models/example-model/metadata.json") -DRAFT_4_SCHEMA = get_json_fixture("jsdraft4schema.json") -DF_101 = get_json_fixture("dataformat_101.json") -CS_SCHEMA = get_json_fixture("dcae-cli-v2_component-spec-schema.json") -def test_generate_spec(): +def test_generate_spec(mock_schemas): """ Test generating data formats from the protobuf """ test_proto_path = get_fixture_path("models/example-model/model.proto") - data_formats = dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, DF_101, DRAFT_4_SCHEMA) + data_formats = dataformat_gen._generate_dcae_data_formats(test_proto_path, TEST_META, utils.dataformat_schema.get(), utils.schema_schema.get()) assert spec_gen._generate_spec( - "example-model", TEST_META, CS_SCHEMA, data_formats, "nexus01.fake.com:18443/example-model:latest" + "example-model", TEST_META, utils.component_schema.get(), data_formats, "nexus01.fake.com:18443/example-model:latest" ) == { "self": { "version": "1.0.0", diff --git a/mod/onboardingapi/Dockerfile b/mod/onboardingapi/Dockerfile index 2c3cd0d..d302b51 100644 --- a/mod/onboardingapi/Dockerfile +++ b/mod/onboardingapi/Dockerfile @@ -3,13 +3,21 @@ FROM python:3.7-alpine ARG UID=1000 ARG GID=1000 +ENV SERVER_URL=https://git.onap.org/dcaegen2/platform/plain/mod +ENV PATH_COMPONENT_SPEC=/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json +ENV PATH_DATA_FORMAT=/component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json +ENV PG_CONN= + COPY . /code WORKDIR /code RUN apk update && \ - apk add --virtual build-deps gcc python-dev musl-dev && \ + apk add --virtual build-deps gcc python3-dev musl-dev && \ apk add postgresql-dev bash && \ addgroup -g $GID dcaemod && \ adduser -s /bin/bash -u $UID -G dcaemod -D dcaemod && \ + mkdir schemas && \ + wget -O schemas/compspec.json $SERVER_URL$PATH_COMPONENT_SPEC && \ + wget -O schemas/dataformat.json $SERVER_URL$PATH_DATA_FORMAT && \ pip install . EXPOSE 8080 diff --git a/mod/onboardingapi/dcae_cli/_version.py b/mod/onboardingapi/dcae_cli/_version.py index b61bd9a..a636b5e 100644 --- a/mod/onboardingapi/dcae_cli/_version.py +++ b/mod/onboardingapi/dcae_cli/_version.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,4 +19,4 @@ # ECOMP is a trademark and service mark of AT&T Intellectual Property. # -*- coding: utf-8 -*- -__version__ = "2.12.1" +__version__ = "2.12.2" diff --git a/mod/onboardingapi/dcae_cli/catalog/mock/schema.py b/mod/onboardingapi/dcae_cli/catalog/mock/schema.py index 640d125..abfd9d5 100644 --- a/mod/onboardingapi/dcae_cli/catalog/mock/schema.py +++ b/mod/onboardingapi/dcae_cli/catalog/mock/schema.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -65,16 +65,23 @@ log = get_logger('Schema') class FetchSchemaError(RuntimeError): pass -def _fetch_schema(schema_path): - try: - server_url = cli_config.get_server_url() - return fetch_file_from_web(server_url, schema_path) - except requests.HTTPError as e: - raise FetchSchemaError("HTTP error from fetching schema", e) - except Exception as e: - raise FetchSchemaError("Unexpected error from fetching schema", e) - - +class _Schema: + def __init__(self, path): + self.ret = None + self.path = path + + def get(self): + try: + if self.ret is None: + with open(self.path, 'r') as f: + self.ret = json.loads(f.read()) + return self.ret + except Exception as e: + raise FetchSchemaError("Unexpected error from fetching schema", e) + +component_schema = _Schema('schemas/compspec.json') +dataformat_schema = _Schema('schemas/dataformat.json') + def _safe_dict(obj): '''Returns a dict from a dict or json string''' if isinstance(obj, str): @@ -82,7 +89,7 @@ def _safe_dict(obj): else: return obj -def _validate(fetch_schema_func, schema_path, spec): +def _validate(schema, spec): '''Validate the given spec Fetch the schema and then validate. Upon a error from fetching or validation, @@ -100,16 +107,12 @@ def _validate(fetch_schema_func, schema_path, spec): Nothing, silence is golden ''' try: - schema = fetch_schema_func(schema_path) - validate(_safe_dict(spec), schema) + validate(_safe_dict(spec), schema.get()) except ValidationError as e: reraise_with_msg(e, as_dcae=True) except FetchSchemaError as e: reraise_with_msg(e, as_dcae=True) -_validate_using_nexus = partial(_validate, _fetch_schema) - - def apply_defaults(properties_definition, properties): """Utility method to enforce expected defaults @@ -157,7 +160,7 @@ def apply_defaults_docker_config(config): """ # Apply health check defaults healthcheck_type = config["healthcheck"]["type"] - component_spec = _fetch_schema(cli_config.get_path_component_spec()) + component_spec = component_schema.get() if healthcheck_type in ["http", "https"]: apply_defaults_func = partial(apply_defaults, @@ -174,7 +177,7 @@ def apply_defaults_docker_config(config): return config def validate_component(spec): - _validate_using_nexus(cli_config.get_path_component_spec(), spec) + _validate(component_schema, spec) # REVIEW: Could not determine how to do this nicely in json schema. This is # not ideal. We want json schema to be the "it" for validation. @@ -188,4 +191,4 @@ def validate_component(spec): def validate_format(spec): path = cli_config.get_path_data_format() - _validate_using_nexus(path, spec) + _validate(dataformat_schema, spec) diff --git a/mod/onboardingapi/dcae_cli/catalog/mock/tests/test_schema.py b/mod/onboardingapi/dcae_cli/catalog/mock/tests/test_schema.py index 90674d9..9270268 100644 --- a/mod/onboardingapi/dcae_cli/catalog/mock/tests/test_schema.py +++ b/mod/onboardingapi/dcae_cli/catalog/mock/tests/test_schema.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -394,6 +394,8 @@ def test_validate(): good_path = "/correct_path" + goodschema = schema._Schema(good_path) + goodschema.ret = fake_schema def fetch_schema(path): if path == good_path: return fake_schema @@ -404,18 +406,20 @@ def test_validate(): good_instance = { "foo": "hello", "bar": 1776 } - schema._validate(fetch_schema, good_path, good_instance) + schema._validate(goodschema, good_instance) # Error from validating bad_instance = {} with pytest.raises(DcaeException): - schema._validate(fetch_schema, good_path, bad_instance) + schema._validate(goodschema, bad_instance) # Error from fetching bad_path = "/wrong_path" + badschema = schema._Schema(bad_path) + with pytest.raises(DcaeException): - schema._validate(fetch_schema, bad_path, good_instance) + schema._validate(badschema, good_instance) diff --git a/mod/onboardingapi/dcae_cli/cli.py b/mod/onboardingapi/dcae_cli/cli.py index fc2849b..afe547a 100644 --- a/mod/onboardingapi/dcae_cli/cli.py +++ b/mod/onboardingapi/dcae_cli/cli.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -74,11 +74,6 @@ def cli(ctx, verbose): if 'config' not in ctx.obj: config = conf.get_config() - if conf.should_force_reinit(config): - if click.confirm("You must reinitialize your dcae-cli configuration. Reinitialize now?", - abort=True): - _reinit_cli() - ctx.obj['config'] = config else: config = ctx.obj['config'] diff --git a/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py b/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py index be89722..580399f 100644 --- a/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py +++ b/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py @@ -40,8 +40,8 @@ def test_basic(monkeypatch, tmpdir, mock_db_url): # Setup config test_db_url = mock_db_url - config_dict = { "user": "ninny", "active_profile": "fake-solutioning", - "db_url": test_db_url, "cli_version": "2.0.0" } + config_dict = { "active_profile": "fake-solutioning", + "db_url": test_db_url } config_file = tmpdir.join("config.json") config_file.write(json.dumps(config_dict)) diff --git a/mod/onboardingapi/dcae_cli/conftest.py b/mod/onboardingapi/dcae_cli/conftest.py index 7956a81..154fa01 100644 --- a/mod/onboardingapi/dcae_cli/conftest.py +++ b/mod/onboardingapi/dcae_cli/conftest.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,14 +29,22 @@ import dcae_cli # The pytest recommendation was to place this file high up in the project. @pytest.fixture -def mock_cli_config(monkeypatch): +def mock_schemas(monkeypatch): + import os + cwd = os.getcwd() + schemadir = cwd[:cwd.find('/onboardingapi')] + '/component-json-schemas' + monkeypatch.setattr(dcae_cli.catalog.mock.schema.component_schema, 'path', schemadir + '/component-specification/dcae-cli-v2/component-spec-schema.json') + monkeypatch.setattr(dcae_cli.catalog.mock.schema.dataformat_schema, 'path', schemadir + '/data-format/dcae-cli-v1/data-format-schema.json') + +@pytest.fixture +def mock_cli_config(mock_schemas, monkeypatch): """Fixture to provide a mock dcae-cli configuration and profiles This fixture monkeypatches the respective get calls to return mock objects """ # NOTE: The component spec and data format in gerrit moved once already. # Might move again.. - fake_config = { "active_profile": "default", "user": "bob", + fake_config = { "active_profile": "default", "server_url": "https://git.onap.org/dcaegen2/platform/cli/plain", "db_url": "postgresql://postgres:abc123@localhost:5432/dcae_onboarding_db", "path_component_spec": "/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json", diff --git a/mod/onboardingapi/dcae_cli/util/config.py b/mod/onboardingapi/dcae_cli/util/config.py index f9936c3..8eb28c8 100644 --- a/mod/onboardingapi/dcae_cli/util/config.py +++ b/mod/onboardingapi/dcae_cli/util/config.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,16 +40,6 @@ def get_config_path(): return os.path.join(get_app_dir(), 'config.json') -def _init_config_user(): - while True: - user = click.prompt('Please enter your user id', type=str).strip() - - # There should be no special characters - if re.match("(?:\w*)\Z", user): - return user - else: - click.echo("Invalid user id. Please try again.") - def _init_config_server_url(): return click.prompt("Please enter the remote server url", type=str).strip() @@ -78,8 +68,6 @@ def _init_config(): # UPDATE: Keeping the server url even though the config was not found there. new_config["server_url"] = server_url - new_config["user"] = _init_config_user() - new_config["cli_version"] = _version.__version__ if "db_url" not in new_config or not new_config["db_url"]: # The seed configuration was not provided so manually set up the db @@ -94,11 +82,6 @@ def _init_config(): return new_config -def should_force_reinit(config): - """Configs older than 2.0.0 should be replaced""" - ver = config.get("cli_version", "0.0.0") - return int(ver.split(".")[0]) < 2 - def get_config(): '''Returns the configuration dictionary''' return get_pref(get_config_path(), _init_config) diff --git a/mod/onboardingapi/dcae_cli/util/tests/test_config.py b/mod/onboardingapi/dcae_cli/util/tests/test_config.py index 3b4cd6e..047cdf8 100644 --- a/mod/onboardingapi/dcae_cli/util/tests/test_config.py +++ b/mod/onboardingapi/dcae_cli/util/tests/test_config.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ def test_no_config(monkeypatch, tmpdir): '''Tests the creation and initialization of a config on a clean install''' monkeypatch.setattr(click, "get_app_dir", lambda app: str(tmpdir.realpath())) - mock_config = {'user': 'mock-user'} + mock_config = {} config_file = tmpdir.join("config.json") config_file.write(json.dumps(mock_config)) @@ -46,26 +46,13 @@ def test_no_config(monkeypatch, tmpdir): assert get_config() == mock_config -def test_init_config_user(monkeypatch): - good_case = "abc123" - values = [ good_case, "d-e-f", "g*h*i", "j k l" ] - - def fake_input(values, message, type="red"): - return values.pop() - - monkeypatch.setattr(click, 'prompt', partial(fake_input, values)) - assert config._init_config_user() == good_case - - def test_init_config(monkeypatch): - monkeypatch.setattr(config, '_init_config_user', lambda: "bigmama") monkeypatch.setattr(config, '_init_config_server_url', lambda: "http://some-nexus-in-the-sky.com") monkeypatch.setattr(dcae_cli.util, 'fetch_file_from_web', lambda server_url, path: { "db_url": "conn" }) - monkeypatch.setattr("dcae_cli._version.__version__", "2.X.X") - expected = {'cli_version': '2.X.X', 'user': 'bigmama', 'db_url': 'conn', + expected = {'db_url': 'conn', 'server_url': 'http://some-nexus-in-the-sky.com', 'active_profile': 'default' } assert expected == config._init_config() @@ -104,28 +91,17 @@ def test_init_config(monkeypatch): config._init_config() -def test_should_force_reinit(): - bad_config = {} - assert config.should_force_reinit(bad_config) == True - - old_config = { "cli_version": "1.0.0" } - assert config.should_force_reinit(old_config) == True - - uptodate_config = { "cli_version": "2.0.0" } - assert config.should_force_reinit(uptodate_config) == False - - def test_reinit_config(monkeypatch, tmpdir): monkeypatch.setattr(click, "get_app_dir", lambda app: str(tmpdir.realpath())) - new_config = { "user": "ninny", "db_url": "some-db" } + new_config = { "db_url": "some-db" } def init(): return new_config assert config._reinit_config(init) == new_config - old_config = { "user": "super", "db_url": "other-db", "hidden": "yo" } + old_config = { "db_url": "other-db", "hidden": "yo" } write_pref(old_config, get_config_path()) new_config["hidden"] = "yo" diff --git a/mod/onboardingapi/pom.xml b/mod/onboardingapi/pom.xml index be5e683..2c85ea2 100644 --- a/mod/onboardingapi/pom.xml +++ b/mod/onboardingapi/pom.xml @@ -1,7 +1,7 @@ <?xml version="1.0"?> <!-- ================================================================================ -Copyright (c) 2017-2019 AT&T Intellectual Property. All rights reserved. +Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved. ================================================================================ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ limitations under the License. <groupId>org.onap.dcaegen2.platform.mod</groupId> <artifactId>dcaegen2-platform-mod-onboardingapi</artifactId> <name>dcaegen2-platform-mod-onboardingapi</name> - <version>2.12.1-SNAPSHOT</version> + <version>2.12.2-SNAPSHOT</version> <url>http://maven.apache.org</url> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> diff --git a/mod/onboardingapi/start.sh b/mod/onboardingapi/start.sh index a938c98..5fae389 100755 --- a/mod/onboardingapi/start.sh +++ b/mod/onboardingapi/start.sh @@ -9,8 +9,7 @@ fi mkdir -p ~/.config/dcae-cli if [ ! -f ~/.config/dcae-cli/config.json ]; then echo "Creating dcae-cli config" - # TODO: Make this into a variable that gets fed in via docker run - echo "{\"server_url\": \"https://git.onap.org/dcaegen2/platform/plain/mod\", \"user\": \"api\", \"db_url\": \"$PG_CONN\", \"cli_version\": \"2.12.0\", \"path_component_spec\": \"/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json\", \"path_data_format\": \"/component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json\"}" > ~/.config/dcae-cli/config.json + echo "{\"server_url\": \"$SERVER_URL\", \"db_url\": \"$PG_CONN\", \"path_component_spec\": \"$PATH_COMPONENT_SPEC\", \"path_data_format\": \"$PATH_DATA_FORMAT\"}" > ~/.config/dcae-cli/config.json fi dcae_cli http --live |