Pull JSON schemas at build/test not run time

Change-Id: I4f095a8a80f1c2868319c58bb0e9db916044d067
Issue-ID: DCAEGEN2-2221
Signed-off-by: Andrew Gauld <agauld@att.com>

11 files changed, 65 insertions, 89 deletions

diff --git a/mod/onboardingapi/Dockerfile b/mod/onboardingapi/Dockerfile
index 2c3cd0d..d302b51 100644
--- a/mod/onboardingapi/Dockerfile
+++ b/mod/onboardingapi/Dockerfile
@@ -3,13 +3,21 @@ FROM python:3.7-alpine
 ARG UID=1000
 ARG GID=1000
 
+ENV SERVER_URL=https://git.onap.org/dcaegen2/platform/plain/mod
+ENV PATH_COMPONENT_SPEC=/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json
+ENV PATH_DATA_FORMAT=/component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json
+ENV PG_CONN=
+
 COPY . /code
 WORKDIR /code
 RUN apk update && \
-    apk add --virtual build-deps gcc python-dev musl-dev && \
+    apk add --virtual build-deps gcc python3-dev musl-dev && \
     apk add postgresql-dev bash && \
     addgroup -g $GID dcaemod && \
     adduser -s /bin/bash -u $UID -G dcaemod -D dcaemod && \
+    mkdir schemas && \
+    wget -O schemas/compspec.json $SERVER_URL$PATH_COMPONENT_SPEC && \
+    wget -O schemas/dataformat.json $SERVER_URL$PATH_DATA_FORMAT && \
     pip install .
 
 EXPOSE 8080
diff --git a/mod/onboardingapi/dcae_cli/_version.py b/mod/onboardingapi/dcae_cli/_version.py
index b61bd9a..a636b5e 100644
--- a/mod/onboardingapi/dcae_cli/_version.py
+++ b/mod/onboardingapi/dcae_cli/_version.py
@@ -1,7 +1,7 @@
 # ============LICENSE_START=======================================================
 # org.onap.dcae
 # ================================================================================
-# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,4 +19,4 @@
 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
 
 # -*- coding: utf-8 -*-
-__version__ = "2.12.1"
+__version__ = "2.12.2"
diff --git a/mod/onboardingapi/dcae_cli/catalog/mock/schema.py b/mod/onboardingapi/dcae_cli/catalog/mock/schema.py
index 640d125..abfd9d5 100644
--- a/mod/onboardingapi/dcae_cli/catalog/mock/schema.py
+++ b/mod/onboardingapi/dcae_cli/catalog/mock/schema.py
@@ -1,7 +1,7 @@
 # ============LICENSE_START=======================================================
 # org.onap.dcae
 # ================================================================================
-# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,16 +65,23 @@ log = get_logger('Schema')
 class FetchSchemaError(RuntimeError):
     pass
 
-def _fetch_schema(schema_path):
-    try:
-        server_url = cli_config.get_server_url()
-        return fetch_file_from_web(server_url, schema_path)
-    except requests.HTTPError as e:
-        raise FetchSchemaError("HTTP error from fetching schema", e)
-    except Exception as e:
-        raise FetchSchemaError("Unexpected error from fetching schema", e)
-
-
+class _Schema:
+    def __init__(self, path):
+      self.ret = None
+      self.path = path
+
+    def get(self):
+        try:
+            if self.ret is None:
+                with open(self.path, 'r') as f:
+                    self.ret = json.loads(f.read())
+            return self.ret
+        except Exception as e:
+            raise FetchSchemaError("Unexpected error from fetching schema", e)
+
+component_schema = _Schema('schemas/compspec.json')
+dataformat_schema = _Schema('schemas/dataformat.json')
+          
 def _safe_dict(obj):
     '''Returns a dict from a dict or json string'''
     if isinstance(obj, str):
@@ -82,7 +89,7 @@ def _safe_dict(obj):
     else:
         return obj
 
-def _validate(fetch_schema_func, schema_path, spec):
+def _validate(schema, spec):
     '''Validate the given spec
 
     Fetch the schema and then validate. Upon a error from fetching or validation,
@@ -100,16 +107,12 @@ def _validate(fetch_schema_func, schema_path, spec):
     Nothing, silence is golden
     '''
     try:
-        schema = fetch_schema_func(schema_path)
-        validate(_safe_dict(spec), schema)
+        validate(_safe_dict(spec), schema.get())
     except ValidationError as e:
         reraise_with_msg(e, as_dcae=True)
     except FetchSchemaError as e:
         reraise_with_msg(e, as_dcae=True)
 
-_validate_using_nexus = partial(_validate, _fetch_schema)
-
-
 def apply_defaults(properties_definition, properties):
     """Utility method to enforce expected defaults
 
@@ -157,7 +160,7 @@ def apply_defaults_docker_config(config):
     """
     # Apply health check defaults
     healthcheck_type = config["healthcheck"]["type"]
-    component_spec = _fetch_schema(cli_config.get_path_component_spec())
+    component_spec = component_schema.get()
 
     if healthcheck_type in ["http", "https"]:
         apply_defaults_func = partial(apply_defaults,
@@ -174,7 +177,7 @@ def apply_defaults_docker_config(config):
     return config
 
 def validate_component(spec):
-    _validate_using_nexus(cli_config.get_path_component_spec(), spec)
+    _validate(component_schema, spec)
 
     # REVIEW: Could not determine how to do this nicely in json schema. This is
     # not ideal. We want json schema to be the "it" for validation.
@@ -188,4 +191,4 @@ def validate_component(spec):
 
 def validate_format(spec):
     path = cli_config.get_path_data_format()
-    _validate_using_nexus(path, spec)
+    _validate(dataformat_schema, spec)
diff --git a/mod/onboardingapi/dcae_cli/catalog/mock/tests/test_schema.py b/mod/onboardingapi/dcae_cli/catalog/mock/tests/test_schema.py
index 90674d9..9270268 100644
--- a/mod/onboardingapi/dcae_cli/catalog/mock/tests/test_schema.py
+++ b/mod/onboardingapi/dcae_cli/catalog/mock/tests/test_schema.py
@@ -1,7 +1,7 @@
 # ============LICENSE_START=======================================================
 # org.onap.dcae
 # ================================================================================
-# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -394,6 +394,8 @@ def test_validate():
 
     good_path = "/correct_path"
 
+    goodschema = schema._Schema(good_path)
+    goodschema.ret = fake_schema
     def fetch_schema(path):
         if path == good_path:
             return fake_schema
@@ -404,18 +406,20 @@ def test_validate():
 
     good_instance = { "foo": "hello", "bar": 1776 }
 
-    schema._validate(fetch_schema, good_path, good_instance)
+    schema._validate(goodschema, good_instance)
 
     # Error from validating
 
     bad_instance = {}
 
     with pytest.raises(DcaeException):
-        schema._validate(fetch_schema, good_path, bad_instance)
+        schema._validate(goodschema, bad_instance)
 
     # Error from fetching
 
     bad_path = "/wrong_path"
 
+    badschema = schema._Schema(bad_path)
+
     with pytest.raises(DcaeException):
-        schema._validate(fetch_schema, bad_path, good_instance)
+        schema._validate(badschema, good_instance)
diff --git a/mod/onboardingapi/dcae_cli/cli.py b/mod/onboardingapi/dcae_cli/cli.py
index fc2849b..afe547a 100644
--- a/mod/onboardingapi/dcae_cli/cli.py
+++ b/mod/onboardingapi/dcae_cli/cli.py
@@ -1,7 +1,7 @@
 # ============LICENSE_START=======================================================
 # org.onap.dcae
 # ================================================================================
-# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -74,11 +74,6 @@ def cli(ctx, verbose):
     if 'config' not in ctx.obj:
         config = conf.get_config()
 
-        if conf.should_force_reinit(config):
-            if click.confirm("You must reinitialize your dcae-cli configuration. Reinitialize now?",
-                    abort=True):
-                _reinit_cli()
-
         ctx.obj['config'] = config
     else:
         config = ctx.obj['config']
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py b/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py
index be89722..580399f 100644
--- a/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py
+++ b/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py
@@ -40,8 +40,8 @@ def test_basic(monkeypatch, tmpdir, mock_db_url):
 
     # Setup config
     test_db_url = mock_db_url
-    config_dict = { "user": "ninny", "active_profile": "fake-solutioning",
-            "db_url": test_db_url, "cli_version": "2.0.0" }
+    config_dict = { "active_profile": "fake-solutioning",
+            "db_url": test_db_url }
     config_file = tmpdir.join("config.json")
     config_file.write(json.dumps(config_dict))
 
diff --git a/mod/onboardingapi/dcae_cli/conftest.py b/mod/onboardingapi/dcae_cli/conftest.py
index 7956a81..154fa01 100644
--- a/mod/onboardingapi/dcae_cli/conftest.py
+++ b/mod/onboardingapi/dcae_cli/conftest.py
@@ -1,7 +1,7 @@
 # ============LICENSE_START=======================================================
 # org.onap.dcae
 # ================================================================================
-# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -29,14 +29,22 @@ import dcae_cli
 # The pytest recommendation was to place this file high up in the project.
 
 @pytest.fixture
-def mock_cli_config(monkeypatch):
+def mock_schemas(monkeypatch):
+    import os
+    cwd = os.getcwd()
+    schemadir = cwd[:cwd.find('/onboardingapi')] + '/component-json-schemas'
+    monkeypatch.setattr(dcae_cli.catalog.mock.schema.component_schema, 'path', schemadir + '/component-specification/dcae-cli-v2/component-spec-schema.json')
+    monkeypatch.setattr(dcae_cli.catalog.mock.schema.dataformat_schema, 'path', schemadir + '/data-format/dcae-cli-v1/data-format-schema.json')
+    
+@pytest.fixture
+def mock_cli_config(mock_schemas, monkeypatch):
     """Fixture to provide a mock dcae-cli configuration and profiles
 
     This fixture monkeypatches the respective get calls to return mock objects
     """
     # NOTE: The component spec and data format in gerrit moved once already.
     # Might move again..
-    fake_config = { "active_profile": "default", "user": "bob",
+    fake_config = { "active_profile": "default",
             "server_url": "https://git.onap.org/dcaegen2/platform/cli/plain",
             "db_url": "postgresql://postgres:abc123@localhost:5432/dcae_onboarding_db",
             "path_component_spec": "/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json",
diff --git a/mod/onboardingapi/dcae_cli/util/config.py b/mod/onboardingapi/dcae_cli/util/config.py
index f9936c3..8eb28c8 100644
--- a/mod/onboardingapi/dcae_cli/util/config.py
+++ b/mod/onboardingapi/dcae_cli/util/config.py
@@ -1,7 +1,7 @@
 # ============LICENSE_START=======================================================
 # org.onap.dcae
 # ================================================================================
-# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -40,16 +40,6 @@ def get_config_path():
     return os.path.join(get_app_dir(), 'config.json')
 
 
-def _init_config_user():
-    while True:
-        user = click.prompt('Please enter your user id', type=str).strip()
-
-        # There should be no special characters
-        if re.match("(?:\w*)\Z", user):
-            return user
-        else:
-            click.echo("Invalid user id. Please try again.")
-
 def _init_config_server_url():
     return click.prompt("Please enter the remote server url", type=str).strip()
 
@@ -78,8 +68,6 @@ def _init_config():
 
     # UPDATE: Keeping the server url even though the config was not found there.
     new_config["server_url"] = server_url
-    new_config["user"] = _init_config_user()
-    new_config["cli_version"] = _version.__version__
 
     if "db_url" not in new_config or not new_config["db_url"]:
         # The seed configuration was not provided so manually set up the db
@@ -94,11 +82,6 @@ def _init_config():
     return new_config
 
 
-def should_force_reinit(config):
-    """Configs older than 2.0.0 should be replaced"""
-    ver = config.get("cli_version", "0.0.0")
-    return int(ver.split(".")[0]) < 2
-
 def get_config():
     '''Returns the configuration dictionary'''
     return get_pref(get_config_path(), _init_config)
diff --git a/mod/onboardingapi/dcae_cli/util/tests/test_config.py b/mod/onboardingapi/dcae_cli/util/tests/test_config.py
index 3b4cd6e..047cdf8 100644
--- a/mod/onboardingapi/dcae_cli/util/tests/test_config.py
+++ b/mod/onboardingapi/dcae_cli/util/tests/test_config.py
@@ -1,7 +1,7 @@
 # ============LICENSE_START=======================================================
 # org.onap.dcae
 # ================================================================================
-# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -38,7 +38,7 @@ def test_no_config(monkeypatch, tmpdir):
     '''Tests the creation and initialization of a config on a clean install'''
     monkeypatch.setattr(click, "get_app_dir", lambda app: str(tmpdir.realpath()))
 
-    mock_config = {'user': 'mock-user'}
+    mock_config = {}
 
     config_file = tmpdir.join("config.json")
     config_file.write(json.dumps(mock_config))
@@ -46,26 +46,13 @@ def test_no_config(monkeypatch, tmpdir):
     assert get_config() == mock_config
 
 
-def test_init_config_user(monkeypatch):
-    good_case = "abc123"
-    values = [ good_case, "d-e-f", "g*h*i", "j k l" ]
-
-    def fake_input(values, message, type="red"):
-        return values.pop()
-
-    monkeypatch.setattr(click, 'prompt', partial(fake_input, values))
-    assert config._init_config_user() == good_case
-
-
 def test_init_config(monkeypatch):
-    monkeypatch.setattr(config, '_init_config_user', lambda: "bigmama")
     monkeypatch.setattr(config, '_init_config_server_url',
             lambda: "http://some-nexus-in-the-sky.com")
     monkeypatch.setattr(dcae_cli.util, 'fetch_file_from_web',
             lambda server_url, path: { "db_url": "conn" })
-    monkeypatch.setattr("dcae_cli._version.__version__", "2.X.X")
 
-    expected = {'cli_version': '2.X.X', 'user': 'bigmama', 'db_url': 'conn',
+    expected = {'db_url': 'conn',
             'server_url': 'http://some-nexus-in-the-sky.com',
             'active_profile': 'default' }
     assert expected == config._init_config()
@@ -104,28 +91,17 @@ def test_init_config(monkeypatch):
         config._init_config()
 
 
-def test_should_force_reinit():
-    bad_config = {}
-    assert config.should_force_reinit(bad_config) == True
-
-    old_config = { "cli_version": "1.0.0" }
-    assert config.should_force_reinit(old_config) == True
-
-    uptodate_config = { "cli_version": "2.0.0" }
-    assert config.should_force_reinit(uptodate_config) == False
-
-
 def test_reinit_config(monkeypatch, tmpdir):
     monkeypatch.setattr(click, "get_app_dir", lambda app: str(tmpdir.realpath()))
 
-    new_config = { "user": "ninny", "db_url": "some-db" }
+    new_config = { "db_url": "some-db" }
 
     def init():
         return new_config
 
     assert config._reinit_config(init) == new_config
 
-    old_config = { "user": "super", "db_url": "other-db", "hidden": "yo" }
+    old_config = { "db_url": "other-db", "hidden": "yo" }
     write_pref(old_config, get_config_path())
 
     new_config["hidden"] = "yo"
diff --git a/mod/onboardingapi/pom.xml b/mod/onboardingapi/pom.xml
index be5e683..2c85ea2 100644
--- a/mod/onboardingapi/pom.xml
+++ b/mod/onboardingapi/pom.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <!--
 ================================================================================
-Copyright (c) 2017-2019 AT&T Intellectual Property. All rights reserved.
+Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved.
 ================================================================================
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@ limitations under the License.
   <groupId>org.onap.dcaegen2.platform.mod</groupId>
   <artifactId>dcaegen2-platform-mod-onboardingapi</artifactId>
   <name>dcaegen2-platform-mod-onboardingapi</name>
-  <version>2.12.1-SNAPSHOT</version>
+  <version>2.12.2-SNAPSHOT</version>
   <url>http://maven.apache.org</url>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/mod/onboardingapi/start.sh b/mod/onboardingapi/start.sh
index a938c98..5fae389 100755
--- a/mod/onboardingapi/start.sh
+++ b/mod/onboardingapi/start.sh
@@ -9,8 +9,7 @@ fi
 mkdir -p ~/.config/dcae-cli
 if [ ! -f ~/.config/dcae-cli/config.json ]; then
     echo "Creating dcae-cli config"
-    # TODO: Make this into a variable that gets fed in via docker run
-    echo "{\"server_url\": \"https://git.onap.org/dcaegen2/platform/plain/mod\", \"user\": \"api\", \"db_url\": \"$PG_CONN\", \"cli_version\": \"2.12.0\", \"path_component_spec\": \"/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json\", \"path_data_format\": \"/component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json\"}" > ~/.config/dcae-cli/config.json
+    echo "{\"server_url\": \"$SERVER_URL\", \"db_url\": \"$PG_CONN\", \"path_component_spec\": \"$PATH_COMPONENT_SPEC\", \"path_data_format\": \"$PATH_DATA_FORMAT\"}" > ~/.config/dcae-cli/config.json
 fi
 
 dcae_cli http --live