Merge in changes there were made since seeding

* Fix DR config keys issue
* Add data format generate command
* Improve error messaging
* Add in support for inputs otherwise known as "sourced at deployment"

Change-Id: I9d97c30aeba587315d7fd1a18c38f71d8199d42b
Issue-Id: DCAEGEN2-91
Signed-off-by: Michael Hwang <mhwang@research.att.com>

6 files changed, 151 insertions, 5 deletions

diff --git a/dcae-cli/dcae_cli/commands/component/commands.py b/dcae-cli/dcae_cli/commands/component/commands.py
index 9c7a56d..4326636 100644
--- a/dcae-cli/dcae_cli/commands/component/commands.py
+++ b/dcae-cli/dcae_cli/commands/component/commands.py
@@ -29,7 +29,7 @@ import click
 
 from discovery_client import resolve_name
 
-from dcae_cli.util import profiles, load_json, dmaap
+from dcae_cli.util import profiles, load_json, dmaap, inputs
 from dcae_cli.util.run import run_component, dev_component
 from dcae_cli.util import discovery as dis
 from dcae_cli.util.discovery import DiscoveryNoDownstreamComponentError
@@ -165,6 +165,28 @@ def _parse_dmaap_file(dmaap_file):
         message = "Problems with parsing the dmaap file. Check to make sure that it is a valid json and is in the expected structure."
         raise DcaeException(message)
 
+_help_inputs_file = """
+Path to a file that contains a json that contains values to be used to bind to configuration parameters that have been marked as "sourced_at_deployment". The structure of the json is expected to be:
+
+ {
+   <parameter1 name>: value,
+   <parameter2 name>: value
+ }
+
+The "parameter name" is the value of the "name" property for the given configuration parameter.
+"""
+
+def _parse_inputs_file(inputs_file):
+    try:
+        with open(inputs_file, 'r+') as f:
+            inputs_map = json.load(f)
+            # TODO: Validation of schema in the future? Skipping this because
+            # dti_payload is not being intended to be used.
+            return inputs_map
+    except Exception as e:
+        message = "Problems with parsing the inputs file. Check to make sure that it is a valid json and is in the expected structure."
+        raise DcaeException(message)
+
 
 @component.command()
 @click.option('--external-ip', '-ip', default=None, help='The external IP address of the Docker host. Only used for Docker components.')
@@ -173,21 +195,29 @@ def _parse_dmaap_file(dmaap_file):
 @click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies')
 @click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
         help=_help_dmaap_file)
+@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
+        help=_help_inputs_file)
 @click.argument('component')
 @click.pass_obj
-def run(obj, external_ip, additional_user, attached, force, dmaap_file, component):
+def run(obj, external_ip, additional_user, attached, force, dmaap_file, component,
+        inputs_file):
     '''Runs the latest version of COMPONENT. You may optionally specify version via COMPONENT:VERSION'''
     cname, cver = parse_input(component)
     user, catalog = obj['config']['user'], obj['catalog']
 
     dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {}
+    inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {}
 
     try:
         run_component(user, cname, cver, catalog, additional_user, attached, force,
-                dmaap_map, external_ip)
+                dmaap_map, inputs_map, external_ip)
     except DiscoveryNoDownstreamComponentError as e:
         message = "Either run a compatible downstream component first or run with the --force flag to ignore this error"
         raise DcaeException(message)
+    except inputs.InputsValidationError as e:
+        click.echo("There is a problem. {0}".format(e))
+        message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct"
+        raise DcaeException(message)
 
 @component.command()
 @click.argument('component')
@@ -205,20 +235,28 @@ def undeploy(obj,  component):
 @click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies')
 @click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
         help=_help_dmaap_file)
+@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
+        help=_help_inputs_file)
 @click.pass_obj
-def dev(obj, specification, additional_user, force, dmaap_file):
+def dev(obj, specification, additional_user, force, dmaap_file, inputs_file):
     '''Set up component in development for discovery, use for local development'''
     user, catalog = obj['config']['user'], obj['catalog']
 
     dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {}
+    inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {}
 
     with open(specification, 'r+') as f:
         spec = json.loads(f.read())
         try:
-            dev_component(user, catalog, spec, additional_user, force, dmaap_map)
+            dev_component(user, catalog, spec, additional_user, force, dmaap_map,
+                    inputs_map)
         except DiscoveryNoDownstreamComponentError as e:
             message = "Either run a compatible downstream component first or run with the --force flag to ignore this error"
             raise DcaeException(message)
+        except inputs.InputsValidationError as e:
+            click.echo("There is a problem. {0}".format(e))
+            message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct"
+            raise DcaeException(message)
 
 
 @component.command()
diff --git a/dcae-cli/dcae_cli/commands/data_format/commands.py b/dcae-cli/dcae_cli/commands/data_format/commands.py
index 9114688..b942442 100644
--- a/dcae-cli/dcae_cli/commands/data_format/commands.py
+++ b/dcae-cli/dcae_cli/commands/data_format/commands.py
@@ -26,12 +26,22 @@ import json
 
 import click
 
+import genson
+
+import sys
+
+import os
+
+from jsonschema import Draft4Validator
+
 from dcae_cli.util import load_json
 from dcae_cli.util.logger import get_logger
+
 from dcae_cli.commands import util
 from dcae_cli.commands.util import create_table, parse_input
 
 from dcae_cli.catalog.exc import MissingEntry
+from dcae_cli.catalog.exc import DcaeException
 
 
 logger = get_logger('DataFormatCommand')
@@ -96,3 +106,59 @@ def publish(obj, data_format):
         click.echo("Data format has been published")
     else:
         click.echo("Data format could not be published")
+
+@data_format.command()
+@click.option('--keywords', is_flag=True, help='Adds a template of possible descriptive keywords', default=False)
+@click.argument('name_version', metavar="name:version",  required = True)
+@click.argument('file-or-dir-path', type=click.Path(resolve_path=True, exists=True, dir_okay=True, file_okay=True, readable=True),   metavar="file-or-dir-path")
+@click.pass_obj
+def generate(obj, name_version, file_or_dir_path, keywords):
+    '''Create schema from a file or directory examples'''
+    name, version = parse_input(name_version)
+    if version == None: 
+      version = ""
+    schema = genson.Schema()
+    if os.path.isfile(file_or_dir_path):
+      addfile(file_or_dir_path, schema)
+    else:
+      foundJSON = False
+      for root, dirs, files in os.walk(file_or_dir_path):
+          for filename in files:
+            fullfilename = os.path.join(file_or_dir_path, filename)
+            addfile(fullfilename,schema)
+            foundJSON = True
+      if foundJSON == False:
+        raise DcaeException('No JSON files found in ' + file_or_dir_path)
+
+    json_obj = json.loads(schema.to_json())
+    json_obj['$schema'] = "http://json-schema.org/draft-04/schema#"
+    jschema = json.dumps(json_obj)
+    jschema = jschema.replace('"required":', '"additionalproperties": true, "required":')
+    jschema = jschema.replace('"type":', ' "description": "", "type":')
+
+    if (keywords):
+      jschema = jschema.replace('"type": "string"', ' "maxLength": 0, "minLength": 0, "pattern": "", "type": "string"')
+      jschema = jschema.replace('"type": "integer"', ' "maximum": 0, "mininimum": 0, "multipleOf": 0, "type": "integer"')
+      jschema = jschema.replace('"type": "array"', ' "maxItems": 0, "minItems": 0, "uniqueItems": "false", "type": "array"')
+
+    jschema = '{ "self": { "name": "' + name + '", "version": "' + version + '", "description": ""} , "dataformatversion": "1.0.0", "jsonschema": ' + jschema + '}'
+    #Draft4Validator.check_schema(json.loads(jschema))
+    try:
+      print(json.dumps(json.loads(jschema), sort_keys=True, indent=4 ))
+    except ValueError:
+      raise DcaeException('Problem with JSON generation')
+
+def addfile(filename, schema):
+  try: 
+    fileadd = open(filename, "r")
+  except IOError:
+    raise DcaeException('Cannot open' + filename)
+  try: 
+    json_object = json.loads(fileadd.read())
+    schema.add_object(json_object)
+  except ValueError:
+    raise DcaeException('Bad JSON file: ' + filename)
+  finally:
+    fileadd.close()
+
+
diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/model/badjson b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/badjson
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/badjson
diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json
new file mode 100755
index 0000000..7db1e06
--- /dev/null
+++ b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json
@@ -0,0 +1,3 @@
+{
+   "foobar": "test 1"
+}
diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json
new file mode 100755
index 0000000..75a5fa4
--- /dev/null
+++ b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json
@@ -0,0 +1,4 @@
+{
+   "foobar2": "test 1"
+}
+
diff --git a/dcae-cli/dcae_cli/commands/tests/test_data_format_cmd.py b/dcae-cli/dcae_cli/commands/tests/test_data_format_cmd.py
index 754e6a7..b8402f6 100644
--- a/dcae-cli/dcae_cli/commands/tests/test_data_format_cmd.py
+++ b/dcae-cli/dcae_cli/commands/tests/test_data_format_cmd.py
@@ -77,6 +77,41 @@ def test_basic():
     spec_str = runner.invoke(cli, cmd, obj=obj).output
     assert df_spec == json.loads(spec_str)
 
+    # test of generate
+    bad_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'baddir')
+    cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_dir).split()
+    err_str = runner.invoke(cli, cmd, obj=obj).output
+    assert "does not exist" in err_str 
+
+    empty_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'emptydir')
+    try:
+      os.stat(empty_dir)
+    except:
+      os.mkdir(empty_dir)
+    cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(empty_dir).split()
+    err_str = runner.invoke(cli, cmd, obj=obj).output
+    assert "No JSON files found" in err_str 
+
+    bad_json = os.path.join(TEST_DIR, 'mocked_components', 'model', 'badjson')
+    cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_json).split()
+    err_str = runner.invoke(cli, cmd, obj=obj).output
+    assert "Bad JSON file" in err_str 
+
+    generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir')
+    cmd = "data_format generate --keywords name:1.0.2 {:} ".format(generate_dir).split()
+    out_str = runner.invoke(cli, cmd, obj=obj).output
+    assert '{\n    "dataformatversion": "1.0.0", \n    "jsonschema": {\n        "$schema": "http://json-schema.org/draft-04/schema#", \n        "description": "", \n        "properties": {\n            "foobar": {\n                "description": "", \n                "maxLength": 0, \n                "minLength": 0, \n                "pattern": "", \n                "type": "string"\n            }, \n            "foobar2": {\n                "description": "", \n                "maxLength": 0, \n                "minLength": 0, \n                "pattern": "", \n                "type": "string"\n            }\n        }, \n        "type": "object"\n    }, \n    "self": {\n        "description": "", \n        "name": "name", \n        "version": "1.0.2"\n    }\n}\n' ==  out_str
+
+    generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir')
+    cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split()
+    out_str = runner.invoke(cli, cmd, obj=obj).output
+    assert '{\n    "dataformatversion": "1.0.0", \n    "jsonschema": {\n        "$schema": "http://json-schema.org/draft-04/schema#", \n        "description": "", \n        "properties": {\n            "foobar": {\n                "description": "", \n                "type": "string"\n            }, \n            "foobar2": {\n                "description": "", \n                "type": "string"\n            }\n        }, \n        "type": "object"\n    }, \n    "self": {\n        "description": "", \n        "name": "name", \n        "version": "1.0.2"\n    }\n}\n' == out_str
+
+    generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir', 'ex1.json')
+    cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split()
+    out_str = runner.invoke(cli, cmd, obj=obj).output
+    assert '{\n    "dataformatversion": "1.0.0", \n    "jsonschema": {\n        "$schema": "http://json-schema.org/draft-04/schema#", \n        "additionalproperties": true, \n        "description": "", \n        "properties": {\n            "foobar": {\n                "description": "", \n                "type": "string"\n            }\n        }, \n        "required": [\n            "foobar"\n        ], \n        "type": "object"\n    }, \n    "self": {\n        "description": "", \n        "name": "name", \n        "version": "1.0.2"\n    }\n}\n' == out_str
+
 
 if __name__ == '__main__':
     '''Test area'''