From 0d4c19a9389a933cf5b5e83173f97f8cd72b7f5e Mon Sep 17 00:00:00 2001 From: Michael Hwang Date: Thu, 14 Sep 2017 13:06:21 -0400 Subject: Merge in changes there were made since seeding * Fix DR config keys issue * Add data format generate command * Improve error messaging * Add in support for inputs otherwise known as "sourced at deployment" Change-Id: I9d97c30aeba587315d7fd1a18c38f71d8199d42b Issue-Id: DCAEGEN2-91 Signed-off-by: Michael Hwang --- dcae-cli/dcae_cli/commands/component/commands.py | 48 ++++++++++++++-- dcae-cli/dcae_cli/commands/data_format/commands.py | 66 ++++++++++++++++++++++ .../commands/tests/mocked_components/model/badjson | 0 .../mocked_components/model/generatedir/ex1.json | 3 + .../mocked_components/model/generatedir/ex2.json | 4 ++ .../commands/tests/test_data_format_cmd.py | 35 ++++++++++++ 6 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 dcae-cli/dcae_cli/commands/tests/mocked_components/model/badjson create mode 100755 dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json create mode 100755 dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json (limited to 'dcae-cli/dcae_cli/commands') diff --git a/dcae-cli/dcae_cli/commands/component/commands.py b/dcae-cli/dcae_cli/commands/component/commands.py index 9c7a56d..4326636 100644 --- a/dcae-cli/dcae_cli/commands/component/commands.py +++ b/dcae-cli/dcae_cli/commands/component/commands.py @@ -29,7 +29,7 @@ import click from discovery_client import resolve_name -from dcae_cli.util import profiles, load_json, dmaap +from dcae_cli.util import profiles, load_json, dmaap, inputs from dcae_cli.util.run import run_component, dev_component from dcae_cli.util import discovery as dis from dcae_cli.util.discovery import DiscoveryNoDownstreamComponentError @@ -165,6 +165,28 @@ def _parse_dmaap_file(dmaap_file): message = "Problems with parsing the dmaap file. Check to make sure that it is a valid json and is in the expected structure." raise DcaeException(message) +_help_inputs_file = """ +Path to a file that contains a json that contains values to be used to bind to configuration parameters that have been marked as "sourced_at_deployment". The structure of the json is expected to be: + + { + : value, + : value + } + +The "parameter name" is the value of the "name" property for the given configuration parameter. +""" + +def _parse_inputs_file(inputs_file): + try: + with open(inputs_file, 'r+') as f: + inputs_map = json.load(f) + # TODO: Validation of schema in the future? Skipping this because + # dti_payload is not being intended to be used. + return inputs_map + except Exception as e: + message = "Problems with parsing the inputs file. Check to make sure that it is a valid json and is in the expected structure." + raise DcaeException(message) + @component.command() @click.option('--external-ip', '-ip', default=None, help='The external IP address of the Docker host. Only used for Docker components.') @@ -173,21 +195,29 @@ def _parse_dmaap_file(dmaap_file): @click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies') @click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), help=_help_dmaap_file) +@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), + help=_help_inputs_file) @click.argument('component') @click.pass_obj -def run(obj, external_ip, additional_user, attached, force, dmaap_file, component): +def run(obj, external_ip, additional_user, attached, force, dmaap_file, component, + inputs_file): '''Runs the latest version of COMPONENT. You may optionally specify version via COMPONENT:VERSION''' cname, cver = parse_input(component) user, catalog = obj['config']['user'], obj['catalog'] dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {} + inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {} try: run_component(user, cname, cver, catalog, additional_user, attached, force, - dmaap_map, external_ip) + dmaap_map, inputs_map, external_ip) except DiscoveryNoDownstreamComponentError as e: message = "Either run a compatible downstream component first or run with the --force flag to ignore this error" raise DcaeException(message) + except inputs.InputsValidationError as e: + click.echo("There is a problem. {0}".format(e)) + message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct" + raise DcaeException(message) @component.command() @click.argument('component') @@ -205,20 +235,28 @@ def undeploy(obj, component): @click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies') @click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), help=_help_dmaap_file) +@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), + help=_help_inputs_file) @click.pass_obj -def dev(obj, specification, additional_user, force, dmaap_file): +def dev(obj, specification, additional_user, force, dmaap_file, inputs_file): '''Set up component in development for discovery, use for local development''' user, catalog = obj['config']['user'], obj['catalog'] dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {} + inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {} with open(specification, 'r+') as f: spec = json.loads(f.read()) try: - dev_component(user, catalog, spec, additional_user, force, dmaap_map) + dev_component(user, catalog, spec, additional_user, force, dmaap_map, + inputs_map) except DiscoveryNoDownstreamComponentError as e: message = "Either run a compatible downstream component first or run with the --force flag to ignore this error" raise DcaeException(message) + except inputs.InputsValidationError as e: + click.echo("There is a problem. {0}".format(e)) + message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct" + raise DcaeException(message) @component.command() diff --git a/dcae-cli/dcae_cli/commands/data_format/commands.py b/dcae-cli/dcae_cli/commands/data_format/commands.py index 9114688..b942442 100644 --- a/dcae-cli/dcae_cli/commands/data_format/commands.py +++ b/dcae-cli/dcae_cli/commands/data_format/commands.py @@ -26,12 +26,22 @@ import json import click +import genson + +import sys + +import os + +from jsonschema import Draft4Validator + from dcae_cli.util import load_json from dcae_cli.util.logger import get_logger + from dcae_cli.commands import util from dcae_cli.commands.util import create_table, parse_input from dcae_cli.catalog.exc import MissingEntry +from dcae_cli.catalog.exc import DcaeException logger = get_logger('DataFormatCommand') @@ -96,3 +106,59 @@ def publish(obj, data_format): click.echo("Data format has been published") else: click.echo("Data format could not be published") + +@data_format.command() +@click.option('--keywords', is_flag=True, help='Adds a template of possible descriptive keywords', default=False) +@click.argument('name_version', metavar="name:version", required = True) +@click.argument('file-or-dir-path', type=click.Path(resolve_path=True, exists=True, dir_okay=True, file_okay=True, readable=True), metavar="file-or-dir-path") +@click.pass_obj +def generate(obj, name_version, file_or_dir_path, keywords): + '''Create schema from a file or directory examples''' + name, version = parse_input(name_version) + if version == None: + version = "" + schema = genson.Schema() + if os.path.isfile(file_or_dir_path): + addfile(file_or_dir_path, schema) + else: + foundJSON = False + for root, dirs, files in os.walk(file_or_dir_path): + for filename in files: + fullfilename = os.path.join(file_or_dir_path, filename) + addfile(fullfilename,schema) + foundJSON = True + if foundJSON == False: + raise DcaeException('No JSON files found in ' + file_or_dir_path) + + json_obj = json.loads(schema.to_json()) + json_obj['$schema'] = "http://json-schema.org/draft-04/schema#" + jschema = json.dumps(json_obj) + jschema = jschema.replace('"required":', '"additionalproperties": true, "required":') + jschema = jschema.replace('"type":', ' "description": "", "type":') + + if (keywords): + jschema = jschema.replace('"type": "string"', ' "maxLength": 0, "minLength": 0, "pattern": "", "type": "string"') + jschema = jschema.replace('"type": "integer"', ' "maximum": 0, "mininimum": 0, "multipleOf": 0, "type": "integer"') + jschema = jschema.replace('"type": "array"', ' "maxItems": 0, "minItems": 0, "uniqueItems": "false", "type": "array"') + + jschema = '{ "self": { "name": "' + name + '", "version": "' + version + '", "description": ""} , "dataformatversion": "1.0.0", "jsonschema": ' + jschema + '}' + #Draft4Validator.check_schema(json.loads(jschema)) + try: + print(json.dumps(json.loads(jschema), sort_keys=True, indent=4 )) + except ValueError: + raise DcaeException('Problem with JSON generation') + +def addfile(filename, schema): + try: + fileadd = open(filename, "r") + except IOError: + raise DcaeException('Cannot open' + filename) + try: + json_object = json.loads(fileadd.read()) + schema.add_object(json_object) + except ValueError: + raise DcaeException('Bad JSON file: ' + filename) + finally: + fileadd.close() + + diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/model/badjson b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/badjson new file mode 100644 index 0000000..e69de29 diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json new file mode 100755 index 0000000..7db1e06 --- /dev/null +++ b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json @@ -0,0 +1,3 @@ +{ + "foobar": "test 1" +} diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json new file mode 100755 index 0000000..75a5fa4 --- /dev/null +++ b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json @@ -0,0 +1,4 @@ +{ + "foobar2": "test 1" +} + diff --git a/dcae-cli/dcae_cli/commands/tests/test_data_format_cmd.py b/dcae-cli/dcae_cli/commands/tests/test_data_format_cmd.py index 754e6a7..b8402f6 100644 --- a/dcae-cli/dcae_cli/commands/tests/test_data_format_cmd.py +++ b/dcae-cli/dcae_cli/commands/tests/test_data_format_cmd.py @@ -77,6 +77,41 @@ def test_basic(): spec_str = runner.invoke(cli, cmd, obj=obj).output assert df_spec == json.loads(spec_str) + # test of generate + bad_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'baddir') + cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_dir).split() + err_str = runner.invoke(cli, cmd, obj=obj).output + assert "does not exist" in err_str + + empty_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'emptydir') + try: + os.stat(empty_dir) + except: + os.mkdir(empty_dir) + cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(empty_dir).split() + err_str = runner.invoke(cli, cmd, obj=obj).output + assert "No JSON files found" in err_str + + bad_json = os.path.join(TEST_DIR, 'mocked_components', 'model', 'badjson') + cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_json).split() + err_str = runner.invoke(cli, cmd, obj=obj).output + assert "Bad JSON file" in err_str + + generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir') + cmd = "data_format generate --keywords name:1.0.2 {:} ".format(generate_dir).split() + out_str = runner.invoke(cli, cmd, obj=obj).output + assert '{\n "dataformatversion": "1.0.0", \n "jsonschema": {\n "$schema": "http://json-schema.org/draft-04/schema#", \n "description": "", \n "properties": {\n "foobar": {\n "description": "", \n "maxLength": 0, \n "minLength": 0, \n "pattern": "", \n "type": "string"\n }, \n "foobar2": {\n "description": "", \n "maxLength": 0, \n "minLength": 0, \n "pattern": "", \n "type": "string"\n }\n }, \n "type": "object"\n }, \n "self": {\n "description": "", \n "name": "name", \n "version": "1.0.2"\n }\n}\n' == out_str + + generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir') + cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split() + out_str = runner.invoke(cli, cmd, obj=obj).output + assert '{\n "dataformatversion": "1.0.0", \n "jsonschema": {\n "$schema": "http://json-schema.org/draft-04/schema#", \n "description": "", \n "properties": {\n "foobar": {\n "description": "", \n "type": "string"\n }, \n "foobar2": {\n "description": "", \n "type": "string"\n }\n }, \n "type": "object"\n }, \n "self": {\n "description": "", \n "name": "name", \n "version": "1.0.2"\n }\n}\n' == out_str + + generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir', 'ex1.json') + cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split() + out_str = runner.invoke(cli, cmd, obj=obj).output + assert '{\n "dataformatversion": "1.0.0", \n "jsonschema": {\n "$schema": "http://json-schema.org/draft-04/schema#", \n "additionalproperties": true, \n "description": "", \n "properties": {\n "foobar": {\n "description": "", \n "type": "string"\n }\n }, \n "required": [\n "foobar"\n ], \n "type": "object"\n }, \n "self": {\n "description": "", \n "name": "name", \n "version": "1.0.2"\n }\n}\n' == out_str + if __name__ == '__main__': '''Test area''' -- cgit 1.2.3-korg