diff options
Diffstat (limited to 'dcae-cli/dcae_cli/commands')
9 files changed, 179 insertions, 77 deletions
diff --git a/dcae-cli/dcae_cli/commands/catalog/commands.py b/dcae-cli/dcae_cli/commands/catalog/commands.py index 44771fa..dc6b27a 100644 --- a/dcae-cli/dcae_cli/commands/catalog/commands.py +++ b/dcae-cli/dcae_cli/commands/catalog/commands.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,10 +32,11 @@ def catalog(): @catalog.command(name="list") -@click.option("--expanded", is_flag=True, default=False, help="Display the expanded view - show all versions and all status") +@click.option("--expanded", is_flag=True, default=False, help="Display the expanded view - show all versions and all statuses") #TODO: @click.argument('query') @click.pass_obj def action_list(obj, expanded): + """Lists resources in the onboarding catalog""" # Query both components and data formats. Display both sets. user, catalog = obj['config']['user'], obj['catalog'] @@ -82,6 +83,7 @@ def action_list(obj, expanded): @click.argument("resource", metavar="name:version") @click.pass_obj def action_show(obj, resource): + """Provides more information about a resource""" # Query both components and data formats. Display both sets. name, ver = util.parse_input(resource) catalog = obj['catalog'] diff --git a/dcae-cli/dcae_cli/commands/component/commands.py b/dcae-cli/dcae_cli/commands/component/commands.py index 4326636..d91027f 100644 --- a/dcae-cli/dcae_cli/commands/component/commands.py +++ b/dcae-cli/dcae_cli/commands/component/commands.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,15 +26,18 @@ import json from pprint import pformat import click +import os from discovery_client import resolve_name -from dcae_cli.util import profiles, load_json, dmaap, inputs +from dcae_cli.util import profiles, load_json, dmaap, inputs, policy from dcae_cli.util.run import run_component, dev_component from dcae_cli.util import discovery as dis +from dcae_cli.util import docker_util as du from dcae_cli.util.discovery import DiscoveryNoDownstreamComponentError from dcae_cli.util.undeploy import undeploy_component from dcae_cli.util.exc import DcaeException + from dcae_cli.commands import util from dcae_cli.commands.util import parse_input, parse_input_pair, create_table @@ -133,7 +136,7 @@ def list_component(obj, latest, subscribes, publishes, provides, calls, deployed @click.argument('component', metavar="name:version") @click.pass_obj def show(obj, component): - '''Provides more information about COMPONENT''' + '''Provides more information about a COMPONENT''' cname, cver = parse_input(component) catalog = obj['catalog'] comp_spec = catalog.get_component_spec(cname, cver) @@ -162,9 +165,10 @@ def _parse_dmaap_file(dmaap_file): dmaap.validate_dmaap_map_schema(dmaap_map) return dmaap.apply_defaults_dmaap_map(dmaap_map) except Exception as e: - message = "Problems with parsing the dmaap file. Check to make sure that it is a valid json and is in the expected structure." + message = "Problems with parsing the dmaap file. Check to make sure that it is a valid json and is in the expected format." raise DcaeException(message) + _help_inputs_file = """ Path to a file that contains a json that contains values to be used to bind to configuration parameters that have been marked as "sourced_at_deployment". The structure of the json is expected to be: @@ -180,14 +184,36 @@ def _parse_inputs_file(inputs_file): try: with open(inputs_file, 'r+') as f: inputs_map = json.load(f) - # TODO: Validation of schema in the future? Skipping this because - # dti_payload is not being intended to be used. + # TODO: Validation of schema in the future? return inputs_map except Exception as e: - message = "Problems with parsing the inputs file. Check to make sure that it is a valid json and is in the expected structure." + message = "Problems with parsing the inputs file. Check to make sure that it is a valid json and is in the expected format." raise DcaeException(message) +_help_policy_file = """ +Path to a file that contains a json of an (update/remove) Policy change. +All "policies" can also be specified. +The structure of the json is expected to be: + +{ +"updated_policies": [{"policyName": "value", "": ""},{"policyName": "value", "": ""}], +"removed_policies": [{"policyName": "value", "": ""},{"policyName": "value", "": ""}], +"policies": [{"policyName": "value", "": ""},{"policyName": "value", "": ""}] +} +""" + +def _parse_policy_file(policy_file): + try: + with open(policy_file, 'r+') as f: + policy_change_file = json.load(f) + policy.validate_against_policy_schema(policy_change_file) + return policy_change_file + except Exception as e: + click.echo(format(e)) + message = "Problems with parsing the Policy file. Check to make sure that it is a valid json and is in the expected format." + raise DcaeException(message) + @component.command() @click.option('--external-ip', '-ip', default=None, help='The external IP address of the Docker host. Only used for Docker components.') @click.option('--additional-user', default=None, help='Additional user to grab instances from.') @@ -201,7 +227,11 @@ def _parse_inputs_file(inputs_file): @click.pass_obj def run(obj, external_ip, additional_user, attached, force, dmaap_file, component, inputs_file): - '''Runs the latest version of COMPONENT. You may optionally specify version via COMPONENT:VERSION''' + '''Runs latest (or specific) COMPONENT version. You may optionally specify version via COMPONENT:VERSION''' + + click.echo("Running the Component.....") + click.echo("") + cname, cver = parse_input(component) user, catalog = obj['config']['user'], obj['catalog'] @@ -215,7 +245,8 @@ def run(obj, external_ip, additional_user, attached, force, dmaap_file, componen message = "Either run a compatible downstream component first or run with the --force flag to ignore this error" raise DcaeException(message) except inputs.InputsValidationError as e: - click.echo("There is a problem. {0}".format(e)) + click.echo("ERROR: There is a problem. {0}".format(e)) + click.echo("") message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct" raise DcaeException(message) @@ -223,7 +254,7 @@ def run(obj, external_ip, additional_user, attached, force, dmaap_file, componen @click.argument('component') @click.pass_obj def undeploy(obj, component): - '''Undeploys the latest version of COMPONENT. You may optionally specify version via COMPONENT:VERSION''' + '''Undeploy latest (or specific) COMPONENT version. You may optionally specify version via COMPONENT:VERSION''' cname, cver = parse_input(component) user, catalog = obj['config']['user'], obj['catalog'] undeploy_component(user, cname, cver, catalog) @@ -254,7 +285,8 @@ def dev(obj, specification, additional_user, force, dmaap_file, inputs_file): message = "Either run a compatible downstream component first or run with the --force flag to ignore this error" raise DcaeException(message) except inputs.InputsValidationError as e: - click.echo("There is a problem. {0}".format(e)) + click.echo("ERROR: There is a problem. {0}".format(e)) + click.echo("") message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct" raise DcaeException(message) @@ -263,7 +295,7 @@ def dev(obj, specification, additional_user, force, dmaap_file, inputs_file): @click.argument('component') @click.pass_obj def publish(obj, component): - """Pushes COMPONENT to the public catalog""" + """Pushes a COMPONENT to the public catalog""" name, version = parse_input(component) user, catalog = obj['config']['user'], obj['catalog'] @@ -273,7 +305,7 @@ def publish(obj, component): unpub_formats = catalog.get_unpublished_formats(name, version) if unpub_formats: - click.echo("You must publish dependent data formats first:") + click.echo("ERROR: You must publish dependent data formats first:") click.echo("") click.echo("\n".join([":".join(uf) for uf in unpub_formats])) click.echo("") @@ -284,15 +316,79 @@ def publish(obj, component): if catalog.publish_component(user, name, version): click.echo("Component has been published") else: - click.echo("Component could not be published") + click.echo("ERROR: Component could not be published") @component.command() -@click.option('--update', is_flag=True, help='Updates a locally added component if it has not been already pushed') +@click.option('--update', is_flag=True, help='Updates a locally added component if it has not already been published') @click.argument('specification', type=click.Path(resolve_path=True, exists=True)) @click.pass_obj def add(obj, update, specification): + """Add Component to local onboarding catalog""" user, catalog = obj['config']['user'], obj['catalog'] spec = load_json(specification) catalog.add_component(user, spec, update) + + +@component.command() +@click.option('--policy-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), help=_help_policy_file) +@click.argument('component') +@click.pass_obj +def reconfig(obj, policy_file, component): + """Reconfigure COMPONENT for Policy change. + Modify Consul KV pairs for ('updated_policies', 'removed_policies', and 'policies') for Policy change event, + Execute the reconfig script(s) in the Docker container""" + + click.echo("Running Component Reconfiguration.....") + click.echo("") + + # Read and Validate the policy-file + policy_change_file = _parse_policy_file(policy_file) if policy_file else {} + + if not (policy_change_file): + click.echo("ERROR: For component 'reconfig', you must specify a --policy-file") + click.echo("") + return + else: + # The Component Spec contains the Policy 'Reconfig Script Path/ScriptName' + cname, cver = parse_input(component) + catalog = obj['catalog'] + comp_spec = catalog.get_component_spec(cname, cver) + + # Check if component is running and healthy + active_profile = profiles.get_profile() + consul_host = active_profile.consul_host + service_name = os.environ["SERVICE_NAME"] + if dis.is_healthy(consul_host, service_name): + pass + else: + click.echo("ERROR: The component must be running and healthy. It is not.") + click.echo("") + return + + try: + policy_reconfig_path = comp_spec['auxilary']['policy']['script_path'] + except KeyError: + click.echo("ERROR: Policy Reconfig Path (auxilary/policy/script_path) is not specified in the Component Spec") + click.echo("") + return + + kvUpdated = dis.policy_update(policy_change_file) + + if kvUpdated: + active_profile = profiles.get_profile() + docker_logins = dis.get_docker_logins() + + command = dis.build_policy_command(policy_reconfig_path, policy_change_file) + + # Run the Policy Reconfig script + client = du.get_docker_client(active_profile, docker_logins) + du.reconfigure(client, service_name, command) + else: + click.echo("ERROR: There was a problem updating the policies in Consul") + click.echo("") + return + + click.echo("") + click.echo("The End of Component Reconfiguration") diff --git a/dcae-cli/dcae_cli/commands/data_format/commands.py b/dcae-cli/dcae_cli/commands/data_format/commands.py index b942442..b952336 100644 --- a/dcae-cli/dcae_cli/commands/data_format/commands.py +++ b/dcae-cli/dcae_cli/commands/data_format/commands.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -53,11 +53,11 @@ def data_format(): @data_format.command() -@click.option('--update', is_flag=True, help='Updates a locally added data format if it has not been already pushed') +@click.option('--update', is_flag=True, help='Updates a locally added data format if it has not already been published') @click.argument('specification', type=click.Path(resolve_path=True, exists=True)) @click.pass_obj def add(obj, update, specification): - '''Tracks a data format file SPECIFICATION locally but does not push to the catalog''' + '''Tracks a Format file Specification locally, but does not push to the catalog''' spec = load_json(specification) user, catalog = obj['config']['user'], obj['catalog'] catalog.add_format(spec, user, update) @@ -67,7 +67,7 @@ def add(obj, update, specification): @click.option('--latest', is_flag=True, help='Only list the latest version of data formats') @click.pass_obj def list_format(obj, latest): - """Lists all your data formats""" + """Lists all your Data Formats""" user, catalog = obj['config']['user'], obj['catalog'] dfs = catalog.list_formats(latest, user=user) @@ -87,7 +87,7 @@ def list_format(obj, latest): @click.argument('data-format', metavar="name:version") @click.pass_obj def show(obj, data_format): - '''Provides more information about FORMAT''' + '''Provides more information about a Data Format''' name, ver = parse_input(data_format) spec = obj['catalog'].get_format_spec(name, ver) @@ -98,14 +98,14 @@ def show(obj, data_format): @click.argument('data-format') @click.pass_obj def publish(obj, data_format): - """Publishes data format to make publicly available""" + """Publish Format to make publicly available""" name, version = parse_input(data_format) user, catalog = obj['config']['user'], obj['catalog'] if catalog.publish_format(user, name, version): click.echo("Data format has been published") else: - click.echo("Data format could not be published") + click.echo("ERROR: Data format could not be published") @data_format.command() @click.option('--keywords', is_flag=True, help='Adds a template of possible descriptive keywords', default=False) @@ -115,7 +115,7 @@ def publish(obj, data_format): def generate(obj, name_version, file_or_dir_path, keywords): '''Create schema from a file or directory examples''' name, version = parse_input(name_version) - if version == None: + if version == None: version = "" schema = genson.Schema() if os.path.isfile(file_or_dir_path): @@ -149,11 +149,11 @@ def generate(obj, name_version, file_or_dir_path, keywords): raise DcaeException('Problem with JSON generation') def addfile(filename, schema): - try: + try: fileadd = open(filename, "r") except IOError: raise DcaeException('Cannot open' + filename) - try: + try: json_object = json.loads(fileadd.read()) schema.add_object(json_object) except ValueError: @@ -161,4 +161,3 @@ def addfile(filename, schema): finally: fileadd.close() - diff --git a/dcae-cli/dcae_cli/commands/profiles/commands.py b/dcae-cli/dcae_cli/commands/profiles/commands.py index dfd5517..df34b5c 100644 --- a/dcae-cli/dcae_cli/commands/profiles/commands.py +++ b/dcae-cli/dcae_cli/commands/profiles/commands.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ def profiles(): @profiles.command() @click.argument('name') def activate(name): - '''Sets profile NAME as the active profile''' + '''Sets profile (name) as the active profile''' activate_profile(name) @@ -56,7 +56,7 @@ def list_profiles(): @profiles.command() @click.argument('name') def show(name): - '''Prints the profile dictionary''' + '''Provides more information about a Profile''' profiles = get_profiles() try: click.echo(json.dumps(profiles[name], sort_keys=True, indent=4)) @@ -67,7 +67,7 @@ def show(name): @profiles.command() @click.argument('name', type=click.STRING) def create(name): - '''Creates a new profile NAME initialized with defaults''' + '''Creates new profile (name), with defaults''' create_profile(name) @@ -76,12 +76,12 @@ def create(name): @click.argument('key') @click.argument('value') def update(name, key, value): - '''Updates profile NAME such that KEY=VALUE''' + '''Updates profile (name) for specific Key/Value''' update_profile(name, **{key: value}) @profiles.command() @click.argument('name') def delete(name): - '''Deletes profile NAME''' + '''Deletes profile (name)''' delete_profile(name) diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json b/dcae-cli/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json index f2f12f9..9642a6e 100644 --- a/dcae-cli/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json +++ b/dcae-cli/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json @@ -35,15 +35,28 @@ "parameters": { "app_config" : [ {"name" : "some_param", - "description" : "some desc", - "value" : "some_value"} + "description" : "some desc", + "value" : "some_value", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} ], "app_preferences" : [ {"name" : "some_param2", "description" : "some desc2", - "value" : "some_value2"} + "value" : "some_value2", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} ], - "program_preferences" : [{"program_type" : "flows", "program_id" : "WhoFlow", "program_pref" : [{"name" : "some_param3","description" : "some desc3", "value" : "some_value3"}]}] + "program_preferences" : [{"program_type" : "flows", + "program_id" : "WhoFlow", + "program_pref" : [{"name" : "some_param3", + "description" : "some desc3", + "value" : "some_value3", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false}]}] }, "auxilary": { "streamname":"who", diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json b/dcae-cli/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json index c4a807e..83b5c28 100644 --- a/dcae-cli/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json +++ b/dcae-cli/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json @@ -35,14 +35,28 @@ "app_config" : [ {"name" : "some_param", "description" : "some desc", - "value" : "some_value"} + "value" : "some_value", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} ], "app_preferences" : [ {"name" : "some_param2", "description" : "some desc2", - "value" : "some_value2"} + "value" : "some_value2", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} ], - "program_preferences" : [{"program_type" : "flows", "program_id" : "WhoFlow", "program_pref" : [{"name" : "some_param3","description" : "some desc3", "value" : "some_value3"}]}] + "program_preferences" : [{"program_type" : "flows", + "program_id" : "WhoFlow", + "program_pref" : [{"name" : "some_param3", + "description" : "some desc3", + "value" : "some_value3", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} + ]}] }, "auxilary": { "streamname":"who", diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json b/dcae-cli/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json index 5508e90..5b86d9c 100644 --- a/dcae-cli/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json +++ b/dcae-cli/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json @@ -24,7 +24,10 @@ { "name": "sleep_sec", "value": 0.75, - "description": "Number of seconds to sleep between publishes" + "description": "Number of seconds to sleep between publishes", + "designer_editable": false, + "sourced_at_deployment": false, + "policy_editable": false } ], "auxilary": { diff --git a/dcae-cli/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json index e5f5889..3e2d142 100644 --- a/dcae-cli/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json +++ b/dcae-cli/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json @@ -29,7 +29,10 @@ { "name": "threshold", "value": 0.75, - "description": "Probability threshold to exceed to be anomalous" + "description": "Probability threshold to exceed to be anomalous", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false } ], "auxilary": { diff --git a/dcae-cli/dcae_cli/commands/tests/test_component_cmd.py b/dcae-cli/dcae_cli/commands/tests/test_component_cmd.py index 67769eb..2bba4cf 100644 --- a/dcae-cli/dcae_cli/commands/tests/test_component_cmd.py +++ b/dcae-cli/dcae_cli/commands/tests/test_component_cmd.py @@ -41,7 +41,7 @@ def _get_spec(path): def test_comp_docker(mock_cli_config, mock_db_url, obj=None): - obj = {'catalog': MockCatalog(purge_existing=True, db_name='dcae_cli.test.db', + obj = {'catalog': MockCatalog(purge_existing=True, db_name='dcae_cli.test.db', enforce_image=False, db_url=mock_db_url), 'config': {'user': 'test-user'}} @@ -113,63 +113,35 @@ def test_comp_cdap(obj=None): 3) runs a cdap component using our "Rework" broker 4) undeploys the cdap component using our "Rework" broker - NOTE: TODO: Mocking out the broker would be an improvement over this, probably. This is impure. Mocking the broker owuld be a huge undertaking, though. + NOTE: TODO: Mocking out the broker would be an improvement over this, probably. This is impure. Mocking the broker owuld be a huge undertaking, though. """ obj = {'catalog': MockCatalog(purge_existing=True, db_name='dcae_cli.test.db'), 'config': {'user': 'test-user'}} runner = CliRunner() - + #add the data format df = os.path.join(TEST_DIR, 'mocked_components', 'cdap', 'format.json') cmd = "data_format add {:}".format(df).split() assert runner.invoke(cli, cmd, obj=obj).exit_code == 0 - + #add the CDAP components # TODO: Need to update the host jar = 'http://make-me-valid/HelloWorld-3.4.3.jar' - + comp_cdap_start = os.path.join(TEST_DIR, 'mocked_components', 'cdap', 'spec_start.json') cmd = "component add {0}".format(comp_cdap_start).split() print(cmd) result = runner.invoke(cli, cmd, obj=obj) print(result.output) assert result.exit_code == 0 - + comp_cdap_end = os.path.join(TEST_DIR, 'mocked_components', 'cdap', 'spec_end.json') cmd = "component add {0}".format(comp_cdap_end).split() print(cmd) result = runner.invoke(cli, cmd, obj=obj) print(result.output) assert result.exit_code == 0 - - #run the terminating component first - cmd = "component run --force cdap.helloworld.mock.catalog.testing.endnode".split() - print(cmd) - result = runner.invoke(cli, cmd, obj=obj) - print(result.output) - assert result.exit_code == 0 - - #run the component again: this time the second component finds the first - cmd = "component run --force cdap.helloworld.mock.catalog.testing.startnode".split() - print(cmd) - result = runner.invoke(cli, cmd, obj=obj) - assert "config_key 'service_call_example' has no compatible downstream components." not in result.output #touchdown baby - assert result.exit_code == 0 - - #sleep - time.sleep(5) - - #delete the components - cmd = "component undeploy cdap.helloworld.mock.catalog.testing.startnode".split() - print(cmd) - result = runner.invoke(cli, cmd, obj=obj) - assert result.exit_code == 0 - - cmd = "component undeploy cdap.helloworld.mock.catalog.testing.endnode".split() - print(cmd) - result = runner.invoke(cli, cmd, obj=obj) - assert result.exit_code == 0 if __name__ == '__main__': '''Test area''' |