From c698e66797bad69b4c77b26b487bf8322989beb0 Mon Sep 17 00:00:00 2001 From: Michael Hwang Date: Tue, 12 Nov 2019 16:04:20 -0500 Subject: Copy dcae-cli->onboardingapi, copy component specs Issue-ID: DCAEGEN2-1860 Change-Id: I4805398c76479fad51cbdb74470ccc8f706ce9dc Signed-off-by: Michael Hwang --- mod/onboardingapi/dcae_cli/commands/__init__.py | 21 ++ .../dcae_cli/commands/catalog/__init__.py | 21 ++ .../dcae_cli/commands/catalog/commands.py | 115 ++++++ .../dcae_cli/commands/component/__init__.py | 25 ++ .../dcae_cli/commands/component/commands.py | 394 +++++++++++++++++++++ .../dcae_cli/commands/data_format/__init__.py | 25 ++ .../dcae_cli/commands/data_format/commands.py | 163 +++++++++ .../dcae_cli/commands/profiles/__init__.py | 25 ++ .../dcae_cli/commands/profiles/commands.py | 87 +++++ .../tests/mocked_components/cdap/format.json | 10 + .../tests/mocked_components/cdap/spec_end.json | 77 ++++ .../tests/mocked_components/cdap/spec_start.json | 78 ++++ .../collector/kpi-collector.comp.json | 47 +++ .../collector/vnf-kpi.format.json | 13 + .../model/anomaly-model.comp.json | 52 +++ .../commands/tests/mocked_components/model/badjson | 0 .../mocked_components/model/generatedir/ex1.json | 3 + .../mocked_components/model/generatedir/ex2.json | 4 + .../mocked_components/model/int-class.format.json | 18 + .../tests/mocked_components/viz/empty.format.json | 10 + .../tests/mocked_components/viz/line-viz.comp.json | 51 +++ .../mocked_components/viz/web-url.format.json | 19 + .../dcae_cli/commands/tests/test_component_cmd.py | 149 ++++++++ .../commands/tests/test_data_format_cmd.py | 122 +++++++ .../dcae_cli/commands/tests/test_profiles_cmd.py | 84 +++++ mod/onboardingapi/dcae_cli/commands/util.py | 111 ++++++ 26 files changed, 1724 insertions(+) create mode 100644 mod/onboardingapi/dcae_cli/commands/__init__.py create mode 100644 mod/onboardingapi/dcae_cli/commands/catalog/__init__.py create mode 100644 mod/onboardingapi/dcae_cli/commands/catalog/commands.py create mode 100644 mod/onboardingapi/dcae_cli/commands/component/__init__.py create mode 100644 mod/onboardingapi/dcae_cli/commands/component/commands.py create mode 100644 mod/onboardingapi/dcae_cli/commands/data_format/__init__.py create mode 100644 mod/onboardingapi/dcae_cli/commands/data_format/commands.py create mode 100644 mod/onboardingapi/dcae_cli/commands/profiles/__init__.py create mode 100644 mod/onboardingapi/dcae_cli/commands/profiles/commands.py create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/format.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/vnf-kpi.format.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/badjson create mode 100755 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json create mode 100755 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/int-class.format.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/empty.format.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/line-viz.comp.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/web-url.format.json create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/test_component_cmd.py create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/test_data_format_cmd.py create mode 100644 mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py create mode 100644 mod/onboardingapi/dcae_cli/commands/util.py (limited to 'mod/onboardingapi/dcae_cli/commands') diff --git a/mod/onboardingapi/dcae_cli/commands/__init__.py b/mod/onboardingapi/dcae_cli/commands/__init__.py new file mode 100644 index 0000000..ceddbb9 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/__init__.py @@ -0,0 +1,21 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- diff --git a/mod/onboardingapi/dcae_cli/commands/catalog/__init__.py b/mod/onboardingapi/dcae_cli/commands/catalog/__init__.py new file mode 100644 index 0000000..93f14ee --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/catalog/__init__.py @@ -0,0 +1,21 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from .commands import catalog diff --git a/mod/onboardingapi/dcae_cli/commands/catalog/commands.py b/mod/onboardingapi/dcae_cli/commands/catalog/commands.py new file mode 100644 index 0000000..dc6b27a --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/catalog/commands.py @@ -0,0 +1,115 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +""" +Queries onboarding catalog +""" +import click + +from dcae_cli.commands import util + + +@click.group() +def catalog(): + pass + + +@catalog.command(name="list") +@click.option("--expanded", is_flag=True, default=False, help="Display the expanded view - show all versions and all statuses") +#TODO: @click.argument('query') +@click.pass_obj +def action_list(obj, expanded): + """Lists resources in the onboarding catalog""" + # Query both components and data formats. Display both sets. + + user, catalog = obj['config']['user'], obj['catalog'] + + only_latest = not expanded + only_published = not expanded + + # TODO: Probably want to implement pagination + comps = catalog.list_components(latest=only_latest, only_published=only_published) + dfs = catalog.list_formats(latest=only_latest, only_published=only_published) + + def format_record_component(obj): + when_published = obj["when_published"].date() \ + if obj["when_published"] else "" + + return (obj["name"], obj["version"], obj["component_type"], + util.format_description(obj["description"]), obj["owner"], + util.get_status_string(obj), when_published) + + comps = [ format_record_component(comp) for comp in comps ] + + click.echo("") + click.echo("Components:") + click.echo(util.create_table(('Name', 'Version', 'Type', 'Description', 'Owner', 'Status', + 'Published'), comps)) + + def format_record_format(obj): + when_published = obj["when_published"].date() \ + if obj["when_published"] else "" + + return (obj["name"], obj["version"], + util.format_description(obj["description"]), obj["owner"], + util.get_status_string(obj), when_published) + + dfs = [ format_record_format(df) for df in dfs ] + + click.echo("") + click.echo("Data formats:") + click.echo(util.create_table(('Name', 'Version', 'Description', 'Owner', 'Status', + 'Published'), dfs)) + + +@catalog.command(name="show") +@click.argument("resource", metavar="name:version") +@click.pass_obj +def action_show(obj, resource): + """Provides more information about a resource""" + # Query both components and data formats. Display both sets. + name, ver = util.parse_input(resource) + catalog = obj['catalog'] + spec = None + + try: + spec = catalog.get_component_spec(name, ver) + + click.echo("") + click.echo("Component specification") + click.echo("-----------------------") + click.echo(util.format_json(spec)) + click.echo("") + except: + pass + + try: + spec = obj['catalog'].get_format_spec(name, ver) + + click.echo("") + click.echo("Data format") + click.echo("-----------") + click.echo(util.format_json(spec)) + click.echo("") + except: + pass + + if not spec: + click.echo("No matching component nor data format found") diff --git a/mod/onboardingapi/dcae_cli/commands/component/__init__.py b/mod/onboardingapi/dcae_cli/commands/component/__init__.py new file mode 100644 index 0000000..b1f4a8f --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/component/__init__.py @@ -0,0 +1,25 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +""" +Provides the component group +""" +from .commands import component diff --git a/mod/onboardingapi/dcae_cli/commands/component/commands.py b/mod/onboardingapi/dcae_cli/commands/component/commands.py new file mode 100644 index 0000000..b2483d1 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/component/commands.py @@ -0,0 +1,394 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +""" +Provides component commands +""" +import json +from pprint import pformat + +import click +import os + +from discovery_client import resolve_name + +from dcae_cli.util import profiles, load_json, dmaap, inputs, policy +from dcae_cli.util.run import run_component, dev_component +from dcae_cli.util import discovery as dis +from dcae_cli.util import docker_util as du +from dcae_cli.util.discovery import DiscoveryNoDownstreamComponentError +from dcae_cli.util.undeploy import undeploy_component +from dcae_cli.util.exc import DcaeException + +from dcae_cli.commands import util +from dcae_cli.commands.util import parse_input, parse_input_pair, create_table + +from dcae_cli.catalog.exc import MissingEntry + + +@click.group() +def component(): + pass + + +@component.command(name='list') +@click.option('--latest', is_flag=True, default=True, help='Only list the latest version of components which match the filter criteria') +@click.option('--subscribes', '-sub', multiple=True, help='Only list components which subscribe to FORMAT') +@click.option('--publishes', '-pub', multiple=True, help='Only list components which publish FORMAT') +@click.option('--provides', '-pro', multiple=True, type=(str, str), help='Only list components which provide services REQ_FORMAT RESP_FORMAT') +@click.option('--calls', '-cal', multiple=True, type=(str, str), help='Only list components which call services REQ_FORMAT RESP_FORMAT') +@click.option('--deployed', is_flag=True, default=False, help='Display the deployed view. Shows details of deployed instances.') +@click.pass_obj +def list_component(obj, latest, subscribes, publishes, provides, calls, deployed): + '''Lists components in the public catalog. Uses flags to filter results.''' + subs = list(map(parse_input, subscribes)) if subscribes else None + pubs = list(map(parse_input, publishes)) if publishes else None + provs = list(map(parse_input_pair, provides)) if provides else None + cals = list(map(parse_input_pair, calls)) if calls else None + + user, catalog = obj['config']['user'], obj['catalog'] + # TODO: How about components that you don't own but you have deployed? + comps = catalog.list_components(subs, pubs, provs, cals, latest, user=user) + + active_profile = profiles.get_profile() + consul_host = active_profile.consul_host + + click.echo("Active profile: {0}".format(profiles.get_active_name())) + click.echo("") + + def format_resolve_results(results): + """Format the results from the resolve_name function call""" + if results: + # Most likely the results will always be length one until we migrate + # to a different way of registering names + return "\n".join([ pformat(result) for result in results ]) + else: + return None + + def get_instances_as_rows(comp): + """Get all deployed running instances of a component plus details about + those instances and return as a list of rows""" + cname = comp["name"] + cver = comp["version"] + ctype = comp["component_type"] + + instances = dis.get_healthy_instances(user, cname, cver) + instances_status = ["Healthy"]*len(instances) + instances_conns = [ format_resolve_results(resolve_name(consul_host, instance)) \ + for instance in instances ] + + instances_defective = dis.get_defective_instances(user, cname, cver) + instances_status += ["Defective"]*len(instances_defective) + instances_conns += [""]*len(instances_defective) + + instances += instances_defective + + return list(zip(instances, instances_status, instances_conns)) + + # Generate grouped rows where a grouped row is (name, version, type, [instances]) + grouped_rows = [ (comp, get_instances_as_rows(comp)) for comp in comps ] + + # Display + if deployed: + def display_deployed(comp, instances): + cname = comp["name"] + cver = comp["version"] + ctype = comp["component_type"] + + click.echo("Name: {0}".format(cname)) + click.echo("Version: {0}".format(cver)) + click.echo("Type: {0}".format(ctype)) + click.echo(create_table(('Instance', 'Status', 'Connection'), instances)) + click.echo("") + + [ display_deployed(*row) for row in grouped_rows ] + else: + def format_row(comp, instances): + return comp["name"], comp["version"], comp["component_type"], \ + util.format_description(comp["description"]), \ + util.get_status_string(comp), comp["modified"], len(instances) + + rows = [ format_row(*grouped_row) for grouped_row in grouped_rows ] + click.echo(create_table(('Name', 'Version', 'Type', 'Description', + 'Status', 'Modified', '#Deployed'), rows)) + click.echo("\nUse the \"--deployed\" option to see more details on deployments") + + +@component.command() +@click.argument('component', metavar="name:version") +@click.pass_obj +def show(obj, component): + '''Provides more information about a COMPONENT''' + cname, cver = parse_input(component) + catalog = obj['catalog'] + comp_spec = catalog.get_component_spec(cname, cver) + + click.echo(util.format_json(comp_spec)) + + +_help_dmaap_file = """ +Path to a file that contains a json of dmaap client information. The structure of the json is expected to be: + + { + : {..client object 1..}, + : {..client object 2..}, + ... + } + +Where "client object" can be for message or data router. The "config_key" matches the value of specified in the message router "streams" in the component specification. + +Please refer to the documentation for examples of "client object". +""" + +def _parse_dmaap_file(dmaap_file): + try: + with open(dmaap_file, 'r+') as f: + dmaap_map = json.load(f) + dmaap.validate_dmaap_map_schema(dmaap_map) + return dmaap.apply_defaults_dmaap_map(dmaap_map) + except Exception as e: + message = "Problems with parsing the dmaap file. Check to make sure that it is a valid json and is in the expected format." + raise DcaeException(message) + + +_help_inputs_file = """ +Path to a file that contains a json that contains values to be used to bind to configuration parameters that have been marked as "sourced_at_deployment". The structure of the json is expected to be: + + { + : value, + : value + } + +The "parameter name" is the value of the "name" property for the given configuration parameter. +""" + +def _parse_inputs_file(inputs_file): + try: + with open(inputs_file, 'r+') as f: + inputs_map = json.load(f) + # TODO: Validation of schema in the future? + return inputs_map + except Exception as e: + message = "Problems with parsing the inputs file. Check to make sure that it is a valid json and is in the expected format." + raise DcaeException(message) + + +_help_policy_file = """ +Path to a file that contains a json of an (update/remove) Policy change. +All "policies" can also be specified. +The structure of the json is expected to be: + +{ +"updated_policies": [{"policyName": "value", "": ""},{"policyName": "value", "": ""}], +"removed_policies": [{"policyName": "value", "": ""},{"policyName": "value", "": ""}], +"policies": [{"policyName": "value", "": ""},{"policyName": "value", "": ""}] +} +""" + +def _parse_policy_file(policy_file): + try: + with open(policy_file, 'r+') as f: + policy_change_file = json.load(f) + policy.validate_against_policy_schema(policy_change_file) + return policy_change_file + except Exception as e: + click.echo(format(e)) + message = "Problems with parsing the Policy file. Check to make sure that it is a valid json and is in the expected format." + raise DcaeException(message) + +@component.command() +@click.option('--external-ip', '-ip', default=None, help='The external IP address of the Docker host. Only used for Docker components.') +@click.option('--additional-user', default=None, help='Additional user to grab instances from.') +@click.option('--attached', is_flag=True, help='(Docker) dcae-cli deploys then attaches to the component when set') +@click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies') +@click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), + help=_help_dmaap_file) +@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), + help=_help_inputs_file) +@click.argument('component') +@click.pass_obj +def run(obj, external_ip, additional_user, attached, force, dmaap_file, component, + inputs_file): + '''Runs latest (or specific) COMPONENT version. You may optionally specify version via COMPONENT:VERSION''' + + click.echo("Running the Component.....") + click.echo("") + + cname, cver = parse_input(component) + user, catalog = obj['config']['user'], obj['catalog'] + + dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {} + inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {} + + try: + run_component(user, cname, cver, catalog, additional_user, attached, force, + dmaap_map, inputs_map, external_ip) + except DiscoveryNoDownstreamComponentError as e: + message = "Either run a compatible downstream component first or run with the --force flag to ignore this error" + raise DcaeException(message) + except inputs.InputsValidationError as e: + click.echo("ERROR: There is a problem. {0}".format(e)) + click.echo("") + message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct" + raise DcaeException(message) + +@component.command() +@click.argument('component') +@click.pass_obj +def undeploy(obj, component): + '''Undeploy latest (or specific) COMPONENT version. You may optionally specify version via COMPONENT:VERSION''' + cname, cver = parse_input(component) + user, catalog = obj['config']['user'], obj['catalog'] + undeploy_component(user, cname, cver, catalog) + + +@component.command() +@click.argument('specification', type=click.Path(resolve_path=True, exists=True)) +@click.option('--additional-user', default=None, help='Additional user to grab instances from.') +@click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies') +@click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), + help=_help_dmaap_file) +@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), + help=_help_inputs_file) +@click.pass_obj +def dev(obj, specification, additional_user, force, dmaap_file, inputs_file): + '''Set up component in development for discovery, use for local development''' + user, catalog = obj['config']['user'], obj['catalog'] + + dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {} + inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {} + + with open(specification, 'r+') as f: + spec = json.loads(f.read()) + try: + dev_component(user, catalog, spec, additional_user, force, dmaap_map, + inputs_map) + except DiscoveryNoDownstreamComponentError as e: + message = "Either run a compatible downstream component first or run with the --force flag to ignore this error" + raise DcaeException(message) + except inputs.InputsValidationError as e: + click.echo("ERROR: There is a problem. {0}".format(e)) + click.echo("") + message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct" + raise DcaeException(message) + + +@component.command() +@click.argument('component') +@click.pass_obj +def publish(obj, component): + """Pushes a COMPONENT to the public catalog""" + name, version = parse_input(component) + user, catalog = obj['config']['user'], obj['catalog'] + + try: + # Dependent data formats must be published first before publishing + # component. Check that here + unpub_formats = catalog.get_unpublished_formats(name, version) + + if unpub_formats: + click.echo("ERROR: You must publish dependent data formats first:") + click.echo("") + click.echo("\n".join([":".join(uf) for uf in unpub_formats])) + click.echo("") + return + except MissingEntry as e: + raise DcaeException("Component not found") + + if catalog.publish_component(user, name, version): + click.echo("Component has been published") + else: + click.echo("ERROR: Component could not be published") + + +@component.command() +@click.option('--update', is_flag=True, help='Updates a locally added component if it has not already been published') +@click.argument('specification', type=click.Path(resolve_path=True, exists=True)) +@click.pass_obj +def add(obj, update, specification): + """Add Component to local onboarding catalog""" + user, catalog = obj['config']['user'], obj['catalog'] + + spec = load_json(specification) + catalog.add_component(user, spec, update) + + +@component.command() +@click.option('--policy-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), help=_help_policy_file) +@click.argument('component') +@click.pass_obj +def reconfig(obj, policy_file, component): + """Reconfigure COMPONENT for Policy change. + Modify Consul KV pairs for ('updated_policies', 'removed_policies', and 'policies') for Policy change event, + Execute the reconfig script(s) in the Docker container""" + + click.echo("Running Component Reconfiguration.....") + click.echo("") + + # Read and Validate the policy-file + policy_change_file = _parse_policy_file(policy_file) if policy_file else {} + + if not (policy_change_file): + click.echo("ERROR: For component 'reconfig', you must specify a --policy-file") + click.echo("") + return + else: + # The Component Spec contains the Policy 'Reconfig Script Path/ScriptName' + cname, cver = parse_input(component) + catalog = obj['catalog'] + comp_spec = catalog.get_component_spec(cname, cver) + + # Check if component is running and healthy + active_profile = profiles.get_profile() + consul_host = active_profile.consul_host + service_name = os.environ["SERVICE_NAME"] + if dis.is_healthy(consul_host, service_name): + pass + else: + click.echo("ERROR: The component must be running and healthy. It is not.") + click.echo("") + return + + try: + policy_reconfig_path = comp_spec['auxilary']['policy']['script_path'] + except KeyError: + click.echo("ERROR: Policy Reconfig Path (auxilary/policy/script_path) is not specified in the Component Spec") + click.echo("") + return + + kvUpdated = dis.policy_update(policy_change_file, dis.default_consul_host()) + + if kvUpdated: + active_profile = profiles.get_profile() + docker_logins = dis.get_docker_logins() + + command = dis.build_policy_command(policy_reconfig_path, policy_change_file, dis.default_consul_host()) + + # Run the Policy Reconfig script + client = du.get_docker_client(active_profile, docker_logins) + du.reconfigure(client, service_name, command) + else: + click.echo("ERROR: There was a problem updating the policies in Consul") + click.echo("") + return + + click.echo("") + click.echo("The End of Component Reconfiguration") diff --git a/mod/onboardingapi/dcae_cli/commands/data_format/__init__.py b/mod/onboardingapi/dcae_cli/commands/data_format/__init__.py new file mode 100644 index 0000000..b025f61 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/data_format/__init__.py @@ -0,0 +1,25 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +""" +Provides the data format group +""" +from .commands import data_format diff --git a/mod/onboardingapi/dcae_cli/commands/data_format/commands.py b/mod/onboardingapi/dcae_cli/commands/data_format/commands.py new file mode 100644 index 0000000..b952336 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/data_format/commands.py @@ -0,0 +1,163 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +""" +Provides data format commands +""" +import json + +import click + +import genson + +import sys + +import os + +from jsonschema import Draft4Validator + +from dcae_cli.util import load_json +from dcae_cli.util.logger import get_logger + +from dcae_cli.commands import util +from dcae_cli.commands.util import create_table, parse_input + +from dcae_cli.catalog.exc import MissingEntry +from dcae_cli.catalog.exc import DcaeException + + +logger = get_logger('DataFormatCommand') + + +@click.group() +def data_format(): + pass + + +@data_format.command() +@click.option('--update', is_flag=True, help='Updates a locally added data format if it has not already been published') +@click.argument('specification', type=click.Path(resolve_path=True, exists=True)) +@click.pass_obj +def add(obj, update, specification): + '''Tracks a Format file Specification locally, but does not push to the catalog''' + spec = load_json(specification) + user, catalog = obj['config']['user'], obj['catalog'] + catalog.add_format(spec, user, update) + + +@data_format.command(name='list') +@click.option('--latest', is_flag=True, help='Only list the latest version of data formats') +@click.pass_obj +def list_format(obj, latest): + """Lists all your Data Formats""" + user, catalog = obj['config']['user'], obj['catalog'] + dfs = catalog.list_formats(latest, user=user) + + def format_record(df): + return (df["name"], df["version"], + util.format_description(df["description"]), + util.get_status_string(df), df["modified"]) + + dfs = [ format_record(df) for df in dfs ] + + click.echo("") + click.echo("Data formats for {0}".format(user)) + click.echo(create_table(('Name', 'Version', 'Description', 'Status', 'Modified'), dfs)) + + +@data_format.command() +@click.argument('data-format', metavar="name:version") +@click.pass_obj +def show(obj, data_format): + '''Provides more information about a Data Format''' + name, ver = parse_input(data_format) + spec = obj['catalog'].get_format_spec(name, ver) + + click.echo(util.format_json(spec)) + + +@data_format.command() +@click.argument('data-format') +@click.pass_obj +def publish(obj, data_format): + """Publish Format to make publicly available""" + name, version = parse_input(data_format) + user, catalog = obj['config']['user'], obj['catalog'] + + if catalog.publish_format(user, name, version): + click.echo("Data format has been published") + else: + click.echo("ERROR: Data format could not be published") + +@data_format.command() +@click.option('--keywords', is_flag=True, help='Adds a template of possible descriptive keywords', default=False) +@click.argument('name_version', metavar="name:version", required = True) +@click.argument('file-or-dir-path', type=click.Path(resolve_path=True, exists=True, dir_okay=True, file_okay=True, readable=True), metavar="file-or-dir-path") +@click.pass_obj +def generate(obj, name_version, file_or_dir_path, keywords): + '''Create schema from a file or directory examples''' + name, version = parse_input(name_version) + if version == None: + version = "" + schema = genson.Schema() + if os.path.isfile(file_or_dir_path): + addfile(file_or_dir_path, schema) + else: + foundJSON = False + for root, dirs, files in os.walk(file_or_dir_path): + for filename in files: + fullfilename = os.path.join(file_or_dir_path, filename) + addfile(fullfilename,schema) + foundJSON = True + if foundJSON == False: + raise DcaeException('No JSON files found in ' + file_or_dir_path) + + json_obj = json.loads(schema.to_json()) + json_obj['$schema'] = "http://json-schema.org/draft-04/schema#" + jschema = json.dumps(json_obj) + jschema = jschema.replace('"required":', '"additionalproperties": true, "required":') + jschema = jschema.replace('"type":', ' "description": "", "type":') + + if (keywords): + jschema = jschema.replace('"type": "string"', ' "maxLength": 0, "minLength": 0, "pattern": "", "type": "string"') + jschema = jschema.replace('"type": "integer"', ' "maximum": 0, "mininimum": 0, "multipleOf": 0, "type": "integer"') + jschema = jschema.replace('"type": "array"', ' "maxItems": 0, "minItems": 0, "uniqueItems": "false", "type": "array"') + + jschema = '{ "self": { "name": "' + name + '", "version": "' + version + '", "description": ""} , "dataformatversion": "1.0.0", "jsonschema": ' + jschema + '}' + #Draft4Validator.check_schema(json.loads(jschema)) + try: + print(json.dumps(json.loads(jschema), sort_keys=True, indent=4 )) + except ValueError: + raise DcaeException('Problem with JSON generation') + +def addfile(filename, schema): + try: + fileadd = open(filename, "r") + except IOError: + raise DcaeException('Cannot open' + filename) + try: + json_object = json.loads(fileadd.read()) + schema.add_object(json_object) + except ValueError: + raise DcaeException('Bad JSON file: ' + filename) + finally: + fileadd.close() + diff --git a/mod/onboardingapi/dcae_cli/commands/profiles/__init__.py b/mod/onboardingapi/dcae_cli/commands/profiles/__init__.py new file mode 100644 index 0000000..0d71c1b --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/profiles/__init__.py @@ -0,0 +1,25 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +""" +Provides the profiles group +""" +from .commands import profiles diff --git a/mod/onboardingapi/dcae_cli/commands/profiles/commands.py b/mod/onboardingapi/dcae_cli/commands/profiles/commands.py new file mode 100644 index 0000000..df34b5c --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/profiles/commands.py @@ -0,0 +1,87 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +""" +Provides profiles commands +""" +import json + +import click + +from dcae_cli.util.exc import DcaeException +from dcae_cli.util.profiles import (get_profiles, activate_profile, get_active_name, update_profile, + delete_profile, create_profile) + + +@click.group() +def profiles(): + pass + + +@profiles.command() +@click.argument('name') +def activate(name): + '''Sets profile (name) as the active profile''' + activate_profile(name) + + +@profiles.command(name='list') +def list_profiles(): + '''Lists available profiles''' + profiles = get_profiles(include_active=False) + active = get_active_name() + names = sorted(profiles.keys()) + outputs = ("{} {}".format(' ' if not name == active else '* ', name) for name in names) + click.echo('\n'.join(outputs)) + + +@profiles.command() +@click.argument('name') +def show(name): + '''Provides more information about a Profile''' + profiles = get_profiles() + try: + click.echo(json.dumps(profiles[name], sort_keys=True, indent=4)) + except KeyError as e: + raise DcaeException("Profile '{}' does not exist.".format(e)) + + +@profiles.command() +@click.argument('name', type=click.STRING) +def create(name): + '''Creates new profile (name), with defaults''' + create_profile(name) + + +@profiles.command(name='set') +@click.argument('name') +@click.argument('key') +@click.argument('value') +def update(name, key, value): + '''Updates profile (name) for specific Key/Value''' + update_profile(name, **{key: value}) + + +@profiles.command() +@click.argument('name') +def delete(name): + '''Deletes profile (name)''' + delete_profile(name) diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/format.json new file mode 100644 index 0000000..8456a30 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/format.json @@ -0,0 +1,10 @@ +{ + "self": { + "name": "std.empty", + "version": "1.0.6", + "description": "Represents an empty message with no content" + }, + "dataformatversion": "1.0.0", + "jsonschema": { + } +} diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json new file mode 100644 index 0000000..9642a6e --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json @@ -0,0 +1,77 @@ +{ + "self":{ + "name":"cdap.helloworld.mock.catalog.testing.endnode", + "version":"0.7.1", + "description":"cdap test component", + "component_type":"cdap" + }, + "streams":{ + "subscribes": [], + "publishes": [{ + "format": "std.empty", + "version": "1.0.6", + "config_key": "stream_publish_example", + "type": "http" + }] + }, + "services":{ + "calls": [], + "provides":[ + { + "request":{ + "format":"std.empty", + "version":"1.0.6" + }, + "response":{ + "format":"std.empty", + "version":"1.0.6" + }, + "service_name":"Greeting", + "service_endpoint":"greet", + "verb":"GET" + } + ] + }, + "parameters": { + "app_config" : [ + {"name" : "some_param", + "description" : "some desc", + "value" : "some_value", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} + ], + "app_preferences" : [ + {"name" : "some_param2", + "description" : "some desc2", + "value" : "some_value2", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} + ], + "program_preferences" : [{"program_type" : "flows", + "program_id" : "WhoFlow", + "program_pref" : [{"name" : "some_param3", + "description" : "some desc3", + "value" : "some_value3", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false}]}] + }, + "auxilary": { + "streamname":"who", + "artifact_name" : "HelloWorld", + "artifact_version" : "3.4.3", + "programs" : [ + {"program_type" : "flows", "program_id" : "WhoFlow"}, + {"program_type" : "services", "program_id" : "Greeting"} + ], + "namespace" : "hw" + }, + "artifacts": [ + { + "uri": "http://make-me-valid/jar_files/HelloWorld-3.4.3.jar", + "type": "jar" + } + ] +} diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json new file mode 100644 index 0000000..83b5c28 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json @@ -0,0 +1,78 @@ +{ + "self":{ + "name":"cdap.helloworld.mock.catalog.testing.startnode", + "version":"0.7.1", + "description":"cdap test component", + "component_type":"cdap" + }, + "streams":{ + "subscribes": [{ + "format": "std.empty", + "version": "1.0.6", + "route": "/unsure_if_needed_for_cdap", + "type": "http" + }], + "publishes": [] + }, + "services":{ + "calls": [ + { + "config_key": "service_call_example", + "verb": "GET", + "request": { + "format": "std.empty", + "version": "1.0.6" + }, + "response": { + "format": "std.empty", + "version": "1.0.6" + } + } + ], + "provides":[] + }, +"parameters": { + "app_config" : [ + {"name" : "some_param", + "description" : "some desc", + "value" : "some_value", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} + ], + "app_preferences" : [ + {"name" : "some_param2", + "description" : "some desc2", + "value" : "some_value2", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} + ], + "program_preferences" : [{"program_type" : "flows", + "program_id" : "WhoFlow", + "program_pref" : [{"name" : "some_param3", + "description" : "some desc3", + "value" : "some_value3", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false} + ]}] + }, + "auxilary": { + "streamname":"who", + "artifact_name" : "HelloWorld", + "artifact_version" : "3.4.3", + "programs" : [ + {"program_type" : "flows", "program_id" : "WhoFlow"}, + {"program_type" : "services", "program_id" : "Greeting"} + ], + "namespace" : "hw" + }, + "artifacts": [ + { + "uri": "http://make-me-valid/jar_files/HelloWorld-3.4.3.jar", + "type": "jar" + } + ] +} + diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json new file mode 100644 index 0000000..5b86d9c --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json @@ -0,0 +1,47 @@ +{ + "self": { + "version": "1.0.0", + "name": "std.vnf.kpi_collector", + "description": "Continuously publishes VNF KPIs", + "component_type": "docker" + }, + "streams": { + "subscribes": [], + "publishes": [ + { + "format": "std.vnf.kpi", + "version": "1.0.0", + "config_key": "kpi_pub", + "type": "http" + } + ] + }, + "services": { + "calls": [], + "provides": [] + }, + "parameters": [ + { + "name": "sleep_sec", + "value": 0.75, + "description": "Number of seconds to sleep between publishes", + "designer_editable": false, + "sourced_at_deployment": false, + "policy_editable": false + } + ], + "auxilary": { + "healthcheck": { + "type": "http", + "endpoint": "/health", + "interval": "15s", + "timeout": "1s" + } + }, + "artifacts": [ + { + "uri": "asimov-anomaly-collector", + "type": "docker image" + } + ] +} diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/vnf-kpi.format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/vnf-kpi.format.json new file mode 100644 index 0000000..6fba1a1 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/vnf-kpi.format.json @@ -0,0 +1,13 @@ +{ + "self": { + "name": "std.vnf.kpi", + "version": "1.0.0", + "description": "Represents a KPI of a VNF at particular instance of time." + }, + "dataformatversion": "1.0.0", + "reference": { + "name": "Common Event Format", + "format": "JSON", + "version": "25.0.0" + } +} diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json new file mode 100644 index 0000000..3e2d142 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json @@ -0,0 +1,52 @@ +{ + "self": { + "version": "1.0.0", + "name": "asimov.anomaly_classifier", + "description": "Classifies VNF KPIs as anommalous or not", + "component_type": "docker" + }, + "streams": { + "subscribes": [{ + "format": "std.vnf.kpi", + "version": "1.0.0", + "route": "/data", + "type": "http" + }], + "publishes": [ + { + "format": "asimov.std.integerClassification", + "version": "1.0.0", + "config_key": "pred", + "type": "http" + } + ] + }, + "services": { + "calls": [], + "provides": [] + }, + "parameters": [ + { + "name": "threshold", + "value": 0.75, + "description": "Probability threshold to exceed to be anomalous", + "designer_editable" : false, + "sourced_at_deployment" : false, + "policy_editable" : false + } + ], + "auxilary": { + "healthcheck": { + "type": "http", + "endpoint": "/health", + "interval": "15s", + "timeout": "1s" + } + }, + "artifacts": [ + { + "uri": "asimov-anomaly-model", + "type": "docker image" + } + ] +} diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/badjson b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/badjson new file mode 100644 index 0000000..e69de29 diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json new file mode 100755 index 0000000..7db1e06 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json @@ -0,0 +1,3 @@ +{ + "foobar": "test 1" +} diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json new file mode 100755 index 0000000..75a5fa4 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json @@ -0,0 +1,4 @@ +{ + "foobar2": "test 1" +} + diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/int-class.format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/int-class.format.json new file mode 100644 index 0000000..7d3dedf --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/int-class.format.json @@ -0,0 +1,18 @@ +{ + "self": { + "name": "asimov.std.integerClassification", + "version": "1.0.0", + "description": "Represents a single classification from a machine learning model" + }, + "dataformatversion": "1.0.0", + "jsonschema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "classification": { + "type": "string" + } + }, + "additionalProperties": false + } +} diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/empty.format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/empty.format.json new file mode 100644 index 0000000..4ff7d3a --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/empty.format.json @@ -0,0 +1,10 @@ +{ + "self": { + "name": "std.empty", + "version": "1.0.0", + "description": "Represents an empty message with no content" + }, + "dataformatversion": "1.0.0", + "jsonschema": { + } +} diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/line-viz.comp.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/line-viz.comp.json new file mode 100644 index 0000000..d3b9e23 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/line-viz.comp.json @@ -0,0 +1,51 @@ +{ + "self": { + "version": "1.0.0", + "name": "asimov.viz.line_plot", + "description": "Plots class probabilities as a line plot in real-time", + "component_type": "docker" + }, + "streams": { + "subscribes": [ + { + "format": "asimov.std.integerClassification", + "version": "1.0.0", + "route": "/prediction", + "type": "http" + } + ], + "publishes": [] + }, + "services": { + "calls": [], + "provides": [ + { + "route": "/viz", + "verb": "GET", + "request": { + "format": "std.empty", + "version": "1.0.0" + }, + "response": { + "format": "std.web.url", + "version": "1.0.0" + } + } + ] + }, + "parameters": [], + "auxilary": { + "healthcheck": { + "type": "http", + "endpoint": "/health", + "interval": "15s", + "timeout": "1s" + } + }, + "artifacts": [ + { + "uri": "asimov-anomaly-viz", + "type": "docker image" + } + ] +} diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/web-url.format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/web-url.format.json new file mode 100644 index 0000000..3e823b9 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/web-url.format.json @@ -0,0 +1,19 @@ +{ + "self": { + "name": "std.web.url", + "version": "1.0.0", + "description": "Represents a web URL" + }, + "dataformatversion": "1.0.0", + "jsonschema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "URL": { + "type": "string" + } + }, + "additionalProperties": false + } +} + diff --git a/mod/onboardingapi/dcae_cli/commands/tests/test_component_cmd.py b/mod/onboardingapi/dcae_cli/commands/tests/test_component_cmd.py new file mode 100644 index 0000000..2bba4cf --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/test_component_cmd.py @@ -0,0 +1,149 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +''' +Tests component CLI commands +''' +import os +import json +from click.testing import CliRunner +import time +import pytest + +from dcae_cli.cli import cli +from dcae_cli.catalog import MockCatalog + +TEST_DIR = os.path.dirname(__file__) + + +def _get_spec(path): + with open(path) as file: + return json.load(file) + + +def test_comp_docker(mock_cli_config, mock_db_url, obj=None): + + obj = {'catalog': MockCatalog(purge_existing=True, db_name='dcae_cli.test.db', + enforce_image=False, db_url=mock_db_url), + 'config': {'user': 'test-user'}} + + df_kpi = os.path.join(TEST_DIR, 'mocked_components', 'collector', 'vnf-kpi.format.json') + comp_coll = os.path.join(TEST_DIR, 'mocked_components', 'collector', 'kpi-collector.comp.json') + + df_cls = os.path.join(TEST_DIR, 'mocked_components', 'model', 'int-class.format.json') + comp_model = os.path.join(TEST_DIR, 'mocked_components', 'model', 'anomaly-model.comp.json') + + df_empty = os.path.join(TEST_DIR, 'mocked_components', 'viz', 'empty.format.json') + df_url = os.path.join(TEST_DIR, 'mocked_components', 'viz', 'web-url.format.json') + comp_viz = os.path.join(TEST_DIR, 'mocked_components', 'viz', 'line-viz.comp.json') + + runner = CliRunner() + + + # add the collector + cmd = "data_format add {:}".format(df_kpi).split() + assert runner.invoke(cli, cmd, obj=obj).exit_code == 0 + + cmd = "component add {:}".format(comp_coll).split() + assert runner.invoke(cli, cmd, obj=obj).exit_code == 0 + + + # add the model + cmd = "data_format add {:}".format(df_cls).split() + assert runner.invoke(cli, cmd, obj=obj).exit_code == 0 + + cmd = "component add {:}".format(comp_model).split() + assert runner.invoke(cli, cmd, obj=obj).exit_code == 0 + + + # add the viz + cmd = "data_format add {:}".format(df_empty).split() + assert runner.invoke(cli, cmd, obj=obj).exit_code == 0 + + cmd = "data_format add {:}".format(df_url).split() + assert runner.invoke(cli, cmd, obj=obj).exit_code == 0 + + cmd = "component add {:}".format(comp_viz).split() + assert runner.invoke(cli, cmd, obj=obj).exit_code == 0 + + + # light test of component list + df_cls_spec = _get_spec(df_cls) + df_cls_name, df_cls_ver = df_cls_spec['self']['name'], df_cls_spec['self']['version'] + comp_model_spec = _get_spec(comp_model) + comp_model_name = comp_model_spec['self']['name'] + + cmd = "component list -pub {:}".format(df_cls_name).split() + #assert comp_model_name in runner.invoke(cli, cmd, obj=obj).output + + cmd = "component list -pub {:}:{:}".format(df_cls_name, df_cls_ver).split() + #assert comp_model_name in runner.invoke(cli, cmd, obj=obj).output + + + # light test of component info + cmd = "component show {:}".format(comp_model_name).split() + spec_str = runner.invoke(cli, cmd, obj=obj).output + assert comp_model_spec == json.loads(spec_str) + + +@pytest.mark.skip(reason="This is not a pure unit test. Need a way to setup dependencies and trigger in the appropriate stages of testing.") +def test_comp_cdap(obj=None): + """ + This is not a unit test. It is bigger than that. It Does a full "workflow" test: + 1) adds a data format + 2) adds a cdap component + 3) runs a cdap component using our "Rework" broker + 4) undeploys the cdap component using our "Rework" broker + + NOTE: TODO: Mocking out the broker would be an improvement over this, probably. This is impure. Mocking the broker owuld be a huge undertaking, though. + """ + + obj = {'catalog': MockCatalog(purge_existing=True, db_name='dcae_cli.test.db'), + 'config': {'user': 'test-user'}} + runner = CliRunner() + + #add the data format + df = os.path.join(TEST_DIR, 'mocked_components', 'cdap', 'format.json') + cmd = "data_format add {:}".format(df).split() + assert runner.invoke(cli, cmd, obj=obj).exit_code == 0 + + #add the CDAP components + # TODO: Need to update the host + jar = 'http://make-me-valid/HelloWorld-3.4.3.jar' + + comp_cdap_start = os.path.join(TEST_DIR, 'mocked_components', 'cdap', 'spec_start.json') + cmd = "component add {0}".format(comp_cdap_start).split() + print(cmd) + result = runner.invoke(cli, cmd, obj=obj) + print(result.output) + assert result.exit_code == 0 + + comp_cdap_end = os.path.join(TEST_DIR, 'mocked_components', 'cdap', 'spec_end.json') + cmd = "component add {0}".format(comp_cdap_end).split() + print(cmd) + result = runner.invoke(cli, cmd, obj=obj) + print(result.output) + assert result.exit_code == 0 + +if __name__ == '__main__': + '''Test area''' + #pytest.main([__file__, ]) + test_comp_cdap() diff --git a/mod/onboardingapi/dcae_cli/commands/tests/test_data_format_cmd.py b/mod/onboardingapi/dcae_cli/commands/tests/test_data_format_cmd.py new file mode 100644 index 0000000..a291a74 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/test_data_format_cmd.py @@ -0,0 +1,122 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +''' +Tests data_format CLI commands +''' +import os +import json + +import pytest +from click.testing import CliRunner + +from dcae_cli.cli import cli +from dcae_cli.catalog import MockCatalog + + +TEST_DIR = os.path.dirname(__file__) + + +def _get_spec(path): + with open(path) as file: + return json.load(file) + + +def test_basic(mock_cli_config, mock_db_url, tmpdir): + obj = {'catalog': MockCatalog(purge_existing=True, db_name='dcae_cli.test.db', + enforce_image=False, db_url=mock_db_url), + 'config': {'user': 'test-user'}} + + runner = CliRunner() + spec_file = os.path.join(TEST_DIR, 'mocked_components', 'model', 'int-class.format.json') + cmd = "data_format add {:}".format(spec_file).split() + + # succeed the first time + result = runner.invoke(cli, cmd, obj=obj) + + assert result.exit_code == 0 + + # adding a duplicate is an error + result = runner.invoke(cli, cmd, obj=obj) + assert result.exit_code == 1 + assert 'exists' in result.output.lower() + + # allow updates + cmd = "data_format add --update {:}".format(spec_file).split() + result = runner.invoke(cli, cmd, obj=obj) + assert result.exit_code == 0 + + + # light test of list format command + cmd = 'data_format list'.split() + df_spec = _get_spec(spec_file) + df_name = df_spec['self']['name'] + assert df_name in runner.invoke(cli, cmd, obj=obj).output + + + # light test of component info + cmd = "data_format show {:}".format(df_name).split() + spec_str = runner.invoke(cli, cmd, obj=obj).output + assert df_spec == json.loads(spec_str) + + # test of generate + bad_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'baddir') + cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_dir).split() + err_str = runner.invoke(cli, cmd, obj=obj).output + assert "does not exist" in err_str + + empty_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'emptydir') + try: + os.stat(empty_dir) + except: + os.mkdir(empty_dir) + cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(empty_dir).split() + err_str = runner.invoke(cli, cmd, obj=obj).output + assert "No JSON files found" in err_str + + bad_json = os.path.join(TEST_DIR, 'mocked_components', 'model', 'badjson') + cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_json).split() + err_str = runner.invoke(cli, cmd, obj=obj).output + assert "Bad JSON file" in err_str + + generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir') + cmd = "data_format generate --keywords name:1.0.2 {:} ".format(generate_dir).split() + actual = json.loads(runner.invoke(cli, cmd, obj=obj).output) + expected = json.loads('{\n "dataformatversion": "1.0.0", \n "jsonschema": {\n "$schema": "http://json-schema.org/draft-04/schema#", \n "description": "", \n "properties": {\n "foobar": {\n "description": "", \n "maxLength": 0, \n "minLength": 0, \n "pattern": "", \n "type": "string"\n }, \n "foobar2": {\n "description": "", \n "maxLength": 0, \n "minLength": 0, \n "pattern": "", \n "type": "string"\n }\n }, \n "type": "object"\n }, \n "self": {\n "description": "", \n "name": "name", \n "version": "1.0.2"\n }\n}\n') + assert actual == expected + + generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir') + cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split() + actual = json.loads(runner.invoke(cli, cmd, obj=obj).output) + expected = json.loads('{\n "dataformatversion": "1.0.0", \n "jsonschema": {\n "$schema": "http://json-schema.org/draft-04/schema#", \n "description": "", \n "properties": {\n "foobar": {\n "description": "", \n "type": "string"\n }, \n "foobar2": {\n "description": "", \n "type": "string"\n }\n }, \n "type": "object"\n }, \n "self": {\n "description": "", \n "name": "name", \n "version": "1.0.2"\n }\n}\n' + ) + assert actual == expected + + generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir', 'ex1.json') + cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split() + actual = json.loads(runner.invoke(cli, cmd, obj=obj).output) + expected = json.loads('{\n "dataformatversion": "1.0.0", \n "jsonschema": {\n "$schema": "http://json-schema.org/draft-04/schema#", \n "additionalproperties": true, \n "description": "", \n "properties": {\n "foobar": {\n "description": "", \n "type": "string"\n }\n }, \n "required": [\n "foobar"\n ], \n "type": "object"\n }, \n "self": {\n "description": "", \n "name": "name", \n "version": "1.0.2"\n }\n}\n') + assert actual == expected + + +if __name__ == '__main__': + '''Test area''' + pytest.main([__file__, ]) diff --git a/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py b/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py new file mode 100644 index 0000000..be89722 --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py @@ -0,0 +1,84 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +''' +Tests profiles CLI commands +''' +import json + +import pytest +import click +from click.testing import CliRunner + +from dcae_cli import util +from dcae_cli.cli import cli +from dcae_cli.util import profiles +from dcae_cli.util import config + + +def test_basic(monkeypatch, tmpdir, mock_db_url): + + runner = CliRunner() + + # Setup config + test_db_url = mock_db_url + config_dict = { "user": "ninny", "active_profile": "fake-solutioning", + "db_url": test_db_url, "cli_version": "2.0.0" } + config_file = tmpdir.join("config.json") + config_file.write(json.dumps(config_dict)) + + # Setup profile + profile_dict = { "fake-solutioning": { "cdap_broker": "cdap_broker", + "config_binding_service": "config_binding_service", + "consul_host": "realsolcnsl00.dcae.solutioning.com", + "docker_host": "realsoldokr00.dcae.solutioning.com:2376" }} + profile_file = tmpdir.join("profiles.json") + profile_file.write(json.dumps(profile_dict)) + + monkeypatch.setattr(click, "get_app_dir", lambda app: str(tmpdir.realpath())) + + cmd = 'profiles show fake-solutioning'.split() + result = runner.invoke(cli, cmd) + assert result.output == "{}\n".format(json.dumps(profile_dict["fake-solutioning"], + sort_keys=True, indent=4)) + + cmd = 'profiles list'.split() + result = runner.invoke(cli, cmd) + assert result.output == '* fake-solutioning\n' + + cmd = 'profiles create foo'.split() + result = runner.invoke(cli, cmd) + + cmd = 'profiles list'.split() + result = runner.invoke(cli, cmd) + assert result.output == '* fake-solutioning\n foo\n' + + cmd = 'profiles activate foo'.split() + result = runner.invoke(cli, cmd) + + cmd = 'profiles list'.split() + result = runner.invoke(cli, cmd) + assert result.output == ' fake-solutioning\n* foo\n' + + +if __name__ == '__main__': + '''Test area''' + pytest.main([__file__, ]) diff --git a/mod/onboardingapi/dcae_cli/commands/util.py b/mod/onboardingapi/dcae_cli/commands/util.py new file mode 100644 index 0000000..f9527fa --- /dev/null +++ b/mod/onboardingapi/dcae_cli/commands/util.py @@ -0,0 +1,111 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- coding: utf-8 -*- +""" +Provides utilities for commands +""" +import json +import textwrap +from terminaltables import AsciiTable + +from dcae_cli.util import DcaeException + + +def parse_input(input_): + '''Returns (name, version) tuple parsed from name:version''' + arg = input_.split(':') + if len(arg) == 1: + cname, cver = arg[0], None + elif len(arg) == 2: + cname, cver = arg + cver = None if not cver else cver + else: + raise DcaeException("Input '{:}' must be NAME or NAME:VERSION".format(input_)) + return cname, cver + + +def parse_input_pair(req, resp): + '''Returns a tuple output of `parse_input` for convenience''' + return parse_input(req), parse_input(resp) + + +def create_table(header, entries): + '''Returns an ASCII table string''' + data = [header, ] + if entries: + data.extend(entries) + else: + data.append(['']*len(header)) + return AsciiTable(data).table + + +# Utility methods used to format records for displaying + +def get_status_string(record): + """Get the status label given a record of either data format or component""" + if "when_revoked" not in record or \ + "when_published" not in record or \ + "when_added" not in record: + return None + + if record["when_revoked"] is not None: + return "revoked" + elif record["when_published"] is not None: + return "published" + else: + return "unpublished" + +def get_status_string_camel(record): + """Get the status label given a record of either data format or component, in camelCase""" + if "whenRevoked" not in record or \ + "whenPublished" not in record or \ + "whenAdded" not in record: + return None + + if record["whenRevoked"] is not None: + return "revoked" + elif record["whenPublished"] is not None: + return "published" + else: + return "unpublished" + +def format_description(description, line_width=50, num_lines=3): + """Formats the description field + + Description field can be long. This function line wraps to a specified number + of lines. The last line trails with ".." if the text still overflows to + signal that there is more. + """ + if not description: + return '' + lines = textwrap.wrap(description) + lines = lines[:num_lines] + last_line = lines.pop() + + if len(last_line) > line_width and line_width > 2: + last_line = "{0}..".format(last_line[:-2]) + + lines.append(last_line) + return "\n".join(lines) + + +def format_json(some_json): + return json.dumps(some_json, sort_keys=True, indent=4) -- cgit 1.2.3-korg