Copy dcae-cli->onboardingapi, copy component specs

Issue-ID: DCAEGEN2-1860
Change-Id: I4805398c76479fad51cbdb74470ccc8f706ce9dc
Signed-off-by: Michael Hwang <mhwang@research.att.com>

26 files changed, 1724 insertions, 0 deletions

diff --git a/mod/onboardingapi/dcae_cli/commands/__init__.py b/mod/onboardingapi/dcae_cli/commands/__init__.py
new file mode 100644
index 0000000..ceddbb9
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/__init__.py
@@ -0,0 +1,21 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
diff --git a/mod/onboardingapi/dcae_cli/commands/catalog/__init__.py b/mod/onboardingapi/dcae_cli/commands/catalog/__init__.py
new file mode 100644
index 0000000..93f14ee
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/catalog/__init__.py
@@ -0,0 +1,21 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+from .commands import catalog
diff --git a/mod/onboardingapi/dcae_cli/commands/catalog/commands.py b/mod/onboardingapi/dcae_cli/commands/catalog/commands.py
new file mode 100644
index 0000000..dc6b27a
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/catalog/commands.py
@@ -0,0 +1,115 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+"""
+Queries onboarding catalog
+"""
+import click
+
+from dcae_cli.commands import util
+
+
+@click.group()
+def catalog():
+    pass
+
+
+@catalog.command(name="list")
+@click.option("--expanded", is_flag=True, default=False, help="Display the expanded view - show all versions and all statuses")
+#TODO: @click.argument('query')
+@click.pass_obj
+def action_list(obj, expanded):
+    """Lists resources in the onboarding catalog"""
+    # Query both components and data formats. Display both sets.
+
+    user, catalog = obj['config']['user'], obj['catalog']
+
+    only_latest = not expanded
+    only_published = not expanded
+
+    # TODO: Probably want to implement pagination
+    comps = catalog.list_components(latest=only_latest, only_published=only_published)
+    dfs = catalog.list_formats(latest=only_latest, only_published=only_published)
+
+    def format_record_component(obj):
+        when_published = obj["when_published"].date() \
+                if obj["when_published"] else ""
+
+        return (obj["name"], obj["version"], obj["component_type"],
+                util.format_description(obj["description"]), obj["owner"],
+                util.get_status_string(obj), when_published)
+
+    comps = [ format_record_component(comp) for comp in comps ]
+
+    click.echo("")
+    click.echo("Components:")
+    click.echo(util.create_table(('Name', 'Version', 'Type', 'Description', 'Owner', 'Status',
+        'Published'), comps))
+
+    def format_record_format(obj):
+        when_published = obj["when_published"].date() \
+                if obj["when_published"] else ""
+
+        return (obj["name"], obj["version"],
+                util.format_description(obj["description"]), obj["owner"],
+                util.get_status_string(obj), when_published)
+
+    dfs = [ format_record_format(df) for df in dfs ]
+
+    click.echo("")
+    click.echo("Data formats:")
+    click.echo(util.create_table(('Name', 'Version', 'Description', 'Owner', 'Status',
+        'Published'), dfs))
+
+
+@catalog.command(name="show")
+@click.argument("resource", metavar="name:version")
+@click.pass_obj
+def action_show(obj, resource):
+    """Provides more information about a resource"""
+    # Query both components and data formats. Display both sets.
+    name, ver = util.parse_input(resource)
+    catalog = obj['catalog']
+    spec = None
+
+    try:
+        spec = catalog.get_component_spec(name, ver)
+
+        click.echo("")
+        click.echo("Component specification")
+        click.echo("-----------------------")
+        click.echo(util.format_json(spec))
+        click.echo("")
+    except:
+        pass
+
+    try:
+        spec = obj['catalog'].get_format_spec(name, ver)
+
+        click.echo("")
+        click.echo("Data format")
+        click.echo("-----------")
+        click.echo(util.format_json(spec))
+        click.echo("")
+    except:
+        pass
+
+    if not spec:
+        click.echo("No matching component nor data format found")
diff --git a/mod/onboardingapi/dcae_cli/commands/component/__init__.py b/mod/onboardingapi/dcae_cli/commands/component/__init__.py
new file mode 100644
index 0000000..b1f4a8f
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/component/__init__.py
@@ -0,0 +1,25 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+"""
+Provides the component group
+"""
+from .commands import component
diff --git a/mod/onboardingapi/dcae_cli/commands/component/commands.py b/mod/onboardingapi/dcae_cli/commands/component/commands.py
new file mode 100644
index 0000000..b2483d1
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/component/commands.py
@@ -0,0 +1,394 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+"""
+Provides component commands
+"""
+import json
+from pprint import pformat
+
+import click
+import os
+
+from discovery_client import resolve_name
+
+from dcae_cli.util import profiles, load_json, dmaap, inputs, policy
+from dcae_cli.util.run import run_component, dev_component
+from dcae_cli.util import discovery as dis
+from dcae_cli.util import docker_util as du
+from dcae_cli.util.discovery import DiscoveryNoDownstreamComponentError
+from dcae_cli.util.undeploy import undeploy_component
+from dcae_cli.util.exc import DcaeException
+
+from dcae_cli.commands import util
+from dcae_cli.commands.util import parse_input, parse_input_pair, create_table
+
+from dcae_cli.catalog.exc import MissingEntry
+
+
+@click.group()
+def component():
+    pass
+
+
+@component.command(name='list')
+@click.option('--latest', is_flag=True, default=True, help='Only list the latest version of components which match the filter criteria')
+@click.option('--subscribes', '-sub', multiple=True, help='Only list components which subscribe to FORMAT')
+@click.option('--publishes', '-pub', multiple=True, help='Only list components which publish FORMAT')
+@click.option('--provides', '-pro', multiple=True, type=(str, str), help='Only list components which provide services REQ_FORMAT RESP_FORMAT')
+@click.option('--calls', '-cal', multiple=True, type=(str, str), help='Only list components which call services REQ_FORMAT RESP_FORMAT')
+@click.option('--deployed', is_flag=True, default=False, help='Display the deployed view. Shows details of deployed instances.')
+@click.pass_obj
+def list_component(obj, latest, subscribes, publishes, provides, calls, deployed):
+    '''Lists components in the public catalog. Uses flags to filter results.'''
+    subs = list(map(parse_input, subscribes)) if subscribes else None
+    pubs = list(map(parse_input, publishes)) if publishes else None
+    provs = list(map(parse_input_pair, provides)) if provides else None
+    cals = list(map(parse_input_pair, calls)) if calls else None
+
+    user, catalog = obj['config']['user'], obj['catalog']
+    # TODO: How about components that you don't own but you have deployed?
+    comps = catalog.list_components(subs, pubs, provs, cals, latest, user=user)
+
+    active_profile = profiles.get_profile()
+    consul_host = active_profile.consul_host
+
+    click.echo("Active profile: {0}".format(profiles.get_active_name()))
+    click.echo("")
+
+    def format_resolve_results(results):
+        """Format the results from the resolve_name function call"""
+        if results:
+            # Most likely the results will always be length one until we migrate
+            # to a different way of registering names
+            return "\n".join([ pformat(result) for result in results ])
+        else:
+            return None
+
+    def get_instances_as_rows(comp):
+        """Get all deployed running instances of a component plus details about
+        those instances and return as a list of rows"""
+        cname = comp["name"]
+        cver = comp["version"]
+        ctype = comp["component_type"]
+
+        instances = dis.get_healthy_instances(user, cname, cver)
+        instances_status = ["Healthy"]*len(instances)
+        instances_conns = [ format_resolve_results(resolve_name(consul_host, instance)) \
+                for instance in instances ]
+
+        instances_defective = dis.get_defective_instances(user, cname, cver)
+        instances_status += ["Defective"]*len(instances_defective)
+        instances_conns += [""]*len(instances_defective)
+
+        instances += instances_defective
+
+        return list(zip(instances, instances_status, instances_conns))
+
+    # Generate grouped rows where a grouped row is (name, version, type, [instances])
+    grouped_rows = [ (comp, get_instances_as_rows(comp)) for comp in comps ]
+
+    # Display
+    if deployed:
+        def display_deployed(comp, instances):
+            cname = comp["name"]
+            cver = comp["version"]
+            ctype = comp["component_type"]
+
+            click.echo("Name: {0}".format(cname))
+            click.echo("Version: {0}".format(cver))
+            click.echo("Type: {0}".format(ctype))
+            click.echo(create_table(('Instance', 'Status', 'Connection'), instances))
+            click.echo("")
+
+        [ display_deployed(*row) for row in grouped_rows ]
+    else:
+        def format_row(comp, instances):
+            return comp["name"], comp["version"], comp["component_type"], \
+                util.format_description(comp["description"]), \
+                util.get_status_string(comp), comp["modified"], len(instances)
+
+        rows = [ format_row(*grouped_row) for grouped_row in grouped_rows ]
+        click.echo(create_table(('Name', 'Version', 'Type', 'Description',
+            'Status', 'Modified', '#Deployed'), rows))
+        click.echo("\nUse the \"--deployed\" option to see more details on deployments")
+
+
+@component.command()
+@click.argument('component', metavar="name:version")
+@click.pass_obj
+def show(obj, component):
+    '''Provides more information about a COMPONENT'''
+    cname, cver = parse_input(component)
+    catalog = obj['catalog']
+    comp_spec = catalog.get_component_spec(cname, cver)
+
+    click.echo(util.format_json(comp_spec))
+
+
+_help_dmaap_file = """
+Path to a file that contains a json of dmaap client information.  The structure of the json is expected to be:
+
+  {
+    <config_key1>: {..client object 1..},
+    <config_key2>: {..client object 2..},
+    ...
+  }
+
+Where "client object" can be for message or data router. The "config_key" matches the value of specified in the message router "streams" in the component specification.
+
+Please refer to the documentation for examples of "client object".
+"""
+
+def _parse_dmaap_file(dmaap_file):
+    try:
+        with open(dmaap_file, 'r+') as f:
+            dmaap_map = json.load(f)
+            dmaap.validate_dmaap_map_schema(dmaap_map)
+            return dmaap.apply_defaults_dmaap_map(dmaap_map)
+    except Exception as e:
+        message = "Problems with parsing the dmaap file. Check to make sure that it is a valid json and is in the expected format."
+        raise DcaeException(message)
+
+
+_help_inputs_file = """
+Path to a file that contains a json that contains values to be used to bind to configuration parameters that have been marked as "sourced_at_deployment". The structure of the json is expected to be:
+
+ {
+   <parameter1 name>: value,
+   <parameter2 name>: value
+ }
+
+The "parameter name" is the value of the "name" property for the given configuration parameter.
+"""
+
+def _parse_inputs_file(inputs_file):
+    try:
+        with open(inputs_file, 'r+') as f:
+            inputs_map = json.load(f)
+            # TODO: Validation of schema in the future?
+            return inputs_map
+    except Exception as e:
+        message = "Problems with parsing the inputs file. Check to make sure that it is a valid json and is in the expected format."
+        raise DcaeException(message)
+
+
+_help_policy_file = """
+Path to a file that contains a json of an (update/remove) Policy change.
+All "policies" can also be specified.
+The structure of the json is expected to be:
+
+{
+"updated_policies": [{"policyName": "value", "": ""},{"policyName": "value", "": ""}],
+"removed_policies": [{"policyName": "value", "": ""},{"policyName": "value", "": ""}],
+"policies": [{"policyName": "value", "": ""},{"policyName": "value", "": ""}]
+}
+"""
+
+def _parse_policy_file(policy_file):
+    try:
+        with open(policy_file, 'r+') as f:
+            policy_change_file = json.load(f)
+            policy.validate_against_policy_schema(policy_change_file)
+            return policy_change_file
+    except Exception as e:
+        click.echo(format(e))
+        message = "Problems with parsing the Policy file. Check to make sure that it is a valid json and is in the expected format."
+        raise DcaeException(message)
+
+@component.command()
+@click.option('--external-ip', '-ip', default=None, help='The external IP address of the Docker host. Only used for Docker components.')
+@click.option('--additional-user', default=None, help='Additional user to grab instances from.')
+@click.option('--attached', is_flag=True, help='(Docker) dcae-cli deploys then attaches to the component when set')
+@click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies')
+@click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
+        help=_help_dmaap_file)
+@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
+        help=_help_inputs_file)
+@click.argument('component')
+@click.pass_obj
+def run(obj, external_ip, additional_user, attached, force, dmaap_file, component,
+        inputs_file):
+    '''Runs latest (or specific) COMPONENT version. You may optionally specify version via COMPONENT:VERSION'''
+
+    click.echo("Running the Component.....")
+    click.echo("")
+
+    cname, cver = parse_input(component)
+    user, catalog = obj['config']['user'], obj['catalog']
+
+    dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {}
+    inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {}
+
+    try:
+        run_component(user, cname, cver, catalog, additional_user, attached, force,
+                dmaap_map, inputs_map, external_ip)
+    except DiscoveryNoDownstreamComponentError as e:
+        message = "Either run a compatible downstream component first or run with the --force flag to ignore this error"
+        raise DcaeException(message)
+    except inputs.InputsValidationError as e:
+        click.echo("ERROR: There is a problem. {0}".format(e))
+        click.echo("")
+        message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct"
+        raise DcaeException(message)
+
+@component.command()
+@click.argument('component')
+@click.pass_obj
+def undeploy(obj,  component):
+    '''Undeploy latest (or specific) COMPONENT version. You may optionally specify version via COMPONENT:VERSION'''
+    cname, cver = parse_input(component)
+    user, catalog = obj['config']['user'], obj['catalog']
+    undeploy_component(user, cname, cver, catalog)
+
+
+@component.command()
+@click.argument('specification', type=click.Path(resolve_path=True, exists=True))
+@click.option('--additional-user', default=None, help='Additional user to grab instances from.')
+@click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies')
+@click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
+        help=_help_dmaap_file)
+@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
+        help=_help_inputs_file)
+@click.pass_obj
+def dev(obj, specification, additional_user, force, dmaap_file, inputs_file):
+    '''Set up component in development for discovery, use for local development'''
+    user, catalog = obj['config']['user'], obj['catalog']
+
+    dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {}
+    inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {}
+
+    with open(specification, 'r+') as f:
+        spec = json.loads(f.read())
+        try:
+            dev_component(user, catalog, spec, additional_user, force, dmaap_map,
+                    inputs_map)
+        except DiscoveryNoDownstreamComponentError as e:
+            message = "Either run a compatible downstream component first or run with the --force flag to ignore this error"
+            raise DcaeException(message)
+        except inputs.InputsValidationError as e:
+            click.echo("ERROR: There is a problem. {0}".format(e))
+            click.echo("")
+            message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct"
+            raise DcaeException(message)
+
+
+@component.command()
+@click.argument('component')
+@click.pass_obj
+def publish(obj, component):
+    """Pushes a COMPONENT to the public catalog"""
+    name, version = parse_input(component)
+    user, catalog = obj['config']['user'], obj['catalog']
+
+    try:
+        # Dependent data formats must be published first before publishing
+        # component. Check that here
+        unpub_formats = catalog.get_unpublished_formats(name, version)
+
+        if unpub_formats:
+            click.echo("ERROR: You must publish dependent data formats first:")
+            click.echo("")
+            click.echo("\n".join([":".join(uf) for uf in unpub_formats]))
+            click.echo("")
+            return
+    except MissingEntry as e:
+        raise DcaeException("Component not found")
+
+    if catalog.publish_component(user, name, version):
+        click.echo("Component has been published")
+    else:
+        click.echo("ERROR: Component could not be published")
+
+
+@component.command()
+@click.option('--update', is_flag=True, help='Updates a locally added component if it has not already been published')
+@click.argument('specification', type=click.Path(resolve_path=True, exists=True))
+@click.pass_obj
+def add(obj, update, specification):
+    """Add Component to local onboarding catalog"""
+    user, catalog = obj['config']['user'], obj['catalog']
+
+    spec = load_json(specification)
+    catalog.add_component(user, spec, update)
+
+
+@component.command()
+@click.option('--policy-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False), help=_help_policy_file)
+@click.argument('component')
+@click.pass_obj
+def reconfig(obj, policy_file, component):
+    """Reconfigure COMPONENT for Policy change.
+        Modify Consul KV pairs for ('updated_policies', 'removed_policies', and 'policies') for Policy change event,
+        Execute the reconfig script(s) in the Docker container"""
+
+    click.echo("Running Component Reconfiguration.....")
+    click.echo("")
+
+    #  Read and Validate the policy-file
+    policy_change_file = _parse_policy_file(policy_file) if policy_file else {}
+
+    if not (policy_change_file):
+        click.echo("ERROR: For component 'reconfig', you must specify a --policy-file")
+        click.echo("")
+        return
+    else:
+        #  The Component Spec contains the Policy 'Reconfig Script Path/ScriptName'
+        cname, cver = parse_input(component)
+        catalog     = obj['catalog']
+        comp_spec   = catalog.get_component_spec(cname, cver)
+
+    #  Check if component is running and healthy
+    active_profile = profiles.get_profile()
+    consul_host    = active_profile.consul_host
+    service_name   = os.environ["SERVICE_NAME"]
+    if dis.is_healthy(consul_host, service_name):
+        pass
+    else:
+        click.echo("ERROR: The component must be running and healthy.  It is not.")
+        click.echo("")
+        return
+
+    try:
+        policy_reconfig_path = comp_spec['auxilary']['policy']['script_path']
+    except KeyError:
+        click.echo("ERROR: Policy Reconfig Path (auxilary/policy/script_path) is not specified in the Component Spec")
+        click.echo("")
+        return
+
+    kvUpdated = dis.policy_update(policy_change_file, dis.default_consul_host())
+
+    if kvUpdated:
+        active_profile = profiles.get_profile()
+        docker_logins  = dis.get_docker_logins()
+
+        command = dis.build_policy_command(policy_reconfig_path, policy_change_file, dis.default_consul_host())
+
+        #  Run the Policy Reconfig script
+        client = du.get_docker_client(active_profile, docker_logins)
+        du.reconfigure(client, service_name, command)
+    else:
+        click.echo("ERROR: There was a problem updating the policies in Consul")
+        click.echo("")
+        return
+
+    click.echo("")
+    click.echo("The End of Component Reconfiguration")
diff --git a/mod/onboardingapi/dcae_cli/commands/data_format/__init__.py b/mod/onboardingapi/dcae_cli/commands/data_format/__init__.py
new file mode 100644
index 0000000..b025f61
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/data_format/__init__.py
@@ -0,0 +1,25 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+"""
+Provides the data format group
+"""
+from .commands import data_format
diff --git a/mod/onboardingapi/dcae_cli/commands/data_format/commands.py b/mod/onboardingapi/dcae_cli/commands/data_format/commands.py
new file mode 100644
index 0000000..b952336
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/data_format/commands.py
@@ -0,0 +1,163 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+"""
+Provides data format commands
+"""
+import json
+
+import click
+
+import genson
+
+import sys
+
+import os
+
+from jsonschema import Draft4Validator
+
+from dcae_cli.util import load_json
+from dcae_cli.util.logger import get_logger
+
+from dcae_cli.commands import util
+from dcae_cli.commands.util import create_table, parse_input
+
+from dcae_cli.catalog.exc import MissingEntry
+from dcae_cli.catalog.exc import DcaeException
+
+
+logger = get_logger('DataFormatCommand')
+
+
+@click.group()
+def data_format():
+    pass
+
+
+@data_format.command()
+@click.option('--update', is_flag=True, help='Updates a locally added data format if it has not already been published')
+@click.argument('specification', type=click.Path(resolve_path=True, exists=True))
+@click.pass_obj
+def add(obj, update, specification):
+    '''Tracks a Format file Specification locally, but does not push to the catalog'''
+    spec = load_json(specification)
+    user, catalog = obj['config']['user'], obj['catalog']
+    catalog.add_format(spec, user, update)
+
+
+@data_format.command(name='list')
+@click.option('--latest', is_flag=True, help='Only list the latest version of data formats')
+@click.pass_obj
+def list_format(obj, latest):
+    """Lists all your Data Formats"""
+    user, catalog = obj['config']['user'], obj['catalog']
+    dfs = catalog.list_formats(latest, user=user)
+
+    def format_record(df):
+        return (df["name"], df["version"],
+                util.format_description(df["description"]),
+                util.get_status_string(df), df["modified"])
+
+    dfs = [ format_record(df) for df in dfs ]
+
+    click.echo("")
+    click.echo("Data formats for {0}".format(user))
+    click.echo(create_table(('Name', 'Version', 'Description', 'Status', 'Modified'), dfs))
+
+
+@data_format.command()
+@click.argument('data-format', metavar="name:version")
+@click.pass_obj
+def show(obj, data_format):
+    '''Provides more information about a Data Format'''
+    name, ver = parse_input(data_format)
+    spec = obj['catalog'].get_format_spec(name, ver)
+
+    click.echo(util.format_json(spec))
+
+
+@data_format.command()
+@click.argument('data-format')
+@click.pass_obj
+def publish(obj, data_format):
+    """Publish Format to make publicly available"""
+    name, version = parse_input(data_format)
+    user, catalog = obj['config']['user'], obj['catalog']
+
+    if catalog.publish_format(user, name, version):
+        click.echo("Data format has been published")
+    else:
+        click.echo("ERROR: Data format could not be published")
+
+@data_format.command()
+@click.option('--keywords', is_flag=True, help='Adds a template of possible descriptive keywords', default=False)
+@click.argument('name_version', metavar="name:version",  required = True)
+@click.argument('file-or-dir-path', type=click.Path(resolve_path=True, exists=True, dir_okay=True, file_okay=True, readable=True),   metavar="file-or-dir-path")
+@click.pass_obj
+def generate(obj, name_version, file_or_dir_path, keywords):
+    '''Create schema from a file or directory examples'''
+    name, version = parse_input(name_version)
+    if version == None:
+      version = ""
+    schema = genson.Schema()
+    if os.path.isfile(file_or_dir_path):
+      addfile(file_or_dir_path, schema)
+    else:
+      foundJSON = False
+      for root, dirs, files in os.walk(file_or_dir_path):
+          for filename in files:
+            fullfilename = os.path.join(file_or_dir_path, filename)
+            addfile(fullfilename,schema)
+            foundJSON = True
+      if foundJSON == False:
+        raise DcaeException('No JSON files found in ' + file_or_dir_path)
+
+    json_obj = json.loads(schema.to_json())
+    json_obj['$schema'] = "http://json-schema.org/draft-04/schema#"
+    jschema = json.dumps(json_obj)
+    jschema = jschema.replace('"required":', '"additionalproperties": true, "required":')
+    jschema = jschema.replace('"type":', ' "description": "", "type":')
+
+    if (keywords):
+      jschema = jschema.replace('"type": "string"', ' "maxLength": 0, "minLength": 0, "pattern": "", "type": "string"')
+      jschema = jschema.replace('"type": "integer"', ' "maximum": 0, "mininimum": 0, "multipleOf": 0, "type": "integer"')
+      jschema = jschema.replace('"type": "array"', ' "maxItems": 0, "minItems": 0, "uniqueItems": "false", "type": "array"')
+
+    jschema = '{ "self": { "name": "' + name + '", "version": "' + version + '", "description": ""} , "dataformatversion": "1.0.0", "jsonschema": ' + jschema + '}'
+    #Draft4Validator.check_schema(json.loads(jschema))
+    try:
+      print(json.dumps(json.loads(jschema), sort_keys=True, indent=4 ))
+    except ValueError:
+      raise DcaeException('Problem with JSON generation')
+
+def addfile(filename, schema):
+  try:
+    fileadd = open(filename, "r")
+  except IOError:
+    raise DcaeException('Cannot open' + filename)
+  try:
+    json_object = json.loads(fileadd.read())
+    schema.add_object(json_object)
+  except ValueError:
+    raise DcaeException('Bad JSON file: ' + filename)
+  finally:
+    fileadd.close()
+
diff --git a/mod/onboardingapi/dcae_cli/commands/profiles/__init__.py b/mod/onboardingapi/dcae_cli/commands/profiles/__init__.py
new file mode 100644
index 0000000..0d71c1b
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/profiles/__init__.py
@@ -0,0 +1,25 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+"""
+Provides the profiles group
+"""
+from .commands import profiles
diff --git a/mod/onboardingapi/dcae_cli/commands/profiles/commands.py b/mod/onboardingapi/dcae_cli/commands/profiles/commands.py
new file mode 100644
index 0000000..df34b5c
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/profiles/commands.py
@@ -0,0 +1,87 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+"""
+Provides profiles commands
+"""
+import json
+
+import click
+
+from dcae_cli.util.exc import DcaeException
+from dcae_cli.util.profiles import (get_profiles, activate_profile, get_active_name, update_profile,
+                                    delete_profile, create_profile)
+
+
+@click.group()
+def profiles():
+    pass
+
+
+@profiles.command()
+@click.argument('name')
+def activate(name):
+    '''Sets profile (name) as the active profile'''
+    activate_profile(name)
+
+
+@profiles.command(name='list')
+def list_profiles():
+    '''Lists available profiles'''
+    profiles = get_profiles(include_active=False)
+    active = get_active_name()
+    names = sorted(profiles.keys())
+    outputs = ("{} {}".format('  ' if not name == active else '* ', name) for name in names)
+    click.echo('\n'.join(outputs))
+
+
+@profiles.command()
+@click.argument('name')
+def show(name):
+    '''Provides more information about a Profile'''
+    profiles = get_profiles()
+    try:
+        click.echo(json.dumps(profiles[name], sort_keys=True, indent=4))
+    except KeyError as e:
+        raise DcaeException("Profile '{}' does not exist.".format(e))
+
+
+@profiles.command()
+@click.argument('name', type=click.STRING)
+def create(name):
+    '''Creates new profile (name), with defaults'''
+    create_profile(name)
+
+
+@profiles.command(name='set')
+@click.argument('name')
+@click.argument('key')
+@click.argument('value')
+def update(name, key, value):
+    '''Updates profile (name) for specific Key/Value'''
+    update_profile(name, **{key: value})
+
+
+@profiles.command()
+@click.argument('name')
+def delete(name):
+    '''Deletes profile (name)'''
+    delete_profile(name)
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/format.json
new file mode 100644
index 0000000..8456a30
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/format.json
@@ -0,0 +1,10 @@
+{
+  "self": {
+    "name": "std.empty",
+    "version": "1.0.6",
+    "description": "Represents an empty message with no content"
+  },
+  "dataformatversion": "1.0.0",
+  "jsonschema": {
+  }
+}
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json
new file mode 100644
index 0000000..9642a6e
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_end.json
@@ -0,0 +1,77 @@
+{  
+   "self":{  
+      "name":"cdap.helloworld.mock.catalog.testing.endnode",
+      "version":"0.7.1",
+      "description":"cdap test component",
+      "component_type":"cdap"
+   },
+   "streams":{  
+      "subscribes": [],
+      "publishes": [{
+         "format": "std.empty",
+         "version": "1.0.6",
+         "config_key": "stream_publish_example",
+         "type": "http"
+      }]
+   },
+   "services":{  
+      "calls": [],
+      "provides":[  
+         {  
+            "request":{  
+               "format":"std.empty",
+               "version":"1.0.6"
+            },
+            "response":{  
+               "format":"std.empty",
+               "version":"1.0.6"
+            },
+            "service_name":"Greeting",
+            "service_endpoint":"greet",
+            "verb":"GET"
+         }
+      ]
+   },
+   "parameters": {  
+       "app_config" : [
+                         {"name" : "some_param",
+                          "description" : "some desc",
+                          "value" : "some_value",
+                          "designer_editable" : false,
+                          "sourced_at_deployment" : false,
+                          "policy_editable" : false}
+                      ],
+       "app_preferences" : [
+                             {"name" : "some_param2",
+                              "description" : "some desc2",
+                              "value" : "some_value2",
+                              "designer_editable" : false,
+                              "sourced_at_deployment" : false,
+                              "policy_editable" : false}
+                           ],
+       "program_preferences" : [{"program_type" : "flows",
+                                 "program_id" : "WhoFlow",
+                                 "program_pref" : [{"name" : "some_param3",
+                                                    "description" : "some desc3",
+						    "value" : "some_value3",
+                                                    "designer_editable" : false,
+                                                    "sourced_at_deployment" : false,
+                                                    "policy_editable" : false}]}]
+   },
+   "auxilary": {
+    "streamname":"who",
+    "artifact_name" : "HelloWorld",
+    "artifact_version" : "3.4.3",
+    "programs" : [
+                    {"program_type" : "flows", "program_id" : "WhoFlow"},
+                    {"program_type" : "services", "program_id" : "Greeting"}
+                  ],
+    "namespace" : "hw"
+    },
+  "artifacts": [
+    {
+      "uri": "http://make-me-valid/jar_files/HelloWorld-3.4.3.jar",
+      "type": "jar"
+    }
+  ]
+}
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json
new file mode 100644
index 0000000..83b5c28
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/cdap/spec_start.json
@@ -0,0 +1,78 @@
+{  
+   "self":{  
+      "name":"cdap.helloworld.mock.catalog.testing.startnode",
+      "version":"0.7.1",
+      "description":"cdap test component",
+      "component_type":"cdap"
+   },
+   "streams":{  
+      "subscribes": [{
+         "format": "std.empty",
+         "version": "1.0.6",
+         "route": "/unsure_if_needed_for_cdap",
+         "type": "http"
+      }],
+      "publishes": []
+   },
+   "services":{  
+      "calls": [
+      {
+        "config_key": "service_call_example",
+        "verb": "GET",
+        "request": {
+          "format": "std.empty",
+          "version": "1.0.6"
+        },
+        "response": {
+          "format": "std.empty",
+          "version": "1.0.6"
+        }
+      }
+    ],
+    "provides":[]
+   },
+"parameters": {  
+       "app_config" : [
+                         {"name" : "some_param",
+                         "description" : "some desc",
+                         "value" : "some_value",
+                         "designer_editable" : false,
+                         "sourced_at_deployment" : false,
+                         "policy_editable" : false}
+                      ],
+       "app_preferences" : [
+                             {"name" : "some_param2",
+                              "description" : "some desc2",
+                              "value" : "some_value2",
+                              "designer_editable" : false,
+                              "sourced_at_deployment" : false,
+                              "policy_editable" : false}
+                           ],
+       "program_preferences" : [{"program_type" : "flows",
+                                 "program_id" : "WhoFlow",
+                                 "program_pref" : [{"name" : "some_param3",
+                                                    "description" : "some desc3",
+						    "value" : "some_value3",
+                                                    "designer_editable" : false,
+                                                    "sourced_at_deployment" : false,
+                                                    "policy_editable" : false}
+                                                    ]}]
+   },
+   "auxilary": {
+    "streamname":"who",
+    "artifact_name" : "HelloWorld",
+    "artifact_version" : "3.4.3",
+    "programs" : [
+                    {"program_type" : "flows", "program_id" : "WhoFlow"},
+                    {"program_type" : "services", "program_id" : "Greeting"}
+                  ],
+    "namespace" : "hw"
+    },
+  "artifacts": [
+    {
+      "uri": "http://make-me-valid/jar_files/HelloWorld-3.4.3.jar",
+      "type": "jar"
+    }
+  ]
+}
+
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json
new file mode 100644
index 0000000..5b86d9c
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/kpi-collector.comp.json
@@ -0,0 +1,47 @@
+{
+  "self": {
+    "version": "1.0.0",
+    "name": "std.vnf.kpi_collector",
+    "description": "Continuously publishes VNF KPIs",
+    "component_type": "docker"
+  },
+  "streams": {
+    "subscribes": [],
+    "publishes": [
+      {
+        "format": "std.vnf.kpi",
+        "version": "1.0.0",
+        "config_key": "kpi_pub",
+        "type": "http"
+      }
+    ]
+  },
+  "services": {
+    "calls": [],
+    "provides": []
+  },
+  "parameters": [
+    {
+      "name": "sleep_sec",
+      "value": 0.75,
+      "description": "Number of seconds to sleep between publishes",
+      "designer_editable": false,
+      "sourced_at_deployment": false,
+      "policy_editable": false
+    }
+  ],
+  "auxilary": {
+    "healthcheck": {
+        "type": "http",
+        "endpoint": "/health",
+        "interval": "15s",
+        "timeout": "1s"
+    }
+  },
+  "artifacts": [
+    {
+      "uri": "asimov-anomaly-collector",
+      "type": "docker image"
+    }
+  ]
+}
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/vnf-kpi.format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/vnf-kpi.format.json
new file mode 100644
index 0000000..6fba1a1
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/collector/vnf-kpi.format.json
@@ -0,0 +1,13 @@
+{
+  "self": {
+    "name": "std.vnf.kpi",
+    "version": "1.0.0",
+    "description": "Represents a KPI of a VNF at particular instance of time."
+  },
+  "dataformatversion": "1.0.0",
+  "reference": {
+      "name": "Common Event Format",
+      "format": "JSON",
+      "version": "25.0.0"
+  }
+}
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json
new file mode 100644
index 0000000..3e2d142
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/anomaly-model.comp.json
@@ -0,0 +1,52 @@
+{
+  "self": {
+    "version": "1.0.0",
+    "name": "asimov.anomaly_classifier",
+    "description": "Classifies VNF KPIs as anommalous or not",
+    "component_type": "docker"
+  },
+  "streams": {
+    "subscribes": [{
+        "format": "std.vnf.kpi",
+        "version": "1.0.0",
+        "route": "/data",
+        "type": "http"
+      }],
+    "publishes": [
+      {
+        "format": "asimov.std.integerClassification",
+        "version": "1.0.0",
+        "config_key": "pred",
+        "type": "http"
+      }
+    ]
+  },
+  "services": {
+    "calls": [],
+    "provides": []
+  },
+  "parameters": [
+    {
+      "name": "threshold",
+      "value": 0.75,
+      "description": "Probability threshold to exceed to be anomalous",
+      "designer_editable" : false,
+      "sourced_at_deployment" : false,
+      "policy_editable" : false
+    }
+  ],
+  "auxilary": {
+    "healthcheck": {
+        "type": "http",
+        "endpoint": "/health",
+        "interval": "15s",
+        "timeout": "1s"
+    }
+  },
+  "artifacts": [
+    {
+      "uri": "asimov-anomaly-model",
+      "type": "docker image"
+    }
+  ]
+}
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/badjson b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/badjson
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/badjson
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json
new file mode 100755
index 0000000..7db1e06
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex1.json
@@ -0,0 +1,3 @@
+{
+   "foobar": "test 1"
+}
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json
new file mode 100755
index 0000000..75a5fa4
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/generatedir/ex2.json
@@ -0,0 +1,4 @@
+{
+   "foobar2": "test 1"
+}
+
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/int-class.format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/int-class.format.json
new file mode 100644
index 0000000..7d3dedf
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/model/int-class.format.json
@@ -0,0 +1,18 @@
+{
+  "self": {
+    "name": "asimov.std.integerClassification",
+    "version": "1.0.0",
+    "description": "Represents a single classification from a machine learning model"
+  },
+  "dataformatversion": "1.0.0",
+  "jsonschema": {
+      "$schema": "http://json-schema.org/draft-04/schema#",
+      "type": "object",
+      "properties": {
+          "classification": {
+              "type": "string"
+          }
+      },
+      "additionalProperties": false
+  }
+}
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/empty.format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/empty.format.json
new file mode 100644
index 0000000..4ff7d3a
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/empty.format.json
@@ -0,0 +1,10 @@
+{
+  "self": {
+    "name": "std.empty",
+    "version": "1.0.0",
+    "description": "Represents an empty message with no content"
+  },
+  "dataformatversion": "1.0.0",
+  "jsonschema": {
+  }
+}
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/line-viz.comp.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/line-viz.comp.json
new file mode 100644
index 0000000..d3b9e23
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/line-viz.comp.json
@@ -0,0 +1,51 @@
+{
+  "self": {
+    "version": "1.0.0",
+    "name": "asimov.viz.line_plot",
+    "description": "Plots class probabilities as a line plot in real-time",
+    "component_type": "docker"
+  },
+  "streams": {
+    "subscribes": [
+      {
+        "format": "asimov.std.integerClassification",
+        "version": "1.0.0",
+        "route": "/prediction",
+        "type": "http"
+      }
+    ],
+    "publishes": []
+  },
+  "services": {
+    "calls": [],
+    "provides": [
+      {
+        "route": "/viz",
+        "verb": "GET",
+        "request": {
+          "format": "std.empty",
+          "version": "1.0.0"
+        },
+        "response": {
+          "format": "std.web.url",
+          "version": "1.0.0"
+        }
+      }
+    ]
+  },
+  "parameters": [],
+  "auxilary": {
+    "healthcheck": {
+        "type": "http",
+        "endpoint": "/health",
+        "interval": "15s",
+        "timeout": "1s"
+    }
+  },
+  "artifacts": [
+    {
+      "uri": "asimov-anomaly-viz",
+      "type": "docker image"
+    }
+  ]
+}
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/web-url.format.json b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/web-url.format.json
new file mode 100644
index 0000000..3e823b9
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/mocked_components/viz/web-url.format.json
@@ -0,0 +1,19 @@
+{
+  "self": {
+    "name": "std.web.url",
+    "version": "1.0.0",
+    "description": "Represents a web URL"
+  },
+  "dataformatversion": "1.0.0",
+  "jsonschema": {
+      "$schema": "http://json-schema.org/draft-04/schema#",
+      "type": "object",
+      "properties": {
+          "URL": {
+              "type": "string"
+          }
+      },
+      "additionalProperties": false
+    }
+}
+  
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/test_component_cmd.py b/mod/onboardingapi/dcae_cli/commands/tests/test_component_cmd.py
new file mode 100644
index 0000000..2bba4cf
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/test_component_cmd.py
@@ -0,0 +1,149 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+'''
+Tests component CLI commands
+'''
+import os
+import json
+from click.testing import CliRunner
+import time
+import pytest
+
+from dcae_cli.cli import cli
+from dcae_cli.catalog import MockCatalog
+
+TEST_DIR = os.path.dirname(__file__)
+
+
+def _get_spec(path):
+    with open(path) as file:
+        return json.load(file)
+
+
+def test_comp_docker(mock_cli_config, mock_db_url, obj=None):
+
+    obj = {'catalog': MockCatalog(purge_existing=True, db_name='dcae_cli.test.db',
+        enforce_image=False, db_url=mock_db_url),
+           'config': {'user': 'test-user'}}
+
+    df_kpi = os.path.join(TEST_DIR, 'mocked_components', 'collector', 'vnf-kpi.format.json')
+    comp_coll = os.path.join(TEST_DIR, 'mocked_components', 'collector', 'kpi-collector.comp.json')
+
+    df_cls = os.path.join(TEST_DIR, 'mocked_components', 'model', 'int-class.format.json')
+    comp_model = os.path.join(TEST_DIR, 'mocked_components', 'model', 'anomaly-model.comp.json')
+
+    df_empty = os.path.join(TEST_DIR, 'mocked_components', 'viz', 'empty.format.json')
+    df_url = os.path.join(TEST_DIR, 'mocked_components', 'viz', 'web-url.format.json')
+    comp_viz = os.path.join(TEST_DIR, 'mocked_components', 'viz', 'line-viz.comp.json')
+
+    runner = CliRunner()
+
+
+    # add the collector
+    cmd = "data_format add {:}".format(df_kpi).split()
+    assert runner.invoke(cli, cmd, obj=obj).exit_code == 0
+
+    cmd = "component add {:}".format(comp_coll).split()
+    assert runner.invoke(cli, cmd, obj=obj).exit_code == 0
+
+
+    # add the model
+    cmd = "data_format add {:}".format(df_cls).split()
+    assert runner.invoke(cli, cmd, obj=obj).exit_code == 0
+
+    cmd = "component add {:}".format(comp_model).split()
+    assert runner.invoke(cli, cmd, obj=obj).exit_code == 0
+
+
+    # add the viz
+    cmd = "data_format add {:}".format(df_empty).split()
+    assert runner.invoke(cli, cmd, obj=obj).exit_code == 0
+
+    cmd = "data_format add {:}".format(df_url).split()
+    assert runner.invoke(cli, cmd, obj=obj).exit_code == 0
+
+    cmd = "component add {:}".format(comp_viz).split()
+    assert runner.invoke(cli, cmd, obj=obj).exit_code == 0
+
+
+    # light test of component list
+    df_cls_spec = _get_spec(df_cls)
+    df_cls_name, df_cls_ver = df_cls_spec['self']['name'], df_cls_spec['self']['version']
+    comp_model_spec = _get_spec(comp_model)
+    comp_model_name = comp_model_spec['self']['name']
+
+    cmd = "component list -pub {:}".format(df_cls_name).split()
+    #assert comp_model_name in runner.invoke(cli, cmd, obj=obj).output
+
+    cmd = "component list -pub {:}:{:}".format(df_cls_name, df_cls_ver).split()
+    #assert comp_model_name in runner.invoke(cli, cmd, obj=obj).output
+
+
+    # light test of component info
+    cmd = "component show {:}".format(comp_model_name).split()
+    spec_str = runner.invoke(cli, cmd, obj=obj).output
+    assert comp_model_spec == json.loads(spec_str)
+
+
+@pytest.mark.skip(reason="This is not a pure unit test. Need a way to setup dependencies and trigger in the appropriate stages of testing.")
+def test_comp_cdap(obj=None):
+    """
+    This is not a unit test. It is bigger than that. It Does a full "workflow" test:
+    1) adds a data format
+    2) adds a cdap component
+    3) runs a cdap component using our "Rework" broker
+    4) undeploys the cdap component using our "Rework" broker
+
+    NOTE: TODO: Mocking out the broker would be an improvement over this, probably. This is impure. Mocking the broker owuld be a huge undertaking, though.
+    """
+
+    obj = {'catalog': MockCatalog(purge_existing=True, db_name='dcae_cli.test.db'),
+           'config': {'user': 'test-user'}}
+    runner = CliRunner()
+
+    #add the data format
+    df = os.path.join(TEST_DIR, 'mocked_components', 'cdap', 'format.json')
+    cmd = "data_format add {:}".format(df).split()
+    assert runner.invoke(cli, cmd, obj=obj).exit_code == 0
+
+    #add the CDAP components
+    # TODO: Need to update the host
+    jar = 'http://make-me-valid/HelloWorld-3.4.3.jar'
+
+    comp_cdap_start = os.path.join(TEST_DIR, 'mocked_components', 'cdap', 'spec_start.json')
+    cmd = "component add {0}".format(comp_cdap_start).split()
+    print(cmd)
+    result = runner.invoke(cli, cmd, obj=obj)
+    print(result.output)
+    assert result.exit_code == 0
+
+    comp_cdap_end = os.path.join(TEST_DIR, 'mocked_components', 'cdap', 'spec_end.json')
+    cmd = "component add {0}".format(comp_cdap_end).split()
+    print(cmd)
+    result = runner.invoke(cli, cmd, obj=obj)
+    print(result.output)
+    assert result.exit_code == 0
+
+if __name__ == '__main__':
+    '''Test area'''
+    #pytest.main([__file__, ])
+    test_comp_cdap()
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/test_data_format_cmd.py b/mod/onboardingapi/dcae_cli/commands/tests/test_data_format_cmd.py
new file mode 100644
index 0000000..a291a74
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/test_data_format_cmd.py
@@ -0,0 +1,122 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+'''
+Tests data_format CLI commands
+'''
+import os
+import json
+
+import pytest
+from click.testing import CliRunner
+
+from dcae_cli.cli import cli
+from dcae_cli.catalog import MockCatalog
+
+
+TEST_DIR = os.path.dirname(__file__)
+
+
+def _get_spec(path):
+    with open(path) as file:
+        return json.load(file)
+
+
+def test_basic(mock_cli_config, mock_db_url, tmpdir):
+    obj = {'catalog': MockCatalog(purge_existing=True, db_name='dcae_cli.test.db', 
+        enforce_image=False, db_url=mock_db_url),
+           'config': {'user': 'test-user'}}
+
+    runner = CliRunner()
+    spec_file = os.path.join(TEST_DIR, 'mocked_components', 'model', 'int-class.format.json')
+    cmd = "data_format add {:}".format(spec_file).split()
+
+    # succeed the first time
+    result = runner.invoke(cli, cmd, obj=obj)
+
+    assert result.exit_code == 0
+
+    # adding a duplicate is an error
+    result = runner.invoke(cli, cmd, obj=obj)
+    assert result.exit_code == 1
+    assert 'exists' in result.output.lower()
+
+    # allow updates
+    cmd = "data_format add --update {:}".format(spec_file).split()
+    result = runner.invoke(cli, cmd, obj=obj)
+    assert result.exit_code == 0
+
+
+    # light test of list format command
+    cmd = 'data_format list'.split()
+    df_spec = _get_spec(spec_file)
+    df_name = df_spec['self']['name']
+    assert df_name in runner.invoke(cli, cmd, obj=obj).output
+
+
+    # light test of component info
+    cmd = "data_format show {:}".format(df_name).split()
+    spec_str = runner.invoke(cli, cmd, obj=obj).output
+    assert df_spec == json.loads(spec_str)
+
+    # test of generate
+    bad_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'baddir')
+    cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_dir).split()
+    err_str = runner.invoke(cli, cmd, obj=obj).output
+    assert "does not exist" in err_str 
+
+    empty_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'emptydir')
+    try:
+      os.stat(empty_dir)
+    except:
+      os.mkdir(empty_dir)
+    cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(empty_dir).split()
+    err_str = runner.invoke(cli, cmd, obj=obj).output
+    assert "No JSON files found" in err_str 
+
+    bad_json = os.path.join(TEST_DIR, 'mocked_components', 'model', 'badjson')
+    cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_json).split()
+    err_str = runner.invoke(cli, cmd, obj=obj).output
+    assert "Bad JSON file" in err_str 
+
+    generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir')
+    cmd = "data_format generate --keywords name:1.0.2 {:} ".format(generate_dir).split()
+    actual = json.loads(runner.invoke(cli, cmd, obj=obj).output)
+    expected = json.loads('{\n    "dataformatversion": "1.0.0", \n    "jsonschema": {\n        "$schema": "http://json-schema.org/draft-04/schema#", \n        "description": "", \n        "properties": {\n            "foobar": {\n                "description": "", \n                "maxLength": 0, \n                "minLength": 0, \n                "pattern": "", \n                "type": "string"\n            }, \n            "foobar2": {\n                "description": "", \n                "maxLength": 0, \n                "minLength": 0, \n                "pattern": "", \n                "type": "string"\n            }\n        }, \n        "type": "object"\n    }, \n    "self": {\n        "description": "", \n        "name": "name", \n        "version": "1.0.2"\n    }\n}\n')
+    assert actual == expected
+
+    generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir')
+    cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split()
+    actual = json.loads(runner.invoke(cli, cmd, obj=obj).output)
+    expected = json.loads('{\n    "dataformatversion": "1.0.0", \n    "jsonschema": {\n        "$schema": "http://json-schema.org/draft-04/schema#", \n        "description": "", \n        "properties": {\n            "foobar": {\n                "description": "", \n                "type": "string"\n            }, \n            "foobar2": {\n                "description": "", \n                "type": "string"\n            }\n        }, \n        "type": "object"\n    }, \n    "self": {\n        "description": "", \n        "name": "name", \n        "version": "1.0.2"\n    }\n}\n'
+            )
+    assert actual == expected
+
+    generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir', 'ex1.json')
+    cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split()
+    actual = json.loads(runner.invoke(cli, cmd, obj=obj).output)
+    expected = json.loads('{\n    "dataformatversion": "1.0.0", \n    "jsonschema": {\n        "$schema": "http://json-schema.org/draft-04/schema#", \n        "additionalproperties": true, \n        "description": "", \n        "properties": {\n            "foobar": {\n                "description": "", \n                "type": "string"\n            }\n        }, \n        "required": [\n            "foobar"\n        ], \n        "type": "object"\n    }, \n    "self": {\n        "description": "", \n        "name": "name", \n        "version": "1.0.2"\n    }\n}\n')
+    assert actual == expected
+
+
+if __name__ == '__main__':
+    '''Test area'''
+    pytest.main([__file__, ])
diff --git a/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py b/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py
new file mode 100644
index 0000000..be89722
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/tests/test_profiles_cmd.py
@@ -0,0 +1,84 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+'''
+Tests profiles CLI commands
+'''
+import json
+
+import pytest
+import click
+from click.testing import CliRunner
+
+from dcae_cli import util
+from dcae_cli.cli import cli
+from dcae_cli.util import profiles
+from dcae_cli.util import config
+
+
+def test_basic(monkeypatch, tmpdir, mock_db_url):
+
+    runner = CliRunner()
+
+    # Setup config
+    test_db_url = mock_db_url
+    config_dict = { "user": "ninny", "active_profile": "fake-solutioning",
+            "db_url": test_db_url, "cli_version": "2.0.0" }
+    config_file = tmpdir.join("config.json")
+    config_file.write(json.dumps(config_dict))
+
+    # Setup profile
+    profile_dict = { "fake-solutioning": { "cdap_broker": "cdap_broker",
+        "config_binding_service": "config_binding_service",
+        "consul_host": "realsolcnsl00.dcae.solutioning.com",
+        "docker_host": "realsoldokr00.dcae.solutioning.com:2376" }}
+    profile_file = tmpdir.join("profiles.json")
+    profile_file.write(json.dumps(profile_dict))
+
+    monkeypatch.setattr(click, "get_app_dir", lambda app: str(tmpdir.realpath()))
+
+    cmd = 'profiles show fake-solutioning'.split()
+    result = runner.invoke(cli, cmd)
+    assert result.output == "{}\n".format(json.dumps(profile_dict["fake-solutioning"],
+        sort_keys=True, indent=4))
+
+    cmd = 'profiles list'.split()
+    result = runner.invoke(cli, cmd)
+    assert result.output == '*  fake-solutioning\n'
+
+    cmd = 'profiles create foo'.split()
+    result = runner.invoke(cli, cmd)
+
+    cmd = 'profiles list'.split()
+    result = runner.invoke(cli, cmd)
+    assert result.output == '*  fake-solutioning\n   foo\n'
+
+    cmd = 'profiles activate foo'.split()
+    result = runner.invoke(cli, cmd)
+
+    cmd = 'profiles list'.split()
+    result = runner.invoke(cli, cmd)
+    assert result.output == '   fake-solutioning\n*  foo\n'
+
+
+if __name__ == '__main__':
+    '''Test area'''
+    pytest.main([__file__, ])
diff --git a/mod/onboardingapi/dcae_cli/commands/util.py b/mod/onboardingapi/dcae_cli/commands/util.py
new file mode 100644
index 0000000..f9527fa
--- /dev/null
+++ b/mod/onboardingapi/dcae_cli/commands/util.py
@@ -0,0 +1,111 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+# -*- coding: utf-8 -*-
+"""
+Provides utilities for commands
+"""
+import json
+import textwrap
+from terminaltables import AsciiTable
+
+from dcae_cli.util import DcaeException
+
+
+def parse_input(input_):
+    '''Returns (name, version) tuple parsed from name:version'''
+    arg = input_.split(':')
+    if len(arg) == 1:
+        cname, cver = arg[0], None
+    elif len(arg) == 2:
+        cname, cver = arg
+        cver = None if not cver else cver
+    else:
+        raise DcaeException("Input '{:}' must be NAME or NAME:VERSION".format(input_))
+    return cname, cver
+
+
+def parse_input_pair(req, resp):
+    '''Returns a tuple output of `parse_input` for convenience'''
+    return parse_input(req), parse_input(resp)
+
+
+def create_table(header, entries):
+    '''Returns an ASCII table string'''
+    data = [header, ]
+    if entries:
+        data.extend(entries)
+    else:
+        data.append(['']*len(header))
+    return AsciiTable(data).table
+
+
+# Utility methods used to format records for displaying
+
+def get_status_string(record):
+    """Get the status label given a record of either data format or component"""
+    if "when_revoked"   not in record or \
+       "when_published" not in record or \
+       "when_added"     not in record:
+        return None
+
+    if record["when_revoked"] is not None:
+        return "revoked"
+    elif record["when_published"] is not None:
+        return "published"
+    else:
+        return "unpublished"
+
+def get_status_string_camel(record):
+    """Get the status label given a record of either data format or component, in camelCase"""
+    if "whenRevoked"   not in record or \
+       "whenPublished" not in record or \
+       "whenAdded"     not in record:
+        return None
+
+    if record["whenRevoked"] is not None:
+        return "revoked"
+    elif record["whenPublished"] is not None:
+        return "published"
+    else:
+        return "unpublished"
+    
+def format_description(description, line_width=50, num_lines=3):
+    """Formats the description field
+
+    Description field can be long. This function line wraps to a specified number
+    of lines. The last line trails with ".." if the text still overflows to
+    signal that there is more.
+    """
+    if not description:
+        return ''
+    lines = textwrap.wrap(description)
+    lines = lines[:num_lines]
+    last_line = lines.pop()
+
+    if len(last_line) > line_width and line_width > 2:
+        last_line = "{0}..".format(last_line[:-2])
+
+    lines.append(last_line)
+    return "\n".join(lines)
+
+
+def format_json(some_json):
+    return json.dumps(some_json, sort_keys=True, indent=4)