Enhance to query Consul for target docker host

* `SelectedDockerHost` actually queries by a name stem and location
* Shorten name
* Tag components with deployment id

Change-Id: I715f1de25fa047ce70eb26a5cc7615cfd3b408e7
Issue-ID: DCAEGEN2-91
Signed-off-by: Michael Hwang <mhwang@research.att.com>

8 files changed, 150 insertions, 40 deletions

diff --git a/docker/ChangeLog.md b/docker/ChangeLog.md
index 673e672..5094816 100644
--- a/docker/ChangeLog.md
+++ b/docker/ChangeLog.md
@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/) 
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.3.0+t.0.3]
+
+* Enhance `SelectedDockerHost` node type with `name_search` and add default to `docker_host_override`
+* Implement the functionality in the `select_docker_host` task to query Consul given location id and name search
+* Deprecate `location_id` on the `DockerContainerForComponents*` node types
+* Change `service_id` to be optional for `DockerContainerForComponents*` node types
+* Add deployment id as a tag for registration on the component
+
 ## [2.3.0]
 
 * Rip out dockering and use common python-dockering library
diff --git a/docker/docker-node-type.yaml b/docker/docker-node-type.yaml
index 5fb0e27..b1bf64c 100644
--- a/docker/docker-node-type.yaml
+++ b/docker/docker-node-type.yaml
@@ -7,7 +7,7 @@ plugins:
   docker:
     executor: 'central_deployment_agent'
     package_name: dockerplugin
-    package_version: 2.3.0
+    package_version: 2.3.0+t.0.3
 
 node_types:
     # The DockerContainerForComponents node type is to be used for DCAE service components that 
@@ -29,11 +29,18 @@ node_types:
 
             service_id:
                 type: string
-                description: Unique id for this DCAE service instance this component belongs to
+                description: >
+                  Unique id for this DCAE service instance this component belongs to. This value
+                  will be applied as a tag in the registration of this component with Consul.
+                default: Null
 
             location_id:
                 type: string
-                description: Location id of where to run the container
+                description: >
+                  Location id of where to run the container.
+                  DEPRECATED - No longer used. Infer the location from the docker host service 
+                    and/or node.
+                default: Null
 
             service_component_name_override:
                 type: string
@@ -228,12 +235,18 @@ node_types:
                 type: string
                 description: Location id of the Docker host to use
 
+            name_search:
+                type: string
+                description: String to use when matching for names
+                default: component-dockerhost
+
             # REVIEW: This field should really be optional but because there's no functionality
             # that provides the dynamic solution sought after yet, it has been promoted to be
             # required.
             docker_host_override:
                 type: string
                 description: Docker hostname here is used as a manual override
+                default: Null
 
         interfaces:
             cloudify.interfaces.lifecycle:
diff --git a/docker/dockerplugin/discovery.py b/docker/dockerplugin/discovery.py
index 32a8cd0..03a51f6 100644
--- a/docker/dockerplugin/discovery.py
+++ b/docker/dockerplugin/discovery.py
@@ -47,19 +47,17 @@ def _wrap_consul_call(consul_func, *args, **kwargs):
         raise DiscoveryConnectionError(e)
 
 
-def generate_service_component_name(service_component_type, service_id, location_id):
+def generate_service_component_name(service_component_type):
     """Generate service component id used to pass into the service component
     instance and used as the key to the service component configuration.
 
     Format:
-    <service component id>.<service component type>.<service id>.<location id>.dcae.com
-
-    TODO: The format will evolve.
+    <service component id>_<service component type>
     """
     # Random generated
-    service_component_id = str(uuid.uuid4())
-    return "{0}.{1}.{2}.{3}.dcae.com".format(
-            service_component_id, service_component_type, service_id, location_id)
+    # Copied from cdap plugin
+    return "{0}_{1}".format(str(uuid.uuid4()).replace("-",""),
+            service_component_type)
 
 
 def create_kv_conn(host):
@@ -204,3 +202,41 @@ def add_to_entry(conn, key, add_name, add_value):
         updated = conn.kv.put(key, new_vstring, cas=mod_index)       # if the key has changed since retrieval, this will return false
         if updated:
             return v
+
+
+def _find_matching_services(services, name_search, tags):
+    """Find matching services given search criteria"""
+    def is_match(service):
+        srv_name, srv_tags = service
+        return name_search in srv_name and \
+                all(map(lambda tag: tag in srv_tags, tags))
+
+    return [ srv[0] for srv in services.items() if is_match(srv) ]
+
+def search_services(conn, name_search, tags):
+    """Search for services that match criteria
+
+    Args:
+    -----
+    name_search: (string) Name to search for as a substring
+    tags: (list) List of strings that are tags. A service must match **all** the
+        tags in the list.
+
+    Retruns:
+    --------
+    List of names of services that matched
+    """
+    # srvs is dict where key is service name and value is list of tags
+    catalog_get_services_func = partial(_wrap_consul_call, conn.catalog.services)
+    index, srvs = catalog_get_services_func()
+
+    if srvs:
+        matches = _find_matching_services(srvs, name_search, tags)
+
+        if matches:
+            return matches
+
+        raise DiscoveryServiceNotFoundError(
+                "No matches found: {0}, {1}".format(name_search, tags))
+    else:
+        raise DiscoveryServiceNotFoundError("No services found")
diff --git a/docker/dockerplugin/tasks.py b/docker/dockerplugin/tasks.py
index a41f143..837c1e9 100644
--- a/docker/dockerplugin/tasks.py
+++ b/docker/dockerplugin/tasks.py
@@ -20,7 +20,7 @@
 
 # Lifecycle interface calls for DockerContainer
 
-import json, time, copy
+import json, time, copy, random
 from cloudify import ctx
 from cloudify.decorators import operation
 from cloudify.exceptions import NonRecoverableError, RecoverableError
@@ -71,14 +71,10 @@ def _setup_for_discovery(**kwargs):
 def _generate_component_name(**kwargs):
     """Generate component name"""
     service_component_type = kwargs['service_component_type']
-    service_id = kwargs['service_id']
-    location_id = kwargs['location_id']
-
     name_override = kwargs['service_component_name_override']
 
     kwargs['name'] = name_override if name_override \
-            else dis.generate_service_component_name(service_component_type,
-                    service_id, location_id)
+            else dis.generate_service_component_name(service_component_type)
     return kwargs
 
 def _done_for_create(**kwargs):
@@ -297,6 +293,14 @@ def _create_and_start_container(container_name, image, docker_host,
         raise DockerPluginDependencyNotReadyError(e)
 
 
+def _parse_cloudify_context(**kwargs):
+    """Parse Cloudify context
+
+    Extract what is needed. This is impure function because it requires ctx.
+    """
+    kwargs["deployment_id"] = ctx.deployment.id
+    return kwargs
+
 def _enhance_docker_params(**kwargs):
     """Setup Docker envs"""
     docker_config = kwargs.get("docker_config", {})
@@ -307,6 +311,14 @@ def _enhance_docker_params(**kwargs):
     envs_healthcheck = doc.create_envs_healthcheck(docker_config) \
             if "healthcheck" in docker_config else {}
     envs.update(envs_healthcheck)
+
+    # Set tags on this component for its Consul registration as a service
+    tags = [kwargs.get("deployment_id", None), kwargs["service_id"]]
+    tags = [ str(tag) for tag in tags if tag is not None ]
+    # Registrator will use this to register this component with tags. Must be
+    # comma delimited.
+    envs["SERVICE_TAGS"] = ",".join(tags)
+
     kwargs["envs"] = envs
 
     def combine_params(key, docker_config, kwargs):
@@ -384,7 +396,8 @@ def create_and_start_container_for_components(**start_inputs):
     _done_for_start(
             **_verify_component(
                 **_create_and_start_component(
-                    **_enhance_docker_params(**start_inputs))))
+                    **_enhance_docker_params(
+                        **_parse_cloudify_context(**start_inputs)))))
 
 
 def _update_delivery_url(**kwargs):
@@ -423,7 +436,8 @@ def create_and_start_container_for_components_with_streams(**start_inputs):
             **_update_delivery_url(
                 **_verify_component(
                     **_create_and_start_component(
-                        **_enhance_docker_params(**start_inputs)))))
+                        **_enhance_docker_params(
+                            **_parse_cloudify_context(**start_inputs))))))
 
 
 @wrap_error_handling_start
@@ -546,15 +560,28 @@ def cleanup_discovery(**kwargs):
 
 # Lifecycle interface calls for dcae.nodes.DockerHost
 
+
+@monkeypatch_loggers
 @operation
 def select_docker_host(**kwargs):
     selected_docker_host = ctx.node.properties['docker_host_override']
+    name_search = ctx.node.properties['name_search']
+    location_id = ctx.node.properties['location_id']
 
     if selected_docker_host:
         ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME] = selected_docker_host
         ctx.logger.info("Selected Docker host: {0}".format(selected_docker_host))
     else:
-        raise NonRecoverableError("Failed to find a suitable Docker host")
+        try:
+            conn = dis.create_kv_conn(CONSUL_HOST)
+            names = dis.search_services(conn, name_search, [location_id])
+            ctx.logger.info("Docker hosts found: {0}".format(names))
+            # Randomly choose one
+            ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME] = random.choice(names)
+        except (dis.DiscoveryConnectionError, dis.DiscoveryServiceNotFoundError) as e:
+            raise RecoverableError(e)
+        except Exception as e:
+            raise NonRecoverableError(e)
 
 @operation
 def unselect_docker_host(**kwargs):
diff --git a/docker/examples/blueprint-laika.yaml b/docker/examples/blueprint-laika.yaml
index 9a8dc46..98d27af 100644
--- a/docker/examples/blueprint-laika.yaml
+++ b/docker/examples/blueprint-laika.yaml
@@ -9,14 +9,9 @@ imports:
   - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/relationship/1.0.0/node-type.yaml
 
 inputs:
-
-  service_id:
-    description: Unique id used for an instance of this DCAE service. Use deployment id
-    default: 'foobar'
   laika_image:
     type: string
 
-
 node_templates:
 
   laika-zero:
@@ -24,10 +19,10 @@ node_templates:
     properties:
         service_component_type:
             'laika'
-        service_id:
-            { get_input: service_id }
         location_id:
             'rework-central'
+	service_id:
+	    'foo-service'
         application_config:
             some-param: "Lorem ipsum dolor sit amet"
             downstream-laika: "{{ laika }}"
@@ -62,10 +57,6 @@ node_templates:
     properties:
         service_component_type:
             'laika'
-        service_id:
-            { get_input: service_id }
-        location_id:
-            'rework-central'
         application_config:
             some-param: "Lorem ipsum dolor sit amet"
         image: { get_input : laika_image }
@@ -85,5 +76,5 @@ node_templates:
     properties:
         location_id:
             'rework-central'
-        docker_host_override:
+        name_search:
             'platform_dockerhost'
diff --git a/docker/setup.py b/docker/setup.py
index 9cdef0e..65ac0e9 100644
--- a/docker/setup.py
+++ b/docker/setup.py
@@ -24,7 +24,7 @@ from setuptools import setup
 setup(
     name='dockerplugin',
     description='Cloudify plugin for applications run in Docker containers',
-    version="2.3.0",
+    version="2.3.0+t.0.3",
     author='Michael Hwang, Tommy Carpenter',
     packages=['dockerplugin'],
     zip_safe=False,
diff --git a/docker/tests/test_discovery.py b/docker/tests/test_discovery.py
index 9a18519..cee75b1 100644
--- a/docker/tests/test_discovery.py
+++ b/docker/tests/test_discovery.py
@@ -38,3 +38,18 @@ def test_wrap_consul_call():
     wrapped_foo = partial(dis._wrap_consul_call, foo_connection_error)
     with pytest.raises(dis.DiscoveryConnectionError):
         wrapped_foo("a", "b", "c")
+
+
+def test_find_matching_services():
+    services = { "component_dockerhost_1": ["foo", "bar"],
+            "platform_dockerhost": [], "component_dockerhost_2": ["baz"] }
+    assert sorted(["component_dockerhost_1", "component_dockerhost_2"]) \
+        == sorted(dis._find_matching_services(services, "component_dockerhost", []))
+
+    assert ["component_dockerhost_1"] == dis._find_matching_services(services, \
+            "component_dockerhost", ["foo", "bar"])
+
+    assert ["component_dockerhost_1"] == dis._find_matching_services(services, \
+            "component_dockerhost", ["foo"])
+
+    assert [] == dis._find_matching_services(services, "unknown", ["foo"])
diff --git a/docker/tests/test_tasks.py b/docker/tests/test_tasks.py
index 74482c6..6661532 100644
--- a/docker/tests/test_tasks.py
+++ b/docker/tests/test_tasks.py
@@ -25,6 +25,17 @@ import dockerplugin
 from dockerplugin import tasks
 
 
+def test_generate_component_name():
+    kwargs = { "service_component_type": "doodle",
+            "service_component_name_override": None }
+
+    assert "doodle" in tasks._generate_component_name(**kwargs)["name"]
+
+    kwargs["service_component_name_override"] = "yankee"
+
+    assert "yankee" == tasks._generate_component_name(**kwargs)["name"]
+
+
 def test_parse_streams(monkeypatch):
     # Good case for streams_publishes
     test_input = { "streams_publishes": [{"name": "topic00", "type": "message_router"},
@@ -166,33 +177,42 @@ def test_update_delivery_url(monkeypatch):
 def test_enhance_docker_params():
     # Good - Test empty docker config
 
-    test_kwargs = { "docker_config": {} }
+    test_kwargs = { "docker_config": {}, "service_id": None }
     actual = tasks._enhance_docker_params(**test_kwargs)
 
-    assert actual == {'envs': {}, 'docker_config': {}}
+    assert actual == {'envs': {"SERVICE_TAGS": ""}, 'docker_config': {}, "service_id": None }
 
     # Good - Test just docker config ports and volumes
 
     test_kwargs = { "docker_config": { "ports": ["1:1", "2:2"],
-        "volumes": [{"container": "somewhere", "host": "somewhere else"}] } }
+        "volumes": [{"container": "somewhere", "host": "somewhere else"}] },
+        "service_id": None }
     actual = tasks._enhance_docker_params(**test_kwargs)
 
-    assert actual == {'envs': {}, 'docker_config': {'ports': ['1:1', '2:2'],
+    assert actual == {'envs': {"SERVICE_TAGS": ""}, 'docker_config': {'ports': ['1:1', '2:2'],
         'volumes': [{'host': 'somewhere else', 'container': 'somewhere'}]},
         'ports': ['1:1', '2:2'], 'volumes': [{'host': 'somewhere else',
-            'container': 'somewhere'}]}
+            'container': 'somewhere'}], "service_id": None}
 
     # Good - Test just docker config ports and volumes with overrrides
 
     test_kwargs = { "docker_config": { "ports": ["1:1", "2:2"],
         "volumes": [{"container": "somewhere", "host": "somewhere else"}] },
         "ports": ["3:3", "4:4"], "volumes": [{"container": "nowhere", "host":
-        "nowhere else"}]}
+        "nowhere else"}],
+        "service_id": None }
     actual = tasks._enhance_docker_params(**test_kwargs)
 
-    assert actual == {'envs': {}, 'docker_config': {'ports': ['1:1', '2:2'],
+    assert actual == {'envs': {"SERVICE_TAGS": ""}, 'docker_config': {'ports': ['1:1', '2:2'],
         'volumes': [{'host': 'somewhere else', 'container': 'somewhere'}]},
         'ports': ['1:1', '2:2', '3:3', '4:4'], 'volumes': [{'host': 'somewhere else', 
             'container': 'somewhere'}, {'host': 'nowhere else', 'container':
-            'nowhere'}]}
+            'nowhere'}], "service_id": None}
+
+    # Good
+
+    test_kwargs = { "docker_config": {}, "service_id": "zed",
+            "deployment_id": "abc" }
+    actual = tasks._enhance_docker_params(**test_kwargs)
 
+    assert actual["envs"] == {"SERVICE_TAGS": "abc,zed"}