diff options
Diffstat (limited to 'docker')
| -rw-r--r-- | docker/ChangeLog.md | 7 | ||||
| -rw-r--r-- | docker/docker-node-type.yaml | 6 | ||||
| -rw-r--r-- | docker/dockerplugin/__init__.py | 3 | ||||
| -rw-r--r-- | docker/dockerplugin/discovery.py | 17 | ||||
| -rw-r--r-- | docker/dockerplugin/tasks.py | 106 | ||||
| -rw-r--r-- | docker/examples/blueprint-laika-policy.yaml | 117 | ||||
| -rw-r--r-- | docker/examples/blueprint-laika.yaml | 8 | ||||
| -rw-r--r-- | docker/requirements.txt | 3 | ||||
| -rw-r--r-- | docker/setup.py | 5 |
9 files changed, 255 insertions, 17 deletions
diff --git a/docker/ChangeLog.md b/docker/ChangeLog.md index 5094816..0d0eafc 100644 --- a/docker/ChangeLog.md +++ b/docker/ChangeLog.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [2.4.0] + +* Change *components* to be policy reconfigurable: + - Add policy execution operation + - Add policy decorators to task so that application configuration will be merged with policy +* Fetch Docker logins from Consul + ## [2.3.0+t.0.3] * Enhance `SelectedDockerHost` node type with `name_search` and add default to `docker_host_override` diff --git a/docker/docker-node-type.yaml b/docker/docker-node-type.yaml index b1bf64c..7efc84d 100644 --- a/docker/docker-node-type.yaml +++ b/docker/docker-node-type.yaml @@ -7,7 +7,7 @@ plugins: docker: executor: 'central_deployment_agent' package_name: dockerplugin - package_version: 2.3.0+t.0.3 + package_version: 2.4.0 node_types: # The DockerContainerForComponents node type is to be used for DCAE service components that @@ -83,6 +83,10 @@ node_types: delete: # Delete configuration from Consul implementation: docker.dockerplugin.cleanup_discovery + dcae.interfaces.policy: + # This is to be invoked by the policy handler upon policy updates + policy_update: + implementation: docker.dockerplugin.policy_update # This node type is intended for DCAE service components that use DMaaP and must use the diff --git a/docker/dockerplugin/__init__.py b/docker/dockerplugin/__init__.py index ef1bfec..669e196 100644 --- a/docker/dockerplugin/__init__.py +++ b/docker/dockerplugin/__init__.py @@ -27,4 +27,5 @@ from .tasks import create_for_components, create_for_components_with_streams, \ create_and_start_container_for_components_with_streams, \ create_for_platforms, create_and_start_container, \ create_and_start_container_for_components, create_and_start_container_for_platforms, \ - stop_and_remove_container, cleanup_discovery, select_docker_host, unselect_docker_host + stop_and_remove_container, cleanup_discovery, select_docker_host, unselect_docker_host, \ + policy_update diff --git a/docker/dockerplugin/discovery.py b/docker/dockerplugin/discovery.py index 03a51f6..8361c13 100644 --- a/docker/dockerplugin/discovery.py +++ b/docker/dockerplugin/discovery.py @@ -38,6 +38,9 @@ class DiscoveryConnectionError(RuntimeError): class DiscoveryServiceNotFoundError(RuntimeError): pass +class DiscoveryKVEntryNotFoundError(RuntimeError): + pass + def _wrap_consul_call(consul_func, *args, **kwargs): """Wrap Consul call to map errors""" @@ -84,6 +87,20 @@ def remove_service_component_config(kv_conn, service_component_name): kv_delete_func(service_component_name) +def get_kv_value(kv_conn, key): + """Get a key-value entry's value from Consul + + Raises DiscoveryKVEntryNotFoundError if entry not found + """ + kv_get_func = partial(_wrap_consul_call, kv_conn.kv.get) + (index, val) = kv_get_func(key) + + if val: + return json.loads(val['Value']) # will raise ValueError if not JSON, let it propagate + else: + raise DiscoveryKVEntryNotFoundError("{0} kv entry not found".format(key)) + + def _create_rel_key(service_component_name): return "{0}:rel".format(service_component_name) diff --git a/docker/dockerplugin/tasks.py b/docker/dockerplugin/tasks.py index 837c1e9..e42e47d 100644 --- a/docker/dockerplugin/tasks.py +++ b/docker/dockerplugin/tasks.py @@ -25,6 +25,7 @@ from cloudify import ctx from cloudify.decorators import operation from cloudify.exceptions import NonRecoverableError, RecoverableError import dockering as doc +from dcaepolicy import Policies, POLICIES, POLICY_MESSAGE_TYPE from dockerplugin import discovery as dis from dockerplugin.decorators import monkeypatch_loggers, wrap_error_handling_start, \ merge_inputs_for_start @@ -46,6 +47,29 @@ DEFAULT_SCHEME = "http" SERVICE_COMPONENT_NAME = "service_component_name" SELECTED_CONTAINER_DESTINATION = "selected_container_destination" CONTAINER_ID = "container_id" +APPLICATION_CONFIG = "application_config" + + +# Utility methods + +def _get_docker_logins(consul_host=CONSUL_HOST): + """Get Docker logins + + The assumption is that all Docker logins to be used will be available in + Consul's key-value store under "docker_plugin/docker_logins" as a list of + json objects where each object is a single login: + + [{ "username": "dcae_dev_ro", "password": "att123ro", + "registry": "nexus01.research.att.com:18443" }] + """ + # REVIEW: The error handling may have to be re-examined. The current thought is + # that the logins *must* be setup even with an empty list otherwise the task + # will fail (fail fast). One alterative is to pass back empty list upon any + # issues but this would push potential issues to a later point of the + # deployment. + kv_conn = dis.create_kv_conn(consul_host) + return dis.get_kv_value(kv_conn, "docker_plugin/docker_logins") + # Lifecycle interface calls for dcae.nodes.DockerContainer @@ -53,7 +77,7 @@ def _setup_for_discovery(**kwargs): """Setup for config discovery""" try: name = kwargs['name'] - application_config = kwargs['application_config'] + application_config = kwargs[APPLICATION_CONFIG] # NOTE: application_config is no longer a json string and is inputed as a # YAML map which translates to a dict. We don't have to do any @@ -89,8 +113,14 @@ def _done_for_create(**kwargs): ctx.logger.info("Done setting up: {0}".format(name)) return kwargs +def _merge_policy_updates(**kwargs): + app_config = kwargs[APPLICATION_CONFIG] + kwargs[APPLICATION_CONFIG] = Policies.shallow_merge_policies_into(app_config) + return kwargs + @monkeypatch_loggers +@Policies.gather_policies_to_node @operation def create_for_components(**kwargs): """Create step for Docker containers that are components @@ -102,8 +132,9 @@ def create_for_components(**kwargs): """ _done_for_create( **_setup_for_discovery( - **_generate_component_name( - **ctx.node.properties))) + **_merge_policy_updates( + **_generate_component_name( + **ctx.node.properties)))) def _parse_streams(**kwargs): @@ -171,6 +202,7 @@ def _setup_for_discovery_streams(**kwargs): @monkeypatch_loggers +@Policies.gather_policies_to_node @operation def create_for_components_with_streams(**kwargs): """Create step for Docker containers that are components that use DMaaP @@ -185,9 +217,10 @@ def create_for_components_with_streams(**kwargs): _done_for_create( **_setup_for_discovery( **_setup_for_discovery_streams( - **_parse_streams( - **_generate_component_name( - **ctx.node.properties))))) + **_merge_policy_updates( + **_parse_streams( + **_generate_component_name( + **ctx.node.properties)))))) @monkeypatch_loggers @@ -261,7 +294,8 @@ def _create_and_start_container(container_name, image, docker_host, docker_host_ip = _lookup_service(docker_host, consul_host=consul_host) - client = doc.create_client(docker_host_ip, DOCKER_PORT) + logins = _get_docker_logins(consul_host=consul_host) + client = doc.create_client(docker_host_ip, DOCKER_PORT, logins=logins) hcp = doc.add_host_config_params_volumes(volumes=kwargs.get("volumes", None)) @@ -523,7 +557,8 @@ def stop_and_remove_container(**kwargs): docker_host_ip = _lookup_service(docker_host) - client = doc.create_client(docker_host_ip, DOCKER_PORT) + logins = _get_docker_logins() + client = doc.create_client(docker_host_ip, DOCKER_PORT, logins=logins) container_id = ctx.instance.runtime_properties[CONTAINER_ID] doc.stop_then_remove_container(client, container_id) @@ -558,6 +593,61 @@ def cleanup_discovery(**kwargs): raise RecoverableError(e) +def _notify_container(**kwargs): + """Notify container using the policy section in the docker_config""" + dc = kwargs["docker_config"] + + if "policy" in dc: + if dc["policy"]["trigger_type"] == "docker": + # REVIEW: Need to finalize on the docker config policy data structure + script_path = dc["policy"]["script_path"] + app_config = kwargs["application_config"] + updated_policies = kwargs["updated_policies"] + cmd = doc.build_policy_update_cmd(script_path, use_sh=False, + updated_policies=updated_policies, + application_config=app_config) + + docker_host = kwargs[SELECTED_CONTAINER_DESTINATION] + docker_host_ip = _lookup_service(docker_host) + logins = _get_docker_logins() + client = doc.create_client(docker_host_ip, DOCKER_PORT, logins=logins) + + container_id = kwargs["container_id"] + + doc.notify_for_policy_update(client, container_id, cmd) + # else the default is no trigger + + return kwargs + +def _done_for_policy_update(**kwargs): + name = kwargs['name'] + ctx.instance.runtime_properties.update(kwargs) + ctx.logger.info("Done updating for policy: {0}".format(name)) + return kwargs + +@monkeypatch_loggers +@Policies.update_policies_on_node(configs_only=True) +@operation +def policy_update(updated_policies, **kwargs): + """Policy update task + + This method is responsible for updating the application configuration and + notifying the applications that the change has occurred. This is to be used + for the dcae.interfaces.policy.policy_update operation. + + :updated_policies: contains the list of changed policy-configs when configs_only=True + (default) Use configs_only=False to bring the full policy objects in :updated_policies:. + """ + update_inputs = copy.deepcopy(ctx.instance.runtime_properties) + update_inputs["updated_policies"] = updated_policies + + # Merge in policy updates into application config and make available + _done_for_policy_update( + **_notify_container( + **_setup_for_discovery( + **_merge_policy_updates(**update_inputs)))) + + # Lifecycle interface calls for dcae.nodes.DockerHost diff --git a/docker/examples/blueprint-laika-policy.yaml b/docker/examples/blueprint-laika-policy.yaml new file mode 100644 index 0000000..1531d21 --- /dev/null +++ b/docker/examples/blueprint-laika-policy.yaml @@ -0,0 +1,117 @@ +tosca_definitions_version: cloudify_dsl_1_3 + +description: > + This Blueprint installs a chain of two laika instances on a Docker cluster + +imports: + - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/docker/2.3.0/node-type.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/relationship/1.0.0/node-type.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/dcaepolicy/1.0.0/node-type.yaml + +inputs: + laika_image: + type: string + + host_capacity_policy_id: + type: string + default: DCAE_alex.Config_host_capacity_policy_id_value + + host_location_policy_id: + type: string + default: DCAE_alex.Config_host_location_policy_id_value + + db_server_policy_id: + type: string + default: DCAE_alex.Config_db_server_policy_id_value + +node_templates: + + host_capacity_policy: + type: dcae.nodes.policy + properties: + policy_id: { get_input: host_capacity_policy_id } + + host_location_policy: + type: dcae.nodes.policy + properties: + policy_id: { get_input: host_location_policy_id } + + db_server_policy: + type: dcae.nodes.policy + properties: + policy_id: { get_input: db_server_policy_id } + + laika-zero: + type: dcae.nodes.DockerContainerForComponents + properties: + service_component_type: + 'laika' + location_id: + 'rework-central' + service_id: + 'foo-service' + application_config: + some-param: "Lorem ipsum dolor sit amet" + downstream-laika: "{{ laika }}" + image: { get_input : laika_image } + docker_config: + healthcheck: + type: "http" + endpoint: "/health" + policy: + trigger_type: "docker" + script_path: "/bin/echo" + relationships: + # Link to downstream laika + - type: dcae.relationships.component_connected_to + target: laika-one + - type: dcae.relationships.component_contained_in + target: docker_host + - type: cloudify.relationships.depends_on + target: host_capacity_policy + - type: cloudify.relationships.depends_on + target: host_location_policy + interfaces: + cloudify.interfaces.lifecycle: + start: + inputs: + ports: + - "8080:5432" + envs: + SOME-ENV: "BAM" + max_wait: + 120 + stop: + inputs: + cleanup_image: + False + + laika-one: + type: dcae.nodes.DockerContainerForComponents + properties: + service_component_type: + 'laika' + application_config: + some-param: "Lorem ipsum dolor sit amet" + image: { get_input : laika_image } + # Trying without health check + relationships: + - type: dcae.relationships.component_contained_in + target: docker_host + - type: cloudify.relationships.depends_on + target: db_server_policy + interfaces: + cloudify.interfaces.lifecycle: + stop: + inputs: + cleanup_image: + False + + docker_host: + type: dcae.nodes.SelectedDockerHost + properties: + location_id: + 'rework-central' + docker_host_override: + 'component_dockerhost' diff --git a/docker/examples/blueprint-laika.yaml b/docker/examples/blueprint-laika.yaml index 98d27af..0db03b8 100644 --- a/docker/examples/blueprint-laika.yaml +++ b/docker/examples/blueprint-laika.yaml @@ -21,8 +21,8 @@ node_templates: 'laika' location_id: 'rework-central' - service_id: - 'foo-service' + service_id: + 'foo-service' application_config: some-param: "Lorem ipsum dolor sit amet" downstream-laika: "{{ laika }}" @@ -76,5 +76,5 @@ node_templates: properties: location_id: 'rework-central' - name_search: - 'platform_dockerhost' + docker_host_override: + 'component_dockerhost' diff --git a/docker/requirements.txt b/docker/requirements.txt index c76c229..bae15ce 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,3 +1,4 @@ # TODO: May need to add the following line # --extra-index-url <onap pypi url> -python-dockering==1.2.0 +python-dockering==1.3.0 +dcaepolicy==0.0.4 diff --git a/docker/setup.py b/docker/setup.py index 65ac0e9..128e90b 100644 --- a/docker/setup.py +++ b/docker/setup.py @@ -24,13 +24,14 @@ from setuptools import setup setup( name='dockerplugin', description='Cloudify plugin for applications run in Docker containers', - version="2.3.0+t.0.3", + version="2.4.0", author='Michael Hwang, Tommy Carpenter', packages=['dockerplugin'], zip_safe=False, install_requires=[ "python-consul>=0.6.0,<1.0.0", "python-dockering>=1.0.0,<2.0.0", - "uuid==1.30" + "uuid==1.30", + "dcaepolicy>=0.0.4" ] ) |