diff options
Diffstat (limited to 'python-dockering/dockering')
| -rw-r--r-- | python-dockering/dockering/__init__.py | 21 | ||||
| -rw-r--r-- | python-dockering/dockering/config_building.py | 269 | ||||
| -rw-r--r-- | python-dockering/dockering/core.py | 136 | ||||
| -rw-r--r-- | python-dockering/dockering/exceptions.py | 34 | ||||
| -rw-r--r-- | python-dockering/dockering/utils.py | 31 |
5 files changed, 491 insertions, 0 deletions
diff --git a/python-dockering/dockering/__init__.py b/python-dockering/dockering/__init__.py new file mode 100644 index 0000000..7c248d8 --- /dev/null +++ b/python-dockering/dockering/__init__.py @@ -0,0 +1,21 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from dockering.core import * +from dockering.config_building import * diff --git a/python-dockering/dockering/config_building.py b/python-dockering/dockering/config_building.py new file mode 100644 index 0000000..d8e3c84 --- /dev/null +++ b/python-dockering/dockering/config_building.py @@ -0,0 +1,269 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +""" +Abstraction in Docker container configuration +""" +from dockering import utils +from dockering.exceptions import DockerConstructionError + + +# +# Methods to build container envs +# + +def create_envs_healthcheck(docker_config, default_interval="15s", + default_timeout="1s"): + """Extract health check environment variables for Docker containers + + Parameters + ---------- + docker_config: dict where there's an entry called "healthcheck" + + Returns + ------- + dict of Docker envs for healthcheck + """ + # TODO: This has been shamefully lifted from the dcae-cli and should probably + # shared as a library. The unit tests are there. The difference is that + # there are defaults that are passed in here but the defaults should really + # come from the component spec definition. The issue is that nothing forwards + # those defaults. + + envs = dict() + hc = docker_config["healthcheck"] + + # NOTE: For the multiple port, schema scenario, you can explicitly set port + # to schema. For example if image EXPOSE 8080, SERVICE_8080_CHECK_HTTP works. + # https://github.com/gliderlabs/registrator/issues/311 + + if hc["type"] == "http": + envs["SERVICE_CHECK_HTTP"] = hc["endpoint"] + elif hc["type"] == "https": + # WATCH: HTTPS health checks don't work. Seems like Registrator bug. + # Submitted issue https://github.com/gliderlabs/registrator/issues/516 + envs["SERVICE_CHECK_HTTPS"] = hc["endpoint"] + utils.logger.warn("Https-based health checks may not work because Registrator issue #516") + elif hc["type"] == "script": + envs["SERVICE_CHECK_SCRIPT"] = hc["script"] + elif hc["type"] == "docker": + # Note this is only supported in the AT&T open source version of registrator + envs["SERVICE_CHECK_DOCKER_SCRIPT"] = hc["script"] + else: + # You should never get here but not having an else block feels weird + raise DockerConstructionError("Unexpected health check type: {0}".format(hc["type"])) + + envs["SERVICE_CHECK_INTERVAL"] = hc.get("interval", default_interval) + envs["SERVICE_CHECK_TIMEOUT"] = hc.get("timeout", default_timeout) + + return envs + + +def create_envs(service_component_name, *envs): + """Merge all environment variables maps + + Creates a complete environment variables map that is to be used for creating + the container. + + Args: + ----- + envs: Arbitrary list of dicts where each dict is of the structure: + + { + <environment variable name>: <environment variable value>, + <environment variable name>: <environment variable value>, + ... + } + + Returns: + -------- + Dict of all environment variable name to environment variable value + """ + master_envs = { "HOSTNAME": service_component_name, + # For Registrator to register with generated name and not the + # image name + "SERVICE_NAME": service_component_name } + for envs_map in envs: + master_envs.update(envs_map) + return master_envs + + +# +# Methods for volume bindings +# + +def _parse_volumes_param(volumes): + """Parse volumes details for Docker containers from blueprint + + Takes in a list of dicts that contains Docker volume info and + transforms them into docker-py compliant (unflattened) data structures. + Look for the `volumes` parameters under the `run` method on + [this page](https://docker-py.readthedocs.io/en/stable/containers.html) + + Args: + volumes (list): List of + + { + "host": { + "path": <target path on host> + }, + "container": { + "bind": <target path in container>, + "mode": <read/write> + } + } + + Returns: + dict of the form + + { + <target path on host>: { + "bind": <target path in container>, + "mode": <read/write> + } + } + + if volumes is None then returns None + """ + if volumes: + return dict([ (vol["host"]["path"], vol["container"]) for vol in volumes ]) + else: + return None + + +# +# Utility methods used to help build the inputs to create the host_config +# + +def add_host_config_params_volumes(volumes=None, host_config_params=None): + """Add volumes input params + + Args: + ----- + volumes (list): List of + + { + "host": { + "path": <target path on host> + }, + "container": { + "bind": <target path in container>, + "mode": <read/write> + } + } + + host_config_params (dict): Target dict to accumulate host config inputs + + Returns: + -------- + Updated host_config_params + """ +# TODO: USE parse_volumes_param here! + if host_config_params == None: + host_config_params = {} + + host_config_params["binds"] = _parse_volumes_param(volumes) + return host_config_params + +def add_host_config_params_ports(ports=None, host_config_params=None): + """Add ports input params + + Args: + ----- + ports (list): Each port mapping entry is of the form + + "<container ports>:<host port>" + + host_config_params (dict): Target dict to accumulate host config inputs + + Returns: + -------- + Updated host_config_params + """ + if host_config_params == None: + host_config_params = {} + + if ports: + ports = [ port.split(":") for port in ports ] + port_bindings = { port[0]: { "HostPort": port[1] } for port in ports } + host_config_params["port_bindings"] = port_bindings + host_config_params["publish_all_ports"] = False + else: + host_config_params["publish_all_ports"] = True + + return host_config_params + +def add_host_config_params_dns(docker_host, host_config_params=None): + """Add dns input params + + This is not a generic implementation. This method will setup dns with the + expectation that a local consul agent is running on the docker host and will + service the dns requests. + + Args: + ----- + docker_host (string): Docker host ip address which will be used as the dns server + host_config_params (dict): Target dict to accumulate host config inputs + + Returns: + -------- + Updated host_config_params + """ + if host_config_params == None: + host_config_params = {} + + host_config_params["dns"] = [docker_host] + host_config_params["dns_search"] = ["service.consul"] + host_config_params["extra_hosts"] = { "consul": docker_host } + return host_config_params + + +def create_container_config(client, image, envs, host_config_params, tty=False): + """Create docker container config + + Args: + ----- + envs (dict): dict of environment variables to pass into the docker containers. + Gets passed into docker-py.create_container call + host_config_params (dict): Dict of input parameters to the docker-py + "create_host_config" method call + """ + # This is the 1.10.6 approach to binding volumes + # http://docker-py.readthedocs.io/en/1.10.6/volumes.html + volumes = host_config_params.get("bind", None) + target_volumes = [ target["bind"] for target in volumes.values() ] \ + if volumes else None + + host_config = client.create_host_config(**host_config_params) + + if "port_bindings" in host_config_params: + # TODO: Use six for creating the list of ports - six.iterkeys + ports = host_config_params["port_bindings"].keys() + else: + ports = None + + command = "" # This is required... + config = client.create_container_config(image, command, detach=True, tty=tty, + host_config=host_config, ports=ports, + environment=envs, volumes=target_volumes) + + utils.logger.info("Docker container config: {0}".format(config)) + + return config + diff --git a/python-dockering/dockering/core.py b/python-dockering/dockering/core.py new file mode 100644 index 0000000..dcd5908 --- /dev/null +++ b/python-dockering/dockering/core.py @@ -0,0 +1,136 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import json +import docker +import requests +from dockering.exceptions import DockerError, DockerConnectionError +from dockering import config_building as cb +from dockering import utils + + +# TODO: Replace default for logins to source it from Consul..perhaps + +def create_client(hostname, port, reauth=False, logins=[]): + """Create Docker client + + Args: + ----- + reauth: (boolean) Forces reauthentication e.g. Docker login + """ + base_url = "tcp://{0}:{1}".format(hostname, port) + try: + client = docker.Client(base_url=base_url) + + for dcl in logins: + dcl["reauth"] = reauth + client.login(**dcl) + + return client + except requests.exceptions.ConnectionError as e: + raise DockerConnectionError(str(e)) + + +def create_container_using_config(client, service_component_name, container_config): + try: + image_name = container_config["Image"] + + if not client.images(image_name): + def parse_pull_response(response): + """Pull response is a giant string of JSON messages concatentated + by `\r\n`. This method returns back those messages in the form of + list of dicts.""" + # NOTE: There's a trailing `\r\n` so the last element is empty + # string. Remove that. + return list(map(json.loads, response.split("\r\n")[:-1])) + + def get_error_message(response): + """Attempts to pull out and return an error message from parsed + response if it exists else return None""" + return response[-1].get("error", None) + + # TODO: Implement this as verbose? + # for resp in client.pull(image, stream=True, decode=True): + response = parse_pull_response(client.pull(image_name)) + error_message = get_error_message(response) + + if error_message: + raise DockerError("Error pulling Docker image: {0}".format(error_message)) + else: + utils.logger.info("Pulled Docker image: {0}".format(image_name)) + + return client.create_container_from_config(container_config, + service_component_name) + except requests.exceptions.ConnectionError as e: + # This separates connection failures so that caller can decide what to do. + # Underlying errors this inspired were socket.errors that are sourced + # from http://www.virtsync.com/c-error-codes-include-errno + raise DockerConnectionError(str(e)) + except Exception as e: + raise DockerError(str(e)) + + +def create_container(client, image_name, service_component_name, envs, + host_config_params): + """Creates Docker container + + Args: + ----- + envs (dict): dict of environment variables to pass into the docker containers. + Gets passed into docker-py.create_container call + host_config_params (dict): Dict of input parameters to the docker-py + "create_host_config" method call + """ + config = cb.create_container_config(client, image_name, envs, host_config_params) + return create_container_using_config(client, service_component_name, config) + + +def start_container(client, container): + try: + # TODO: Have logic to inspect response and through NonRecoverableError + # when start fails. Docker-py docs don't quickly tell me what the + # response looks like. + response = client.start(container=container["Id"]) + utils.logger.info("Container started: {0}".format(container["Id"])) + + # TODO: Maybe check stats? + return container["Id"] + except Exception as e: + raise DockerError(str(e)) + + +def stop_then_remove_container(client, service_component_name): + try: + client.stop(service_component_name) + client.remove_container(service_component_name) + except docker.errors.NotFound as e: + raise DockerError("Container not found: {0}".format(service_component_name)) + except Exception as e: + raise DockerError(str(e)) + + +def remove_image(client, image_name): + """Remove the Docker image""" + try: + client.remove_image(image_name) + return True + except: + # Failure to remove image is not classified as terrible..for now + return False + diff --git a/python-dockering/dockering/exceptions.py b/python-dockering/dockering/exceptions.py new file mode 100644 index 0000000..62ea145 --- /dev/null +++ b/python-dockering/dockering/exceptions.py @@ -0,0 +1,34 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +""" +Library's exceptions +""" + +class DockerError(RuntimeError): + """General error""" + pass + +class DockerConnectionError(DockerError): + """Errors connecting to the Docker engine""" + pass + +class DockerConstructionError(DockerError): + """This class of error captures failures in trying to setup the container""" + pass diff --git a/python-dockering/dockering/utils.py b/python-dockering/dockering/utils.py new file mode 100644 index 0000000..e0f651e --- /dev/null +++ b/python-dockering/dockering/utils.py @@ -0,0 +1,31 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +""" +Utility module +""" +import logging + + +# Unified all logging through this single logger in order to easily monkeypatch +# this guy in the Cloudify docker plugin. I also tried monkeypatching a getter +# function that returns a logger but that didn't work. +# WATCH! The monkeypatching in the Cloudify plugin will not work if you import +# this logger with the following syntax: from dockering.utils import logger. +logger = logging.getLogger("dockering") |