diff options
| author |   Tommy Carpenter <tommy@research.att.com> | 2017-08-23 11:21:44 -0400 |
|---|---|---|
| committer | Tommy Carpenter <tommy@research.att.com> | 2017-08-23 12:32:45 -0400 |
| commit | 81b9ed7a3af1032caa86f316a21e9b2912bb1271 (patch) | |
| tree | fe3bb349f501619d3b4df4d4dbb07b9d72adc824 | |
| parent | 280af47e5342de6bb4657ad7b85adcc9395ace20 (diff) | |
Intiial utils push to ONAP
Issue-Id: DCAEGEN2-80
Change-Id: I1dc8f2d384b0af346fccb86e6767b7e3ca484078
Signed-off-by: Tommy Carpenter <tommy@research.att.com>
32 files changed, 1974 insertions, 0 deletions
diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..241d2ce --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.onap.org +port=29418 +project=dcaegen2/utils.git diff --git a/python-cbs-docker-client/.gitignore b/python-cbs-docker-client/.gitignore new file mode 100644 index 0000000..31c3000 --- /dev/null +++ b/python-cbs-docker-client/.gitignore @@ -0,0 +1,4 @@ +dist/ +.DS_Store +*.egg-info/ +*.pyc diff --git a/python-cbs-docker-client/MANIFEST.in b/python-cbs-docker-client/MANIFEST.in new file mode 100644 index 0000000..f9bd145 --- /dev/null +++ b/python-cbs-docker-client/MANIFEST.in @@ -0,0 +1 @@ +include requirements.txt diff --git a/python-cbs-docker-client/README.md b/python-cbs-docker-client/README.md new file mode 100644 index 0000000..3a323a1 --- /dev/null +++ b/python-cbs-docker-client/README.md @@ -0,0 +1,31 @@ +# Python CBS Docker Client + +Used for DCAE Dockerized microservices written in Python. Pulls your configuration from the config_binding_service. Expects that CONSUL_HOST and HOSTNAME are set as env variables, which is true in DCAE. + +# Client Usage + +## Development outside of Docker +To test your raw code without Docker, you will need to set the env variables CONSUL_HOST and HOSTNAME (name of your key to pull from) that are set in DCAEs Docker enviornment. +1. `CONSUL_HOST` is the hostname only of the Consul instance you are talking to +2. HOSTNAME is the name of your component in Consul + +## Usage in your code +``` +>>> from cbs_docker_client import client +>>> client.get_config() +``` + +# Installation + +## Via pip +``` +pip install --extra-index-url https://YOUR_NEXUS_PYPI_SERVER/simple cbs-docker-client +``` + +## Via requirements.txt +Add the following to your requirements.txt file +``` +--extra-index-url https://YOUR_NEXUS_PYPI_SERVER/simple +cbs-docker-client==0.0.1 +``` + diff --git a/python-cbs-docker-client/cbs_docker_client/__init__.py b/python-cbs-docker-client/cbs_docker_client/__init__.py new file mode 100644 index 0000000..9e81f65 --- /dev/null +++ b/python-cbs-docker-client/cbs_docker_client/__init__.py @@ -0,0 +1,19 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + diff --git a/python-cbs-docker-client/cbs_docker_client/client.py b/python-cbs-docker-client/cbs_docker_client/client.py new file mode 100644 index 0000000..4423995 --- /dev/null +++ b/python-cbs-docker-client/cbs_docker_client/client.py @@ -0,0 +1,85 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import json +import requests +import os +import logging + +root = logging.getLogger() +logger = root.getChild(__name__) + +def _get_uri_from_consul(consul_url, name): + """ + Call consul's catalog + TODO: currently assumes there is only one service with this HOSTNAME + """ + url = "{0}/v1/catalog/service/{1}".format(consul_url, name) + logger.debug("Trying to lookup service: {0}".format(url)) + res = requests.get(url) + try: + res.raise_for_status() + services = res.json() + return "http://{0}:{1}".format(services[0]["ServiceAddress"], services[0]["ServicePort"]) + except Exception as e: + logger.error("Exception occured when querying Consul: either could not hit {0} or no service registered. Error code: {1}, Error Text: {2}".format(url, res.status_code, res.text)) + return None + +def _get_envs(): + """ + Returns HOSTNAME, CONSUL_HOST, CONFIG_BINDING_SERVICE or crashes for caller to deal with + """ + HOSTNAME = os.environ["HOSTNAME"] + CONSUL_HOST = os.environ["CONSUL_HOST"] + return HOSTNAME, CONSUL_HOST + +#Public +def get_config(): + """ + This call does not raise an exception if Consul or the CBS cannot complete the request. + It logs an error and returns {} if the config is not bindable. + It could be a temporary network outage. Call me again later. + + It will raise an exception if the necessary env parameters were not set because that is irrecoverable. + This function is called in my /heatlhcheck, so this will be caught early. + """ + + config = {} + + HOSTNAME, CONSUL_HOST = _get_envs() + + #not sure how I as the component developer is supposed to know consul port + consul_url = "http://{0}:8500".format(CONSUL_HOST) + + #get the CBS URL. Would not need the following hoorahrah if we had DNS. + cbs_url = _get_uri_from_consul(consul_url, "config_binding_service") + if cbs_url is None: + logger.error("Cannot bind config at this time, cbs is unreachable") + else: + #get my config + my_config_endpoint = "{0}/service_component/{1}".format(cbs_url, HOSTNAME) + res = requests.get(my_config_endpoint) + try: + res.raise_for_status() + config = res.json() + logger.info("get_config returned the following configuration: {0}".format(json.dumps(config))) + except: + logger.error("in get_config, the config binding service endpoint {0} blew up on me. Error code: {1}, Error text: {2}".format(my_config_endpoint, res.status_code, res.text)) + return config + diff --git a/python-cbs-docker-client/requirements.txt b/python-cbs-docker-client/requirements.txt new file mode 100644 index 0000000..856c82c --- /dev/null +++ b/python-cbs-docker-client/requirements.txt @@ -0,0 +1 @@ +requests==2.18.3 diff --git a/python-cbs-docker-client/setup.py b/python-cbs-docker-client/setup.py new file mode 100644 index 0000000..7b8ebdb --- /dev/null +++ b/python-cbs-docker-client/setup.py @@ -0,0 +1,39 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import os +from setuptools import setup, find_packages +from pip.req import parse_requirements +from pip.download import PipSession + +install_reqs = parse_requirements("requirements.txt", session=PipSession()) +reqs = [str(ir.req) for ir in install_reqs] + +setup( + name = "cbs_docker_client", + description = "very lightweight client for a DCAE dockerized component to get it's config from the CBS", + version = "0.0.1", + packages=find_packages(), + author = "Tommy Carpenter", + author_email = "tommy at eh tee tee.com", + license = "", + keywords = "", + url = "", + install_requires=reqs +) diff --git a/python-discovery-client/.gitignore b/python-discovery-client/.gitignore new file mode 100644 index 0000000..1dbc687 --- /dev/null +++ b/python-discovery-client/.gitignore @@ -0,0 +1,62 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +#Ipython Notebook +.ipynb_checkpoints diff --git a/python-discovery-client/ChangeLog.md b/python-discovery-client/ChangeLog.md new file mode 100644 index 0000000..da45e86 --- /dev/null +++ b/python-discovery-client/ChangeLog.md @@ -0,0 +1,17 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [2.1.0] + +* Update `get_configuration` to use the config binding service and also use the `CONSUL_HOST` environment variable + +## [2.0.0] + +* Added public `resolve_name` method used to resolve names by looking up in Consul +* Changed `_resolve_name` to return lists over an arbitrary entry from the list +* Changed the `register_for_discovery` method to pass in service ip and avoid unimpressive auto ip discovery +* Changed setup.py to use abstract requirements to be a more friendly library diff --git a/python-discovery-client/LICENSE.txt b/python-discovery-client/LICENSE.txt new file mode 100644 index 0000000..cb8008a --- /dev/null +++ b/python-discovery-client/LICENSE.txt @@ -0,0 +1,32 @@ +============LICENSE_START======================================================= +org.onap.dcae +================================================================================ +Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +================================================================================ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +============LICENSE_END========================================================= + +ECOMP is a trademark and service mark of AT&T Intellectual Property. + + +Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +=================================================================== +Licensed under the Creative Commons License, Attribution 4.0 Intl. (the "License"); +you may not use this documentation except in compliance with the License. +You may obtain a copy of the License at + https://creativecommons.org/licenses/by/4.0/ +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/python-discovery-client/MANIFEST.in b/python-discovery-client/MANIFEST.in new file mode 100644 index 0000000..f9bd145 --- /dev/null +++ b/python-discovery-client/MANIFEST.in @@ -0,0 +1 @@ +include requirements.txt diff --git a/python-discovery-client/README.md b/python-discovery-client/README.md new file mode 100644 index 0000000..b022f5d --- /dev/null +++ b/python-discovery-client/README.md @@ -0,0 +1,2 @@ +# python-discovery-client +Python client to be used by service components for discovery diff --git a/python-discovery-client/discovery_client/__init__.py b/python-discovery-client/discovery_client/__init__.py new file mode 100644 index 0000000..9e0358a --- /dev/null +++ b/python-discovery-client/discovery_client/__init__.py @@ -0,0 +1,21 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from .discovery import get_service_name, get_configuration, \ + register_for_discovery, resolve_name diff --git a/python-discovery-client/discovery_client/discovery.py b/python-discovery-client/discovery_client/discovery.py new file mode 100644 index 0000000..180e933 --- /dev/null +++ b/python-discovery-client/discovery_client/discovery.py @@ -0,0 +1,368 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import time, json, os, re, logging +from itertools import chain +from functools import partial +import requests +import consul +import six +from discovery_client import util + + +_logger = util.get_logger(__name__) + +class DiscoveryInitError(RuntimeError): + pass + +class DiscoveryRegistrationError(RuntimeError): + pass + +class DiscoveryResolvingNameError(RuntimeError): + pass + + +##### +# Consul calls for services +##### + +def _get_configuration_from_consul(consul_handle, service_name): + index = None + while True: + index, data = consul_handle.kv.get(service_name, index=index) + + if data: + return json.loads(data["Value"].decode("utf-8")) + else: + _logger.warn("No configuration found for {0}. Try again in a bit." + .format(service_name)) + time.sleep(5) + +def _get_relationships_from_consul(consul_handle, service_name): + """Fetch the relationship information from Consul for a service by service + name. Returns a list of service names.""" + index = None + rel_key = "{0}:rel".format(service_name) + while True: + index, data = consul_handle.kv.get(rel_key, index=index) + + if data: + return json.loads(data["Value"].decode("utf-8")) + else: + _logger.warn("No relationships found for {0}. Try again in a bit." + .format(service_name)) + time.sleep(5) + +def _lookup_with_consul(consul_handle, service_name, max_attempts=0): + num_attempts = 1 + + while True: + index, results = consul_handle.catalog.service(service_name) + + if results: + return results + else: + num_attempts += 1 + + if max_attempts > 0 and max_attempts < num_attempts: + return None + + _logger.warn("Service not found {0}. Trying again in a bit." + .format(service_name)) + time.sleep(5) + +def _register_with_consul(consul_handle, service_name, service_ip, service_port, + health_endpoint): + # https://www.consul.io/docs/agent/http/agent.html#agent_service_register + # Note: Unhealthy services should not return in queries i.e. + # dig @127.0.0.1 -p 8600 foo.service.consul + health_url = "http://{0}:{1}/{2}".format(service_ip, service_port, health_endpoint) + return consul_handle.agent.service.register(service_name, address=service_ip, + port=service_port, check= { "HTTP": health_url, "Interval": "5s" }) + +##### +# Config binding service call +##### + +def _get_configuration_resolved_from_cbs(consul_handle, service_name): + """ + This is what a minimal python client library that wraps the CBS would look like. + POSSIBLE TODO: break this out into pypi repo + + This call does not raise an exception if Consul or the CBS cannot complete the request. + It logs an error and returns {} if the config is not bindable. + It could be a temporary network outage. Call me again later. + + It will raise an exception if the necessary env parameters were not set because that is irrecoverable. + This function is called in my /heatlhcheck, so this will be caught early. + """ + config = {} + + results = _lookup_with_consul(consul_handle, "config_binding_service", + max_attempts=5) + + if results is None: + logger.error("Cannot bind config at this time, cbs is unreachable") + else: + cbs_hostname = results[0]["ServiceAddress"] + cbs_port = results[0]["ServicePort"] + cbs_url = "http://{hostname}:{port}".format(hostname=cbs_hostname, port=cbs_port) + + #get my config + my_config_endpoint = "{0}/service_component/{1}".format(cbs_url, + service_name) + res = requests.get(my_config_endpoint) + try: + res.raise_for_status() + config = res.json() + _logger.info("get_config returned the following configuration: {0}".format(json.dumps(config))) + except: + _logger.error("in get_config, the config binding service endpoint {0} blew up on me. Error code: {1}, Error text: {2}".format(my_config_endpoint, res.status_code, res.text)) + return config + +##### +# Functionality for putting together service's configuration +##### + +def _get_connection_types(config): + """Get all the connection types for a given configuration json + + Crawls through the entire config dict recursively and returns the entries + that have been identified as service connections in the form of a list of tuples - + + [(config key, component type), ..] + + where "config key" is a compound key in the form of a tuple. Each entry in + the compound key is a key to a level within the json data structure.""" + def grab_component_type(v): + # To support Python2, unicode strings are not type `str`. Specifically, + # the config string values from Consul maybe encoded to utf-8 so better + # be prepared. + if isinstance(v, six.string_types): + # Regex matches on strings like "{{foo}}" and "{{ BAR }}" and + # extracts the alphanumeric string inside the parantheses. + result = re.match("^{{\s*([-_.\w]*)\s*}}", v) + return result.group(1) if result else None + + def crawl(config, parent_key=()): + if isinstance(config, dict): + rels = [ crawl(value, parent_key + (key, )) + for key, value in config.items() ] + rels = chain(*rels) + elif isinstance(config, list): + rels = [ crawl(config[index], parent_key + (index, )) + for index in range(0, len(config)) ] + rels = chain(*rels) + else: + rels = [(parent_key, grab_component_type(config))] + + # Filter out the entries with Nones + rels = [(key, rel) for key, rel in rels if rel] + return rels + + return crawl(config) + +def _has_connections(config): + return True if _get_connection_types(config) else False + +def _resolve_connection_types(service_name, connection_types, relationships): + + def find_match(connection_type): + ret_list = [] + for rel in relationships: + if connection_type in rel: + ret_list.append(rel) + return ret_list + + return [ (key, find_match(connection_type)) + for key, connection_type in connection_types ] + +def _resolve_name(lookup_func, service_name): + """Resolves the service component name to detailed connection information + + Currently this is grouped into two ways: + 1. CDAP applications take a two step approach - call Consul then call the + CDAP broker + 2. All other applications just call Consul to get IP and port + + Args: + ---- + lookup_func: fn(string) -> list of dicts + The function should return a list of dicts that have "ServiceAddress" and + "ServicePort" key value entries + service_name: (string) service name to lookup + + Return depends upon the connection type: + 1. CDAP applications return a dict + 2. All other applications return a string + """ + def handle_result(result): + ip = result["ServiceAddress"] + port = result["ServicePort"] + + if not (ip and port): + raise DiscoveryResolvingNameError( + "Failed to resolve name for {0}: ip, port not set".format(service_name)) + + # TODO: Need a better way to identify CDAP apps. Really need to make this + # better. + if "platform-" in service_name: + return "{0}:{1}".format(ip, port) + elif "cdap" in service_name: + redirectish_url = "http://{0}:{1}/application/{2}".format(ip, port, + service_name) + + r = requests.get(redirectish_url) + r.raise_for_status() + details = r.json() + # Pick out the details to expose to the component developers + return { key: details[key] + for key in ["connectionurl", "serviceendpoints"] } + else: + return "{0}:{1}".format(ip, port) + + try: + results = lookup_func(service_name) + return [ handle_result(result) for result in results ] + except Exception as e: + raise DiscoveryResolvingNameError( + "Failed to resolve name for {0}: {1}".format(service_name, e)) + +def _resolve_configuration_dict(ch, service_name, config): + """ + Helper used by both resolve_configuration_dict and get_configuration + """ + if _has_connections(config): + rels = _get_relationships_from_consul(ch, service_name) + connection_types = _get_connection_types(config) + connection_names = _resolve_connection_types(service_name, connection_types, rels) + # NOTE: The hardcoded use of the first element. This is to keep things backwards + # compatible since resolve name now returns a list. + for key, conn in [(key, [_resolve_name(partial(_lookup_with_consul, ch), name)[0] for name in names]) for key, names in connection_names]: + config = util.update_json(config, key, conn) + + _logger.info("Generated config: {0}".format(config)) + return config + +##### +# Public calls +##### + +def get_consul_hostname(consul_hostname_override=None): + """Get the Consul hostname""" + try: + return consul_hostname_override \ + if consul_hostname_override else os.environ["CONSUL_HOST"] + except: + raise DiscoveryInitError("CONSUL_HOST variable has not been set!") + +def get_service_name(): + """Get the full service name + + This is expected to be given from whatever entity is starting this service + and given by an environment variable called "HOSTNAME".""" + try: + return os.environ["HOSTNAME"] + except: + raise DiscoveryInitError("HOSTNAME variable has not been set!") + + +def resolve_name(consul_host, service_name, max_attempts=3): + """Resolve the service name + + Do a service discovery lookup from Consul and return back the detailed connection + information. + + Returns: + -------- + For CDAP apps, returns a dict. All others a string with the format "<ip>:<port>" + """ + ch = consul.Consul(host=consul_host) + lookup_func = partial(_lookup_with_consul, ch, max_attempts=max_attempts) + return _resolve_name(lookup_func, service_name) + + +def resolve_configuration_dict(consul_host, service_name, config): + """ + Utility method for taking a given service_name, and config dict, and resolving it + """ + ch = consul.Consul(host=consul_host) + return _resolve_configuration_dict(ch, service_name, config) + + +def get_configuration(override_consul_hostname=None, override_service_name=None, + from_cbs=True): + """Provides this service component's configuration information fully resolved + + This method can either resolve the configuration locally here or make a + remote call to the config binding service. The default is to use the config + binding service. + + Args: + ----- + override_consul_hostname (string): Consul hostname to use rather than the one + set by the environment variable CONSUL_HOST + override_service_name (string): Use this name over the name set on the + HOSTNAME environment variable. Default is None. + from_cbs (boolean): True (default) means use the config binding service otherwise + set to False to have the config pulled and resolved by this library + + Returns the fully resolved service component configuration as a dict + """ + # Get config, bootstrap + consul_hostname = get_consul_hostname(override_consul_hostname) + # NOTE: We use the default port 8500 + ch = consul.Consul(host=consul_hostname) + service_name = override_service_name if override_service_name else get_service_name() + _logger.info("service name: {0}".format(service_name)) + + if from_cbs: + return _get_configuration_resolved_from_cbs(ch, service_name) + else: + # The following will happen: + # + # 1. Fetching the configuration by service component name from Consul + # 2. Fetching the relationships for this service component by service component + # name + # 3. Pick out the connection types from the templetized fields in the configuration + # 4. Resolve the connection types with connection names using the step #2 + # information + # 5. Resolve the connection names with the actual connection via queries to + # Consul using the connection name + config = _get_configuration_from_consul(ch, service_name) + return _resolve_configuration_dict(ch, service_name, config) + + +def register_for_discovery(consul_host, service_ip, service_port): + """Register the service component for service discovery + + This is required in order for other services to "discover" you so that you + can service their requests. + + NOTE: Applications may not need to make this call depending upon if the + environment is using Registrator. + """ + ch = consul.Consul(host=consul_host) + service_name = get_service_name() + + if _register_with_consul(ch, service_name, service_ip, service_port, "health"): + _logger.info("Registered to consul: {0}".format(service_name)) + else: + _logger.error("Failed to register to consul: {0}".format(service_name)) + raise DiscoveryRegistrationError() diff --git a/python-discovery-client/discovery_client/util.py b/python-discovery-client/discovery_client/util.py new file mode 100644 index 0000000..b59647a --- /dev/null +++ b/python-discovery-client/discovery_client/util.py @@ -0,0 +1,78 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import collections +import logging, sys +import six + +##### +# Module contains utility methods +##### + +def update_json(src, key, value): + """Updates a nested JSON value + + This method does a recursive lookup for a value given a compound key and then + replaces that value with the passed in new value. + + For example, given a src json { "a": [ { "aa": 1 }, "foo" ], "b": "2 } and a + key ("a", 0, "aa"), the value parameter would replace 1. + + :param src: json to update + :type src: dict or list + :param key: compound key used to lookup + :type key: tuple + :param value: new value used to replace + :type value: object + + :return: updated json + """ + if key: + src[key[0]] = update_json(src[key[0]], key[1:], value) + else: + # We've found the value we want to replace regardless of whether or not + # the object we are replacing is another copmlicated data structure. + src = value + return src + +def _has_handlers(logger): + """Check if logger has handlers""" + if six.PY3: + return logger.hasHandlers() + else: + # TODO: Not sure how to check if a handler has already been attached + # WATCH: Downside is lines get printed multiple times + return False + +def get_logger(name, level=logging.INFO): + """Get a logger with sensible defaults + + This method returns a logger from logging by name that has been set with sensible + defaults if the logger hasn't already been setup with any handlers. The + default handler is a stream handler to stdout. + """ + logger = logging.getLogger(name) + + if not _has_handlers(logger): + # No handlers attached which means logging hasn't been setup. Set + # "sensible" defaults which means stdout, INFO + logger.setLevel(level) + logger.addHandler(logging.StreamHandler(stream=sys.stdout)) + + return logger diff --git a/python-discovery-client/requirements.txt b/python-discovery-client/requirements.txt new file mode 100644 index 0000000..f79aa82 --- /dev/null +++ b/python-discovery-client/requirements.txt @@ -0,0 +1,3 @@ +python-consul==0.6.1 +requests==2.11.1 +six==1.10.0 diff --git a/python-discovery-client/setup.py b/python-discovery-client/setup.py new file mode 100644 index 0000000..2575358 --- /dev/null +++ b/python-discovery-client/setup.py @@ -0,0 +1,35 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from setuptools import setup, find_packages +from pip.req import parse_requirements +from pip.download import PipSession + +setup( + name = "python-discovery-client", + version = "2.1.0", + packages = find_packages(), + author = "Michael Hwang", + email="dcae@lists.openecomp.org", + description = ("Python client to be used by service components for discovery"), + install_requires = [ + 'python-consul>=0.6.0,<1.0.0', + 'requests>=2.11.0,<3.0.0', + 'six>=1.10.0,<2.0.0'] + ) diff --git a/python-discovery-client/tests/test_discovery.py b/python-discovery-client/tests/test_discovery.py new file mode 100644 index 0000000..00f1b5e --- /dev/null +++ b/python-discovery-client/tests/test_discovery.py @@ -0,0 +1,253 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import json, logging +# http://stackoverflow.com/questions/9623114/check-if-two-unordered-lists-are-equal +from collections import Counter +from functools import partial +import pytest +import requests +from discovery_client import discovery as dis + + +def test_get_connection_types(): + config = { "x": "say something", "y": 123, "z": "{{some-analytics}}" } + expected = [(("z", ), "some-analytics"), ] + actual = dis._get_connection_types(config) + assert Counter(expected) == Counter(actual) + + # Whitespaces ok + config = { "x": "say something", "y": 123, "z": "{{ some-analytics }}" } + expected = [(("z", ), "some-analytics"), ] + actual = dis._get_connection_types(config) + assert Counter(expected) == Counter(actual) + + # Paul wanted the ability to include version so match on more than just one + # subfield + config = { "x": "say something", "y": 123, "z": "{{1-0-0.some-analytics}}" } + expected = [(("z", ), "1-0-0.some-analytics"), ] + actual = dis._get_connection_types(config) + assert Counter(expected) == Counter(actual) + + # Need double parantheses + config = { "x": "say something", "y": 123, "z": "{some-analytics}" } + actual = dis._get_connection_types(config) + assert Counter([]) == Counter(actual) + + # Nested in dict dict + config = { "x": "say something", "y": 123, + "z": { "aa": { "bbb": "{{some-analytics}}" } } } + expected = [(("z", "aa", "bbb"), "some-analytics"), ] + actual = dis._get_connection_types(config) + assert Counter(expected) == Counter(actual) + + # Nested in list dict + config = { "x": "say something", "y": 123, + "z": [ "no-op", { "bbb": "{{some-analytics}}" } ] } + expected = [(("z", 1, "bbb"), "some-analytics"), ] + actual = dis._get_connection_types(config) + assert Counter(expected) == Counter(actual) + + # Force strings to be unicode, test for Python2 compatibility + config = { "x": "say something".decode("utf-8"), "y": 123, + "z": "{{some-analytics}}".decode("utf-8") } + expected = [(("z", ), "some-analytics"), ] + actual = dis._get_connection_types(config) + assert Counter(expected) == Counter(actual) + + +def test_resolve_connection_types(): + upstream = "b243b0b8-8a24-4f88-add7-9b530c578149.laika.foobar.rework-central.dcae.ecomp.com" + downstream = "839b0b31-f13d-4bfc-9adf-450d34071304.laika.foobar.rework-central.dcae.ecomp.com" + + connection_types = [("downstream-laika", "laika"),] + relationships = [downstream] + expected = [("downstream-laika", [downstream])] + actual = dis._resolve_connection_types(upstream, connection_types, relationships) + assert sorted(actual) == sorted(expected) + + # NOTE: Removed test that tested the scenario where the name stems don't + # match up. This name stem matching was causing grief to others so lifted the + # constraint. + + +def test_resolve_name_for_platform(): + def fake_lookup(fixture, service_name): + if service_name == fixture["ServiceName"]: + return [fixture] + + # Good case. Grabbed from Consul call + fixture = { 'Node': 'agent-one', 'ModifyIndex': 2892, 'Address': '127.0.0.1', + 'ServiceName': 'e9526e08-f9b8-42c4-99a3-443cd4deeac1.platform-laika.foobar.rework-central.dcae.ecomp.com', + 'ServicePort': 12708, 'CreateIndex': 2825, 'ServiceAddress': '196.207.143.67', + 'ServiceTags': [], 'ServiceEnableTagOverride': False, + 'ServiceID': 'e9526e08-f9b8-42c4-99a3-443cd4deeac1.platform-laika.foobar.rework-central.dcae.ecomp.com'} + + expected = ["{0}:{1}".format(fixture["ServiceAddress"], + fixture["ServicePort"])] + assert dis._resolve_name(partial(fake_lookup, fixture), fixture["ServiceName"]) == expected + + # Fail case. When Registrator is misconfigured and ServiceAddress is not set + fixture = { 'Node': 'agent-one', 'ModifyIndex': 2892, 'Address': '127.0.0.1', + 'ServiceName': 'e9526e08-f9b8-42c4-99a3-443cd4deeac1.platform-laika.foobar.rework-central.dcae.ecomp.com', + 'ServicePort': 12708, 'CreateIndex': 2825, 'ServiceAddress': '', + 'ServiceTags': [], 'ServiceEnableTagOverride': False, + 'ServiceID': 'e9526e08-f9b8-42c4-99a3-443cd4deeac1.platform-laika.foobar.rework-central.dcae.ecomp.com'} + + with pytest.raises(dis.DiscoveryResolvingNameError): + dis._resolve_name(partial(fake_lookup, fixture), fixture["ServiceName"]) + + # Fail case. When lookup just blows up for some reason + def fake_lookup_blows(service_name): + raise RuntimeError("Thar she blows") + + with pytest.raises(dis.DiscoveryResolvingNameError): + dis._resolve_name(fake_lookup_blows, fixture["ServiceName"]) + + +def test_resolve_name_for_docker(): + def fake_lookup(fixture, service_name): + if service_name == fixture["ServiceName"]: + return [fixture] + + # Good case. Grabbed from Consul call + fixture = { 'Node': 'agent-one', 'ModifyIndex': 2892, 'Address': '127.0.0.1', + 'ServiceName': 'e9526e08-f9b8-42c4-99a3-443cd4deeac1.laika.foobar.rework-central.dcae.ecomp.com', + 'ServicePort': 12708, 'CreateIndex': 2825, 'ServiceAddress': '196.207.143.67', + 'ServiceTags': [], 'ServiceEnableTagOverride': False, + 'ServiceID': 'e9526e08-f9b8-42c4-99a3-443cd4deeac1.laika.foobar.rework-central.dcae.ecomp.com'} + + expected = ["{0}:{1}".format(fixture["ServiceAddress"], + fixture["ServicePort"])] + assert dis._resolve_name(partial(fake_lookup, fixture), fixture["ServiceName"]) == expected + + # Fail case. When Registrator is misconfigured and ServiceAddress is not set + fixture = { 'Node': 'agent-one', 'ModifyIndex': 2892, 'Address': '127.0.0.1', + 'ServiceName': 'e9526e08-f9b8-42c4-99a3-443cd4deeac1.laika.foobar.rework-central.dcae.ecomp.com', + 'ServicePort': 12708, 'CreateIndex': 2825, 'ServiceAddress': '', + 'ServiceTags': [], 'ServiceEnableTagOverride': False, + 'ServiceID': 'e9526e08-f9b8-42c4-99a3-443cd4deeac1.laika.foobar.rework-central.dcae.ecomp.com'} + + with pytest.raises(dis.DiscoveryResolvingNameError): + dis._resolve_name(partial(fake_lookup, fixture), fixture["ServiceName"]) + + # Fail case. When lookup just blows up for some reason + def fake_lookup_blows(service_name): + raise RuntimeError("Thar she blows") + + with pytest.raises(dis.DiscoveryResolvingNameError): + dis._resolve_name(fake_lookup_blows, fixture["ServiceName"]) + + +def test_resolve_name_for_cdap(monkeypatch): + def fake_lookup(fixture, service_name): + if service_name == fixture["ServiceName"]: + return [fixture] + + # Good case. Handle CDAP apps + fixture = { + "Node":"agent-one", "Address":"10.170.2.17", + "ServiceID": "00b6210b71e445cdaadf76e620ebffcfhelloworldcdapappfoobardcaereworkdcaeecompcom", + "ServiceName": "00b6210b71e445cdaadf76e620ebffcfhelloworldcdapappfoobardcaereworkdcaeecompcom", + "ServiceTags":[], + "ServiceAddress": "196.207.143.116", + "ServicePort": 7777, "ServiceEnableTagOverride": False, "CreateIndex": 144733, "ModifyIndex":145169 } + + class FakeRequestsResponse(object): + def __init__(self, url, broker_json): + self.url = url + self.broker_json = broker_json + + def raise_for_status(self): + expected_broker_url = "http://{0}:{1}/application/{2}".format( + fixture["ServiceAddress"], fixture["ServicePort"], + fixture["ServiceName"]) + if self.url == expected_broker_url: + return True + else: + raise RuntimeError("Mismatching address") + + def json(self): + return self.broker_json + + # Simulate the call to the CDAP broker + broker_json = { + "appname":"00b6210b71e445cdaadf76e620ebffcfhelloworldcdapappfoobardcaereworkdcaeecompcom", + "healthcheckurl":"http://196.207.143.116:7777/application/00b6210b71e445cdaadf76e620ebffcfhelloworldcdapappfoobardcaereworkdcaeecompcom/healthcheck", + "metricsurl":"http://196.207.143.116:7777/application/00b6210b71e445cdaadf76e620ebffcfhelloworldcdapappfoobardcaereworkdcaeecompcom/metrics", + "url":"http://196.207.143.116:7777/application/00b6210b71e445cdaadf76e620ebffcfhelloworldcdapappfoobardcaereworkdcaeecompcom", + "connectionurl":"http://196.207.160.159:10000/v3/namespaces/default/streams/foo", + "serviceendpoints": "something" } + + monkeypatch.setattr(requests, "get", lambda url: FakeRequestsResponse(url, broker_json)) + + expected = [{ key: broker_json[key] + for key in ["connectionurl", "serviceendpoints"] }] + assert dis._resolve_name(partial(fake_lookup, fixture), fixture["ServiceName"]) == expected + + +def test_resolve_configuration_dict(monkeypatch): + service_name = "123.current-node-type.some-service.some-location.com" + target_service_name = "456.target-node-type.some-service.some-location.com" + + # Fake the Consul calls + + def fake_get_relationship(ch, service_name): + return [ target_service_name ] + + monkeypatch.setattr(dis, "_get_relationships_from_consul", + fake_get_relationship) + + fixture = [{ 'Node': 'agent-one', 'ModifyIndex': 2892, 'Address': '127.0.0.1', + 'ServiceName': target_service_name, + 'ServicePort': 12708, 'CreateIndex': 2825, 'ServiceAddress': '196.207.143.67', + 'ServiceTags': [], 'ServiceEnableTagOverride': False, + 'ServiceID': target_service_name }] + + def fake_lookup(ch, service_name): + return fixture + + monkeypatch.setattr(dis, "_lookup_with_consul", fake_lookup) + + # Simple config case + test_config = { "target-node": "{{ target-node-type }}", "other-param": 123 } + + expected = dict(test_config) + expected["target-node"] = ["196.207.143.67:12708"] + actual = dis._resolve_configuration_dict(None, service_name, test_config) + assert Counter(actual) == Counter(expected) + + # Nested config case + test_config = { "output_formats": { "target-node": "{{ target-node-type }}" }, + "other-param": 123 } + + expected = dict(test_config) + expected["output_formats"]["target-node"] = "196.207.143.67:12708" + actual = dis._resolve_configuration_dict(None, service_name, test_config) + assert Counter(actual) == Counter(expected) + + +def test_get_consul_host(monkeypatch): + with pytest.raises(dis.DiscoveryInitError): + dis.get_consul_hostname() + + monkeypatch.setenv("CONSUL_HOST", "i-am-consul-host") + assert "i-am-consul-host" == dis.get_consul_hostname() + + assert "no-i-am" == dis.get_consul_hostname("no-i-am") diff --git a/python-discovery-client/tests/test_util.py b/python-discovery-client/tests/test_util.py new file mode 100644 index 0000000..8256359 --- /dev/null +++ b/python-discovery-client/tests/test_util.py @@ -0,0 +1,59 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from collections import Counter +from discovery_client import util + + +def test_update_json(): + # Simple + test_json = { "a": "{ funk }", "b": "spring" } + test_key = ("a", ) + test_value = "funk is alive" + expected = dict(test_json) + expected["a"] = test_value + actual = util.update_json(test_json, test_key, test_value) + assert Counter(expected) == Counter(actual) + + # Nothing to replace, key is empty which translates to repalce the entire + # json + test_key = () + expected = test_value + actual = util.update_json(test_json, test_key, test_value) + assert Counter(expected) == Counter(actual) + + # Nested in dicts + test_json = { "a": { "aa": { "aaa": "{ funk }", "bbb": "fall" }, + "bb": "summer" }, "b": "spring" } + test_key = ("a", "aa", "aaa") + test_value = "funk is alive" + expected = dict(test_json) + expected["a"]["aa"]["aaa"] = test_value + actual = util.update_json(test_json, test_key, test_value) + assert Counter(expected) == Counter(actual) + + # Nested in dict list + test_json = { "a": { "aa": [ 123, { "aaa": "{ funk }", "bbb": "fall" } ], + "bb": "summer" }, "b": "spring" } + test_key = ("a", "aa", 1, "aaa") + test_value = "funk is alive" + expected = dict(test_json) + expected["a"]["aa"][1]["aaa"] = test_value + actual = util.update_json(test_json, test_key, test_value) + assert Counter(expected) == Counter(actual) diff --git a/python-dockering/.gitignore b/python-dockering/.gitignore new file mode 100644 index 0000000..d11997c --- /dev/null +++ b/python-dockering/.gitignore @@ -0,0 +1,67 @@ +.cloudify +*.swp +*.swn +*.swo +.DS_Store +.project +.pydevproject +venv + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ diff --git a/python-dockering/ChangeLog.md b/python-dockering/ChangeLog.md new file mode 100644 index 0000000..67ae4f8 --- /dev/null +++ b/python-dockering/ChangeLog.md @@ -0,0 +1,11 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [1.2.0] + +* Add the ability to force reauthentication for Docker login +* Handle connection errors for Docker login diff --git a/python-dockering/LICENSE.txt b/python-dockering/LICENSE.txt new file mode 100644 index 0000000..cb8008a --- /dev/null +++ b/python-dockering/LICENSE.txt @@ -0,0 +1,32 @@ +============LICENSE_START======================================================= +org.onap.dcae +================================================================================ +Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +================================================================================ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +============LICENSE_END========================================================= + +ECOMP is a trademark and service mark of AT&T Intellectual Property. + + +Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +=================================================================== +Licensed under the Creative Commons License, Attribution 4.0 Intl. (the "License"); +you may not use this documentation except in compliance with the License. +You may obtain a copy of the License at + https://creativecommons.org/licenses/by/4.0/ +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/python-dockering/README.md b/python-dockering/README.md new file mode 100644 index 0000000..fd8d436 --- /dev/null +++ b/python-dockering/README.md @@ -0,0 +1,3 @@ +# python-dockering + +Library used to manage Docker containers in DCAE. diff --git a/python-dockering/dockering/__init__.py b/python-dockering/dockering/__init__.py new file mode 100644 index 0000000..7c248d8 --- /dev/null +++ b/python-dockering/dockering/__init__.py @@ -0,0 +1,21 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from dockering.core import * +from dockering.config_building import * diff --git a/python-dockering/dockering/config_building.py b/python-dockering/dockering/config_building.py new file mode 100644 index 0000000..d8e3c84 --- /dev/null +++ b/python-dockering/dockering/config_building.py @@ -0,0 +1,269 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +""" +Abstraction in Docker container configuration +""" +from dockering import utils +from dockering.exceptions import DockerConstructionError + + +# +# Methods to build container envs +# + +def create_envs_healthcheck(docker_config, default_interval="15s", + default_timeout="1s"): + """Extract health check environment variables for Docker containers + + Parameters + ---------- + docker_config: dict where there's an entry called "healthcheck" + + Returns + ------- + dict of Docker envs for healthcheck + """ + # TODO: This has been shamefully lifted from the dcae-cli and should probably + # shared as a library. The unit tests are there. The difference is that + # there are defaults that are passed in here but the defaults should really + # come from the component spec definition. The issue is that nothing forwards + # those defaults. + + envs = dict() + hc = docker_config["healthcheck"] + + # NOTE: For the multiple port, schema scenario, you can explicitly set port + # to schema. For example if image EXPOSE 8080, SERVICE_8080_CHECK_HTTP works. + # https://github.com/gliderlabs/registrator/issues/311 + + if hc["type"] == "http": + envs["SERVICE_CHECK_HTTP"] = hc["endpoint"] + elif hc["type"] == "https": + # WATCH: HTTPS health checks don't work. Seems like Registrator bug. + # Submitted issue https://github.com/gliderlabs/registrator/issues/516 + envs["SERVICE_CHECK_HTTPS"] = hc["endpoint"] + utils.logger.warn("Https-based health checks may not work because Registrator issue #516") + elif hc["type"] == "script": + envs["SERVICE_CHECK_SCRIPT"] = hc["script"] + elif hc["type"] == "docker": + # Note this is only supported in the AT&T open source version of registrator + envs["SERVICE_CHECK_DOCKER_SCRIPT"] = hc["script"] + else: + # You should never get here but not having an else block feels weird + raise DockerConstructionError("Unexpected health check type: {0}".format(hc["type"])) + + envs["SERVICE_CHECK_INTERVAL"] = hc.get("interval", default_interval) + envs["SERVICE_CHECK_TIMEOUT"] = hc.get("timeout", default_timeout) + + return envs + + +def create_envs(service_component_name, *envs): + """Merge all environment variables maps + + Creates a complete environment variables map that is to be used for creating + the container. + + Args: + ----- + envs: Arbitrary list of dicts where each dict is of the structure: + + { + <environment variable name>: <environment variable value>, + <environment variable name>: <environment variable value>, + ... + } + + Returns: + -------- + Dict of all environment variable name to environment variable value + """ + master_envs = { "HOSTNAME": service_component_name, + # For Registrator to register with generated name and not the + # image name + "SERVICE_NAME": service_component_name } + for envs_map in envs: + master_envs.update(envs_map) + return master_envs + + +# +# Methods for volume bindings +# + +def _parse_volumes_param(volumes): + """Parse volumes details for Docker containers from blueprint + + Takes in a list of dicts that contains Docker volume info and + transforms them into docker-py compliant (unflattened) data structures. + Look for the `volumes` parameters under the `run` method on + [this page](https://docker-py.readthedocs.io/en/stable/containers.html) + + Args: + volumes (list): List of + + { + "host": { + "path": <target path on host> + }, + "container": { + "bind": <target path in container>, + "mode": <read/write> + } + } + + Returns: + dict of the form + + { + <target path on host>: { + "bind": <target path in container>, + "mode": <read/write> + } + } + + if volumes is None then returns None + """ + if volumes: + return dict([ (vol["host"]["path"], vol["container"]) for vol in volumes ]) + else: + return None + + +# +# Utility methods used to help build the inputs to create the host_config +# + +def add_host_config_params_volumes(volumes=None, host_config_params=None): + """Add volumes input params + + Args: + ----- + volumes (list): List of + + { + "host": { + "path": <target path on host> + }, + "container": { + "bind": <target path in container>, + "mode": <read/write> + } + } + + host_config_params (dict): Target dict to accumulate host config inputs + + Returns: + -------- + Updated host_config_params + """ +# TODO: USE parse_volumes_param here! + if host_config_params == None: + host_config_params = {} + + host_config_params["binds"] = _parse_volumes_param(volumes) + return host_config_params + +def add_host_config_params_ports(ports=None, host_config_params=None): + """Add ports input params + + Args: + ----- + ports (list): Each port mapping entry is of the form + + "<container ports>:<host port>" + + host_config_params (dict): Target dict to accumulate host config inputs + + Returns: + -------- + Updated host_config_params + """ + if host_config_params == None: + host_config_params = {} + + if ports: + ports = [ port.split(":") for port in ports ] + port_bindings = { port[0]: { "HostPort": port[1] } for port in ports } + host_config_params["port_bindings"] = port_bindings + host_config_params["publish_all_ports"] = False + else: + host_config_params["publish_all_ports"] = True + + return host_config_params + +def add_host_config_params_dns(docker_host, host_config_params=None): + """Add dns input params + + This is not a generic implementation. This method will setup dns with the + expectation that a local consul agent is running on the docker host and will + service the dns requests. + + Args: + ----- + docker_host (string): Docker host ip address which will be used as the dns server + host_config_params (dict): Target dict to accumulate host config inputs + + Returns: + -------- + Updated host_config_params + """ + if host_config_params == None: + host_config_params = {} + + host_config_params["dns"] = [docker_host] + host_config_params["dns_search"] = ["service.consul"] + host_config_params["extra_hosts"] = { "consul": docker_host } + return host_config_params + + +def create_container_config(client, image, envs, host_config_params, tty=False): + """Create docker container config + + Args: + ----- + envs (dict): dict of environment variables to pass into the docker containers. + Gets passed into docker-py.create_container call + host_config_params (dict): Dict of input parameters to the docker-py + "create_host_config" method call + """ + # This is the 1.10.6 approach to binding volumes + # http://docker-py.readthedocs.io/en/1.10.6/volumes.html + volumes = host_config_params.get("bind", None) + target_volumes = [ target["bind"] for target in volumes.values() ] \ + if volumes else None + + host_config = client.create_host_config(**host_config_params) + + if "port_bindings" in host_config_params: + # TODO: Use six for creating the list of ports - six.iterkeys + ports = host_config_params["port_bindings"].keys() + else: + ports = None + + command = "" # This is required... + config = client.create_container_config(image, command, detach=True, tty=tty, + host_config=host_config, ports=ports, + environment=envs, volumes=target_volumes) + + utils.logger.info("Docker container config: {0}".format(config)) + + return config + diff --git a/python-dockering/dockering/core.py b/python-dockering/dockering/core.py new file mode 100644 index 0000000..dcd5908 --- /dev/null +++ b/python-dockering/dockering/core.py @@ -0,0 +1,136 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import json +import docker +import requests +from dockering.exceptions import DockerError, DockerConnectionError +from dockering import config_building as cb +from dockering import utils + + +# TODO: Replace default for logins to source it from Consul..perhaps + +def create_client(hostname, port, reauth=False, logins=[]): + """Create Docker client + + Args: + ----- + reauth: (boolean) Forces reauthentication e.g. Docker login + """ + base_url = "tcp://{0}:{1}".format(hostname, port) + try: + client = docker.Client(base_url=base_url) + + for dcl in logins: + dcl["reauth"] = reauth + client.login(**dcl) + + return client + except requests.exceptions.ConnectionError as e: + raise DockerConnectionError(str(e)) + + +def create_container_using_config(client, service_component_name, container_config): + try: + image_name = container_config["Image"] + + if not client.images(image_name): + def parse_pull_response(response): + """Pull response is a giant string of JSON messages concatentated + by `\r\n`. This method returns back those messages in the form of + list of dicts.""" + # NOTE: There's a trailing `\r\n` so the last element is empty + # string. Remove that. + return list(map(json.loads, response.split("\r\n")[:-1])) + + def get_error_message(response): + """Attempts to pull out and return an error message from parsed + response if it exists else return None""" + return response[-1].get("error", None) + + # TODO: Implement this as verbose? + # for resp in client.pull(image, stream=True, decode=True): + response = parse_pull_response(client.pull(image_name)) + error_message = get_error_message(response) + + if error_message: + raise DockerError("Error pulling Docker image: {0}".format(error_message)) + else: + utils.logger.info("Pulled Docker image: {0}".format(image_name)) + + return client.create_container_from_config(container_config, + service_component_name) + except requests.exceptions.ConnectionError as e: + # This separates connection failures so that caller can decide what to do. + # Underlying errors this inspired were socket.errors that are sourced + # from http://www.virtsync.com/c-error-codes-include-errno + raise DockerConnectionError(str(e)) + except Exception as e: + raise DockerError(str(e)) + + +def create_container(client, image_name, service_component_name, envs, + host_config_params): + """Creates Docker container + + Args: + ----- + envs (dict): dict of environment variables to pass into the docker containers. + Gets passed into docker-py.create_container call + host_config_params (dict): Dict of input parameters to the docker-py + "create_host_config" method call + """ + config = cb.create_container_config(client, image_name, envs, host_config_params) + return create_container_using_config(client, service_component_name, config) + + +def start_container(client, container): + try: + # TODO: Have logic to inspect response and through NonRecoverableError + # when start fails. Docker-py docs don't quickly tell me what the + # response looks like. + response = client.start(container=container["Id"]) + utils.logger.info("Container started: {0}".format(container["Id"])) + + # TODO: Maybe check stats? + return container["Id"] + except Exception as e: + raise DockerError(str(e)) + + +def stop_then_remove_container(client, service_component_name): + try: + client.stop(service_component_name) + client.remove_container(service_component_name) + except docker.errors.NotFound as e: + raise DockerError("Container not found: {0}".format(service_component_name)) + except Exception as e: + raise DockerError(str(e)) + + +def remove_image(client, image_name): + """Remove the Docker image""" + try: + client.remove_image(image_name) + return True + except: + # Failure to remove image is not classified as terrible..for now + return False + diff --git a/python-dockering/dockering/exceptions.py b/python-dockering/dockering/exceptions.py new file mode 100644 index 0000000..62ea145 --- /dev/null +++ b/python-dockering/dockering/exceptions.py @@ -0,0 +1,34 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +""" +Library's exceptions +""" + +class DockerError(RuntimeError): + """General error""" + pass + +class DockerConnectionError(DockerError): + """Errors connecting to the Docker engine""" + pass + +class DockerConstructionError(DockerError): + """This class of error captures failures in trying to setup the container""" + pass diff --git a/python-dockering/dockering/utils.py b/python-dockering/dockering/utils.py new file mode 100644 index 0000000..e0f651e --- /dev/null +++ b/python-dockering/dockering/utils.py @@ -0,0 +1,31 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +""" +Utility module +""" +import logging + + +# Unified all logging through this single logger in order to easily monkeypatch +# this guy in the Cloudify docker plugin. I also tried monkeypatching a getter +# function that returns a logger but that didn't work. +# WATCH! The monkeypatching in the Cloudify plugin will not work if you import +# this logger with the following syntax: from dockering.utils import logger. +logger = logging.getLogger("dockering") diff --git a/python-dockering/setup.py b/python-dockering/setup.py new file mode 100644 index 0000000..1c51ab9 --- /dev/null +++ b/python-dockering/setup.py @@ -0,0 +1,34 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import os +from setuptools import setup + +setup( + name='python-dockering', + description='Library used to manage Docker containers in DCAE', + version="1.2.0", + author="Michael Hwang", + email="dcae@lists.openecomp.org", + packages=['dockering'], + zip_safe=False, + install_requires=[ + "docker-py>=1.0.0,<2.0.0" + ] +) diff --git a/python-dockering/tests/test_config_building.py b/python-dockering/tests/test_config_building.py new file mode 100644 index 0000000..c9251e2 --- /dev/null +++ b/python-dockering/tests/test_config_building.py @@ -0,0 +1,150 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from functools import partial +import pytest +import docker +from dockering import config_building as doc +from dockering.exceptions import DockerConstructionError + + +# The docker-py library sneakily expects version to "know" that there is an +# actual Docker API that you can connect with. +DOCKER_API_VERSION = "1.24" +create_host_config = partial(docker.utils.utils.create_host_config, + version=DOCKER_API_VERSION) + +def test_add_host_config_params_volumes(): + hcp = doc.add_host_config_params_volumes() + hc = create_host_config(**hcp) + expected = { 'NetworkMode': 'default' } + assert expected == hc + + volumes = [{"host": {"path": "some-path-host"}, + "container": {"bind": "some-path-container", "mode": "ro"}}] + hcp = doc.add_host_config_params_volumes(volumes=volumes) + hc = create_host_config(**hcp) + expected = {'Binds': ['some-path-host:some-path-container:ro'], 'NetworkMode': 'default'} + assert expected == hc + + +def test_add_host_config_params_ports(): + ports = [ "22:22", "80:80" ] + hcp = doc.add_host_config_params_ports(ports=ports) + hc = create_host_config(**hcp) + expected = {'PortBindings': {'22/tcp': [{'HostPort': '22', 'HostIp': ''}], + '80/tcp': [{'HostPort': '80', 'HostIp': ''}]}, 'NetworkMode': 'default'} + assert expected == hc + + hcp = doc.add_host_config_params_ports() + hc = create_host_config(**hcp) + expected = {'NetworkMode': 'default', 'PublishAllPorts': True} + assert expected == hc + + +def test_add_host_config_params_dns(): + docker_host = "192.168.1.1" + hcp = doc.add_host_config_params_dns(docker_host) + hc = create_host_config(**hcp) + expected = {'NetworkMode': 'default', 'DnsSearch': ['service.consul'], + 'Dns': ['192.168.1.1'], 'ExtraHosts': ['consul:192.168.1.1']} + assert expected == hc + + +def test_create_envs_healthcheck(): + endpoint = "/foo" + interval = "10s" + timeout = "1s" + + docker_config = { + "healthcheck": { + "type": "http", + "endpoint": endpoint, + "interval": interval, + "timeout": timeout + } + } + + expected = { + "SERVICE_CHECK_HTTP": endpoint, + "SERVICE_CHECK_INTERVAL": interval, + "SERVICE_CHECK_TIMEOUT": timeout + } + + assert expected == doc.create_envs_healthcheck(docker_config) + + docker_config["healthcheck"]["type"] = "https" + expected = { + "SERVICE_CHECK_HTTPS": endpoint, + "SERVICE_CHECK_INTERVAL": interval, + "SERVICE_CHECK_TIMEOUT": timeout + } + + assert expected == doc.create_envs_healthcheck(docker_config) + + # Good case for just script + + script = "/bin/boo" + docker_config["healthcheck"]["type"] = "script" + docker_config["healthcheck"]["script"] = script + expected = { + "SERVICE_CHECK_SCRIPT": script, + "SERVICE_CHECK_INTERVAL": interval, + "SERVICE_CHECK_TIMEOUT": timeout + } + + assert expected == doc.create_envs_healthcheck(docker_config) + + # Good case for Docker script + + script = "/bin/boo" + docker_config["healthcheck"]["type"] = "docker" + docker_config["healthcheck"]["script"] = script + expected = { + "SERVICE_CHECK_DOCKER_SCRIPT": script, + "SERVICE_CHECK_INTERVAL": interval, + "SERVICE_CHECK_TIMEOUT": timeout + } + + assert expected == doc.create_envs_healthcheck(docker_config) + + docker_config["healthcheck"]["type"] = None + with pytest.raises(DockerConstructionError): + doc.create_envs_healthcheck(docker_config) + + +def test_create_envs(): + service_component_name = "foo" + expected_env = { "HOSTNAME": service_component_name, "SERVICE_NAME": service_component_name, + "KEY_ONE": "value_z", "KEY_TWO": "value_y" } + env_one = { "KEY_ONE": "value_z" } + env_two = { "KEY_TWO": "value_y" } + + assert expected_env == doc.create_envs(service_component_name, env_one, env_two) + + +def test_parse_volumes_param(): + volumes = [{ "host": { "path": "/var/run/docker.sock" }, + "container": { "bind": "/tmp/docker.sock", "mode": "ro" } }] + + expected = {'/var/run/docker.sock': {'bind': '/tmp/docker.sock', 'mode': 'ro'}} + actual = doc._parse_volumes_param(volumes) + assert actual == expected + + assert None == doc._parse_volumes_param(None) diff --git a/python-dockering/tests/test_core.py b/python-dockering/tests/test_core.py new file mode 100644 index 0000000..e99dbd4 --- /dev/null +++ b/python-dockering/tests/test_core.py @@ -0,0 +1,71 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import os +from functools import partial +import pytest +import docker +from dockering import core as doc +from dockering.exceptions import DockerError, DockerConnectionError + + +def test_create_client(): + # Bad - Could not connect to docker engine + + with pytest.raises(DockerConnectionError): + doc.create_client("fake", 2376, reauth=True) + + +# TODO: Does pytest provide an env file? +CONSUL_HOST = os.environ["CONSUL_HOST"] +EXTERNAL_IP = os.environ["EXTERNAL_IP"] + +@pytest.mark.skip(reason="Need to automatically setup Docker engine and maybe Consul") +def test_create_container(): + client = doc.create_client("127.0.0.1", 2376) + + scn = "unittest-registrator" + consul_host = CONSUL_HOST + # TODO: This may not work until we push the custom registrator into DockerHub + image_name = "registrator:latest" + envs = { "CONSUL_HOST": CONSUL_HOST, + "EXTERNAL_IP": EXTERNAL_IP } + volumes = {'/var/run/docker.sock': {'bind': '/tmp/docker.sock', 'mode': 'ro'}} + + hcp = doc.add_host_config_params_volumes(volumes=volumes) + container = doc.create_container(client, image_name, scn, envs, hcp) + + # Container is a dict with "Id". Check if container name matches scn. + + try: + inspect_result = client.inspect_container(scn) + import pprint + pprint.pprint(inspect_result) + + actual_mounts = inspect_result["Mounts"][0] + assert actual_mounts["Destination"] == volumes.values()[0]["bind"] + assert actual_mounts["Source"] == volumes.keys()[0] + except Exception as e: + raise e + finally: + # Execute teardown/cleanup + try: + doc.stop_then_remove_container(client, scn) + except: + print("Container removal failed") |