From 8db18d78975af115f44c84c170f678fc34001789 Mon Sep 17 00:00:00 2001 From: Jack Lucas Date: Mon, 18 Jun 2018 14:02:13 +0000 Subject: Create k8s readiness probe Change-Id: Iaf222957bc7aa049e3d8d6d1c290435767487387 Issue-ID: DCAEGEN2-503 Signed-off-by: Jack Lucas --- k8s/ChangeLog.md | 56 +++++++------------------------------------- k8s/k8s-node-type.yaml | 2 +- k8s/k8sclient/k8sclient.py | 57 ++++++++++++++++++++++++++++++++++++++++++--- k8s/k8splugin/decorators.py | 2 -- k8s/k8splugin/tasks.py | 33 ++++++++++++-------------- k8s/pom.xml | 2 +- k8s/requirements.txt | 1 - k8s/setup.py | 3 +-- 8 files changed, 80 insertions(+), 76 deletions(-) diff --git a/k8s/ChangeLog.md b/k8s/ChangeLog.md index 0d0eafc..28df171 100644 --- a/k8s/ChangeLog.md +++ b/k8s/ChangeLog.md @@ -5,58 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [2.4.0] -* Change *components* to be policy reconfigurable: - - Add policy execution operation - - Add policy decorators to task so that application configuration will be merged with policy -* Fetch Docker logins from Consul -## [2.3.0+t.0.3] +## [1.2.0] +* Enhancement: Use the "healthcheck" parameters from node_properties to set up a +Kubernetes readiness probe for the container. -* Enhance `SelectedDockerHost` node type with `name_search` and add default to `docker_host_override` -* Implement the functionality in the `select_docker_host` task to query Consul given location id and name search -* Deprecate `location_id` on the `DockerContainerForComponents*` node types -* Change `service_id` to be optional for `DockerContainerForComponents*` node types -* Add deployment id as a tag for registration on the component +## [1.1.0] +* Enhancement: When Cloudify Manager is running in a Docker container in a Kubernetes environment, the plugin can use the Kubernetes API credentials set up by Kubernetes. -## [2.3.0] - -* Rip out dockering and use common python-dockering library - - Using 1.2.0 of python-dockering supports Docker exec based health checks -* Support mapping ports and volumes when provided in docker config - -## [2.2.0] - -* Add `dcae.nodes.DockerContainerForComponentsUsingDmaap` node type and parse streams_publishes and streams_subscribes to be used by the DMaaP plugin. - - Handle message router wiring in the create operation for components - - Handle data router wiring in the create and in the start operation for components -* Refactor the create operations and the start operations for components. Refactored to be functional to enable for better unit test coverage. -* Add decorators for common cross cutting functionality -* Add example blueprints for different dmaap cases - -## [2.1.0] - -* Add the node type `DockerContainerForPlatforms` which is intended for platform services who are to have well known names and ports -* Add backdoor for `DockerContainerForComponents` to statically map ports -* Add hack fix to allow this plugin access to the research nexus -* Add support for dns through the local Consul agent -* Free this plugin from the CentOS bondage - -## [2.0.0] - -* Remove the magic env.ini code. It's no longer needed because we are now running local agents of Consul. -* Save and use the docker container id -* `DockerContainer` is now a different node type that is much simpler than `DockerContainerforComponents`. It is targeted for the use case of registrator. This involved overhauling the create and start container functionality. -* Classify connection and docker host not found error as recoverable -* Apply CONSUL_HOST to point to the local Consul agent +## [1.0.1] +* Fixes a bug in passing environment variables. ## [1.0.0] -* Implement health checks - expose health checks on the node and register Docker containers with it. Note that health checks are currently optional. -* Add option to remove images in the stop operation -* Verify that the container is running and healthy before finishing the start operation -* Image names passed in are now required to be the fully tagged names including registry -* Remove references to rework in the code namespaces -* Application configuration is now a YAML map to accomodate future blueprint generation -* Update blueprints and cfyhelper.sh +* Initial release of the Kubernetes plugin. It is built on the [Docker plugin](../docker) and preserves the Docker plugin's integration with the policy plugin and the DMaaP plugin. diff --git a/k8s/k8s-node-type.yaml b/k8s/k8s-node-type.yaml index 7086701..00f8c8d 100644 --- a/k8s/k8s-node-type.yaml +++ b/k8s/k8s-node-type.yaml @@ -25,7 +25,7 @@ plugins: k8s: executor: 'central_deployment_agent' package_name: k8splugin - package_version: 1.1.0 + package_version: 1.2.0 data_types: diff --git a/k8s/k8sclient/k8sclient.py b/k8s/k8sclient/k8sclient.py index 017dd36..7ca7b03 100644 --- a/k8s/k8sclient/k8sclient.py +++ b/k8s/k8sclient/k8sclient.py @@ -22,6 +22,10 @@ import uuid from msb import msb from kubernetes import config, client +# Default values for readiness probe +PROBE_DEFAULT_PERIOD = 15 +PROBE_DEFAULT_TIMEOUT = 1 + def _create_deployment_name(component_name): return "dep-{0}".format(component_name) @@ -54,7 +58,37 @@ def _configure_api(): environ=localenv ).load_and_set() -def _create_container_object(name, image, always_pull, env={}, container_ports=[], volume_mounts = []): +def _create_probe(hc, port): + ''' Create a Kubernetes probe based on info in the health check dictionary hc ''' + probe_type = hc['type'] + probe = None + period = hc.get('interval', PROBE_DEFAULT_PERIOD) + timeout = hc.get('timeout', PROBE_DEFAULT_TIMEOUT) + if probe_type == 'http' or probe_type == 'https': + probe = client.V1Probe( + failure_threshold = 1, + initial_delay_seconds = 5, + period_seconds = period, + timeout_seconds = timeout, + http_get = client.V1HTTPGetAction( + path = hc['endpoint'], + port = port, + scheme = probe_type.upper() + ) + ) + elif probe_type == 'script' or probe_type == 'docker': + probe = client.V1Probe( + failure_threshold = 1, + initial_delay_seconds = 5, + period_seconds = period, + timeout_seconds = timeout, + _exec = client.V1ExecAction( + command = [hc['script']] + ) + ) + return probe + +def _create_container_object(name, image, always_pull, env={}, container_ports=[], volume_mounts = [], readiness = None): # Set up environment variables # Copy any passed in environment variables env_vars = [client.V1EnvVar(name=k, value=env[k]) for k in env.keys()] @@ -62,6 +96,16 @@ def _create_container_object(name, image, always_pull, env={}, container_ports=[ pod_ip = client.V1EnvVarSource(field_ref = client.V1ObjectFieldSelector(field_path="status.podIP")) env_vars.append(client.V1EnvVar(name="POD_IP",value_from=pod_ip)) + # If a health check is specified, create a readiness probe + # (For an HTTP-based check, we assume it's at the first container port) + probe = None + + if (readiness): + hc_port = None + if len(container_ports) > 0: + hc_port = container_ports[0] + probe = _create_probe(readiness, hc_port) + # Define container for pod return client.V1Container( name=name, @@ -69,7 +113,8 @@ def _create_container_object(name, image, always_pull, env={}, container_ports=[ image_pull_policy='Always' if always_pull else 'IfNotPresent', env=env_vars, ports=[client.V1ContainerPort(container_port=p) for p in container_ports], - volume_mounts = volume_mounts + volume_mounts = volume_mounts, + readiness_probe = probe ) def _create_deployment_object(component_name, @@ -201,6 +246,12 @@ def deploy(namespace, component_name, image, replicas, always_pull, k8sconfig, * {"log_directory": "/path/to/container/log/directory", "alternate_fb_path" : "/alternate/sidecar/log/path"} - labels: dict with label-name/label-value pairs, e.g. {"cfydeployment" : "lsdfkladflksdfsjkl", "cfynode":"mycomponent"} These label will be set on all the pods deployed as a result of this deploy() invocation. + - readiness: dict with health check info; if present, used to create a readiness probe for the main container. Includes: + - type: check is done by making http(s) request to an endpoint ("http", "https") or by exec'ing a script in the container ("script", "docker") + - interval: period (in seconds) between probes + - timeout: time (in seconds) to allow a probe to complete + - endpoint: the path portion of the URL that points to the readiness endpoint for "http" and "https" types + - path: the full path to the script to be executed in the container for "script" and "docker" types ''' @@ -258,7 +309,7 @@ def deploy(namespace, component_name, image, replicas, always_pull, k8sconfig, * # Create the container for the component # Make it the first container in the pod - containers.insert(0, _create_container_object(component_name, image, always_pull, kwargs.get("env", {}), container_ports, volume_mounts)) + containers.insert(0, _create_container_object(component_name, image, always_pull, kwargs.get("env", {}), container_ports, volume_mounts, kwargs["readiness"])) # Build the k8s Deployment object labels = kwargs.get("labels", {}) diff --git a/k8s/k8splugin/decorators.py b/k8s/k8splugin/decorators.py index 186b212..2edcc0d 100644 --- a/k8s/k8splugin/decorators.py +++ b/k8s/k8splugin/decorators.py @@ -21,7 +21,6 @@ import copy from cloudify import ctx from cloudify.exceptions import NonRecoverableError, RecoverableError -from dockering import utils as doc from k8splugin import discovery as dis from k8splugin.exceptions import DockerPluginDeploymentError, \ DockerPluginDependencyNotReadyError @@ -33,7 +32,6 @@ def monkeypatch_loggers(task_func): def wrapper(**kwargs): # Ouch! Monkeypatch loggers - doc.logger = ctx.logger dis.logger = ctx.logger return task_func(**kwargs) diff --git a/k8s/k8splugin/tasks.py b/k8s/k8splugin/tasks.py index 8fcb582..50087fb 100644 --- a/k8s/k8splugin/tasks.py +++ b/k8s/k8splugin/tasks.py @@ -27,7 +27,6 @@ import time, copy from cloudify import ctx from cloudify.decorators import operation from cloudify.exceptions import NonRecoverableError, RecoverableError -import dockering as doc from onap_dcae_dcaepolicy_lib import Policies from k8splugin import discovery as dis from k8splugin.decorators import monkeypatch_loggers, wrap_error_handling_start, \ @@ -280,6 +279,7 @@ def _create_and_start_container(container_name, image, **kwargs): - log_info: an object with info for setting up ELK logging, with the form: {"log_directory": "/path/to/container/log/directory", "alternate_fb_path" : "/alternate/sidecar/log/path"}" - replicas: number of replicas to be launched initially + - readiness: object with information needed to create a readiness check ''' env = { "CONSUL_HOST": CONSUL_INTERNAL_NAME, "CONFIG_BINDING_SERVICE": "config-binding-service" } @@ -298,7 +298,8 @@ def _create_and_start_container(container_name, image, **kwargs): msb_list=kwargs.get("msb_list"), env = env, labels = kwargs.get("labels", {}), - log_info=kwargs.get("log_info")) + log_info=kwargs.get("log_info"), + readiness=kwargs.get("readiness")) # Capture the result of deployment for future use ctx.instance.runtime_properties["k8s_deployment"] = dep @@ -332,15 +333,17 @@ def _parse_cloudify_context(**kwargs): return kwargs def _enhance_docker_params(**kwargs): - """Setup Docker envs""" + ''' + Set up Docker environment variables and readiness check info + and inject into kwargs. + ''' + + # Get info for setting up readiness probe, if present docker_config = kwargs.get("docker_config", {}) + if "healthcheck" in docker_config: + kwargs["readiness"] = docker_config["healthcheck"] envs = kwargs.get("envs", {}) - # NOTE: Healthchecks are optional until prepared to handle use cases that - # don't necessarily use http - envs_healthcheck = doc.create_envs_healthcheck(docker_config) \ - if "healthcheck" in docker_config else {} - envs.update(envs_healthcheck) # Set tags on this component for its Consul registration as a service tags = [kwargs.get("deployment_id", None), kwargs["service_id"]] @@ -377,7 +380,8 @@ def _create_and_start_component(**kwargs): "ports": kwargs.get("ports", None), "envs": kwargs.get("envs", {}), "log_info": kwargs.get("log_info", {}), - "labels": kwargs.get("labels", {})} + "labels": kwargs.get("labels", {}), + "readiness": kwargs.get("readiness",{})} _create_and_start_container(service_component_name, image, **sub_kwargs) # TODO: Use regular logging here @@ -494,20 +498,13 @@ def create_and_start_container_for_platforms(**kwargs): # Capture node properties image = ctx.node.properties["image"] docker_config = ctx.node.properties.get("docker_config", {}) + if "healthcheck" in docker_config: + kwargs["readiness"] = docker_config["healthcheck"] if "dns_name" in ctx.node.properties: service_component_name = ctx.node.properties["dns_name"] else: service_component_name = ctx.node.properties["name"] - - envs = kwargs.get("envs", {}) - # NOTE: Healthchecks are optional until prepared to handle use cases that - # don't necessarily use http - envs_healthcheck = doc.create_envs_healthcheck(docker_config) \ - if "healthcheck" in docker_config else {} - envs.update(envs_healthcheck) - kwargs["envs"] = envs - # Set some labels for the Kubernetes pods kwargs["labels"] = { "cfydeployment" : ctx.deployment.id, diff --git a/k8s/pom.xml b/k8s/pom.xml index afcf45a..d51eae5 100644 --- a/k8s/pom.xml +++ b/k8s/pom.xml @@ -28,7 +28,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property. org.onap.dcaegen2.platform.plugins k8s k8s-plugin - 1.1.0-SNAPSHOT + 1.2.0-SNAPSHOT http://maven.apache.org UTF-8 diff --git a/k8s/requirements.txt b/k8s/requirements.txt index d107559..f5cac20 100644 --- a/k8s/requirements.txt +++ b/k8s/requirements.txt @@ -1,6 +1,5 @@ python-consul>=0.6.0,<1.0.0 uuid==1.30 -onap-dcae-dockering==1.4.0 onap-dcae-dcaepolicy-lib==2.1.0 kubernetes==4.0.0 cloudify-plugins-common==3.4 \ No newline at end of file diff --git a/k8s/setup.py b/k8s/setup.py index 1d15ff5..5de6a76 100644 --- a/k8s/setup.py +++ b/k8s/setup.py @@ -23,14 +23,13 @@ from setuptools import setup setup( name='k8splugin', description='Cloudify plugin for containerized components deployed using Kubernetes', - version="1.1.0", + version="1.2.0", author='J. F. Lucas, Michael Hwang, Tommy Carpenter', packages=['k8splugin','k8sclient','msb','configure'], zip_safe=False, install_requires=[ "python-consul>=0.6.0,<1.0.0", "uuid==1.30", - "onap-dcae-dockering>=1.0.0,<2.0.0", "onap-dcae-dcaepolicy-lib>=2.1.0,<3.0.0", "cloudify-plugins-common==3.4", "cloudify-python-importer==0.1.0", -- cgit 1.2.3-korg