summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJack Lucas <jflucas@research.att.com>2019-03-05 11:43:01 -0500
committerJack Lucas <jflucas@research.att.com>2019-03-06 18:37:37 -0500
commitb835cbc6eb17fe7d279c1963c520130dda8d519a (patch)
tree1984350a02fc79cbea5bd92be1619caf1704e369
parent059ebc5469200911b901fe41cab29447fa9a9705 (diff)
Undeploy k8s artifacts on timeout
Issue-ID: DCAEGEN2-987 Change-Id: I35a2dcdb7b7f1b7280dba6593a9ee2b8921b6df9 Signed-off-by: Jack Lucas <jflucas@research.att.com>
-rw-r--r--k8s/ChangeLog.md8
-rw-r--r--k8s/k8splugin/tasks.py34
-rw-r--r--k8s/pom.xml2
-rw-r--r--k8s/setup.py2
-rw-r--r--k8s/tests/test_tasks.py3
5 files changed, 32 insertions, 17 deletions
diff --git a/k8s/ChangeLog.md b/k8s/ChangeLog.md
index 7c9a72c..3402581 100644
--- a/k8s/ChangeLog.md
+++ b/k8s/ChangeLog.md
@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
+## [1.4.8]
+* If an installation step times out because a component does not become ready within the maximum wait time,
+delete the Kubernetes artifacts associated with the component. Previously, an installation step might time
+out due to a very slow image pull. Cloudify would report a failure, but the component would come up, much
+later, after Kubernetes finished pulling the image. This should no longer happen.
+
+## [1.4.7]
+* Increase unit test coverage
## [1.4.6]
* Support for specifying CPU and memory resources in a blueprint for a containerized component
diff --git a/k8s/k8splugin/tasks.py b/k8s/k8splugin/tasks.py
index 7f91513..727be78 100644
--- a/k8s/k8splugin/tasks.py
+++ b/k8s/k8splugin/tasks.py
@@ -238,7 +238,6 @@ def _lookup_service(service_component_name, consul_host=CONSUL_HOST,
else:
return results[0]["ServiceAddress"]
-
def _verify_k8s_deployment(service_component_name, max_wait):
"""Verify that the k8s Deployment is ready
@@ -249,8 +248,7 @@ def _verify_k8s_deployment(service_component_name, max_wait):
Return:
-------
- True if deployment is ready else a DockerPluginDeploymentError exception
- will be raised.
+ True if deployment is ready within the maximum wait time, False otherwise
"""
num_attempts = 1
@@ -261,7 +259,7 @@ def _verify_k8s_deployment(service_component_name, max_wait):
num_attempts += 1
if max_wait > 0 and max_wait < num_attempts:
- raise DockerPluginDeploymentError("k8s deployment never became ready for {0}".format(service_component_name))
+ return False
time.sleep(1)
@@ -314,8 +312,10 @@ def _create_and_start_container(container_name, image, **kwargs):
# Capture the result of deployment for future use
ctx.instance.runtime_properties[K8S_DEPLOYMENT] = dep
+ kwargs[K8S_DEPLOYMENT] = dep
ctx.instance.runtime_properties["replicas"] = replicas
ctx.logger.info ("k8s deployment initiated successfully for {0}: {1}".format(container_name, dep))
+ return kwargs
def _parse_cloudify_context(**kwargs):
"""Parse Cloudify context
@@ -399,7 +399,8 @@ def _create_and_start_component(**kwargs):
"labels": kwargs.get("labels", {}),
"resource_config": kwargs.get("resource_config",{}),
"readiness": kwargs.get("readiness",{})}
- _create_and_start_container(service_component_name, image, **sub_kwargs)
+ returned_args = _create_and_start_container(service_component_name, image, **sub_kwargs)
+ kwargs[K8S_DEPLOYMENT] = returned_args[K8S_DEPLOYMENT]
return kwargs
@@ -412,6 +413,16 @@ def _verify_component(**kwargs):
if _verify_k8s_deployment(service_component_name, max_wait):
ctx.logger.info("k8s deployment is ready for: {0}".format(service_component_name))
+ else:
+ # The component did not become ready within the "max_wait" interval.
+ # Delete the k8s components created already and remove configuration from Consul.
+ ctx.logger.error("k8s deployment never became ready for {0}".format(service_component_name))
+ if (K8S_DEPLOYMENT in kwargs) and (len(kwargs[K8S_DEPLOYMENT]["deployment"]) > 0):
+ ctx.logger.info("attempting to delete k8s artifacts: {0}".format(kwargs[K8S_DEPLOYMENT]))
+ k8sclient.undeploy(kwargs[K8S_DEPLOYMENT])
+ ctx.logger.info("deleted k8s artifacts: {0}".format(kwargs[K8S_DEPLOYMENT]))
+ cleanup_discovery(**kwargs)
+ raise DockerPluginDeploymentError("k8s deployment never became ready for {0}".format(service_component_name))
return kwargs
@@ -552,16 +563,13 @@ def create_and_start_container_for_platforms(**kwargs):
kwargs["replicas"] = ctx.node.properties["replicas"]
if "always_pull_image" in ctx.node.properties:
kwargs["always_pull_image"] = ctx.node.properties["always_pull_image"]
- _create_and_start_container(service_component_name, image, **kwargs)
+ returned_args = _create_and_start_container(service_component_name, image, **kwargs)
# Verify that the k8s deployment is ready
-
- max_wait = kwargs.get("max_wait", DEFAULT_MAX_WAIT)
- ctx.logger.info("Waiting up to {0} secs for {1} to become ready".format(max_wait, service_component_name))
-
- if _verify_k8s_deployment(service_component_name, max_wait):
- ctx.logger.info("k8s deployment ready for: {0}".format(service_component_name))
-
+ # - Set service component name into kwargs
+ # - max_wait is already in kwargs if it was set
+ returned_args[SERVICE_COMPONENT_NAME] = service_component_name
+ _verify_component(**returned_args)
@wrap_error_handling_start
@monkeypatch_loggers
diff --git a/k8s/pom.xml b/k8s/pom.xml
index 45c5a39..3bde42e 100644
--- a/k8s/pom.xml
+++ b/k8s/pom.xml
@@ -28,7 +28,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property.
<groupId>org.onap.dcaegen2.platform.plugins</groupId>
<artifactId>k8s</artifactId>
<name>k8s-plugin</name>
- <version>1.4.7-SNAPSHOT</version>
+ <version>1.4.8-SNAPSHOT</version>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/k8s/setup.py b/k8s/setup.py
index 9ad35bf..3d94c96 100644
--- a/k8s/setup.py
+++ b/k8s/setup.py
@@ -23,7 +23,7 @@ from setuptools import setup
setup(
name='k8splugin',
description='Cloudify plugin for containerized components deployed using Kubernetes',
- version="1.4.7",
+ version="1.4.8",
author='J. F. Lucas, Michael Hwang, Tommy Carpenter',
packages=['k8splugin','k8sclient','msb','configure'],
zip_safe=False,
diff --git a/k8s/tests/test_tasks.py b/k8s/tests/test_tasks.py
index 69e866d..948489a 100644
--- a/k8s/tests/test_tasks.py
+++ b/k8s/tests/test_tasks.py
@@ -208,8 +208,7 @@ def test_verify_container(monkeypatch, mockconfig):
monkeypatch.setattr(k8sclient, "is_available",
fake_is_available_never_good)
- with pytest.raises(DockerPluginDeploymentError):
- tasks._verify_k8s_deployment("some-name", 2)
+ assert not tasks._verify_k8s_deployment("some-name", 2)
def test_update_delivery_url(monkeypatch, mockconfig):