Undeploy k8s artifacts on timeout

Issue-ID: DCAEGEN2-987
Change-Id: I35a2dcdb7b7f1b7280dba6593a9ee2b8921b6df9
Signed-off-by: Jack Lucas <jflucas@research.att.com>

5 files changed, 32 insertions, 17 deletions

diff --git a/k8s/ChangeLog.md b/k8s/ChangeLog.md
index 7c9a72c..3402581 100644
--- a/k8s/ChangeLog.md
+++ b/k8s/ChangeLog.md
@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [1.4.8]
+* If an installation step times out because a component does not become ready within the maximum wait time,
+delete the Kubernetes artifacts associated with the component.  Previously, an installation step might time
+out due to a very slow image pull.  Cloudify would report a failure, but the component would come up, much
+later, after Kubernetes finished pulling the image.   This should no longer happen.
+
+## [1.4.7]
+* Increase unit test coverage
 
 ## [1.4.6]
 * Support for specifying CPU and memory resources in a blueprint for a containerized component
diff --git a/k8s/k8splugin/tasks.py b/k8s/k8splugin/tasks.py
index 7f91513..727be78 100644
--- a/k8s/k8splugin/tasks.py
+++ b/k8s/k8splugin/tasks.py
@@ -238,7 +238,6 @@ def _lookup_service(service_component_name, consul_host=CONSUL_HOST,
     else:
         return results[0]["ServiceAddress"]
 
-
 def _verify_k8s_deployment(service_component_name, max_wait):
     """Verify that the k8s Deployment is ready
 
@@ -249,8 +248,7 @@ def _verify_k8s_deployment(service_component_name, max_wait):
 
     Return:
     -------
-    True if deployment is ready else a DockerPluginDeploymentError exception
-    will be raised.
+    True if deployment is ready within the maximum wait time, False otherwise
     """
     num_attempts = 1
 
@@ -261,7 +259,7 @@ def _verify_k8s_deployment(service_component_name, max_wait):
             num_attempts += 1
 
             if max_wait > 0 and max_wait < num_attempts:
-                raise DockerPluginDeploymentError("k8s deployment never became ready for {0}".format(service_component_name))
+                return False
 
             time.sleep(1)
 
@@ -314,8 +312,10 @@ def _create_and_start_container(container_name, image, **kwargs):
 
     # Capture the result of deployment for future use
     ctx.instance.runtime_properties[K8S_DEPLOYMENT] = dep
+    kwargs[K8S_DEPLOYMENT] = dep
     ctx.instance.runtime_properties["replicas"] = replicas
     ctx.logger.info ("k8s deployment initiated successfully for {0}: {1}".format(container_name, dep))
+    return kwargs
 
 def _parse_cloudify_context(**kwargs):
     """Parse Cloudify context
@@ -399,7 +399,8 @@ def _create_and_start_component(**kwargs):
         "labels": kwargs.get("labels", {}),
         "resource_config": kwargs.get("resource_config",{}),
         "readiness": kwargs.get("readiness",{})}
-    _create_and_start_container(service_component_name, image, **sub_kwargs)
+    returned_args = _create_and_start_container(service_component_name, image, **sub_kwargs)
+    kwargs[K8S_DEPLOYMENT] = returned_args[K8S_DEPLOYMENT]
 
     return kwargs
 
@@ -412,6 +413,16 @@ def _verify_component(**kwargs):
 
     if _verify_k8s_deployment(service_component_name, max_wait):
         ctx.logger.info("k8s deployment is ready for: {0}".format(service_component_name))
+    else:
+        # The component did not become ready within the "max_wait" interval.
+        # Delete the k8s components created already and remove configuration from Consul.
+        ctx.logger.error("k8s deployment never became ready for {0}".format(service_component_name))
+        if (K8S_DEPLOYMENT in kwargs) and (len(kwargs[K8S_DEPLOYMENT]["deployment"]) > 0):
+            ctx.logger.info("attempting to delete k8s artifacts: {0}".format(kwargs[K8S_DEPLOYMENT]))
+            k8sclient.undeploy(kwargs[K8S_DEPLOYMENT])
+            ctx.logger.info("deleted k8s artifacts: {0}".format(kwargs[K8S_DEPLOYMENT]))
+        cleanup_discovery(**kwargs)
+        raise DockerPluginDeploymentError("k8s deployment never became ready for {0}".format(service_component_name))
 
     return kwargs
 
@@ -552,16 +563,13 @@ def create_and_start_container_for_platforms(**kwargs):
         kwargs["replicas"] = ctx.node.properties["replicas"]
     if "always_pull_image" in ctx.node.properties:
         kwargs["always_pull_image"] = ctx.node.properties["always_pull_image"]
-    _create_and_start_container(service_component_name, image, **kwargs)
+    returned_args = _create_and_start_container(service_component_name, image, **kwargs)
 
     # Verify that the k8s deployment is ready
-
-    max_wait = kwargs.get("max_wait", DEFAULT_MAX_WAIT)
-    ctx.logger.info("Waiting up to {0} secs for {1} to become ready".format(max_wait, service_component_name))
-
-    if _verify_k8s_deployment(service_component_name, max_wait):
-        ctx.logger.info("k8s deployment ready for: {0}".format(service_component_name))
-
+    #   - Set service component name into kwargs
+    #   - max_wait is already in kwargs if it was set
+    returned_args[SERVICE_COMPONENT_NAME] = service_component_name
+    _verify_component(**returned_args)
 
 @wrap_error_handling_start
 @monkeypatch_loggers
diff --git a/k8s/pom.xml b/k8s/pom.xml
index 45c5a39..3bde42e 100644
--- a/k8s/pom.xml
+++ b/k8s/pom.xml
@@ -28,7 +28,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property.
   <groupId>org.onap.dcaegen2.platform.plugins</groupId>
   <artifactId>k8s</artifactId>
   <name>k8s-plugin</name>
-  <version>1.4.7-SNAPSHOT</version>
+  <version>1.4.8-SNAPSHOT</version>
   <url>http://maven.apache.org</url>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/k8s/setup.py b/k8s/setup.py
index 9ad35bf..3d94c96 100644
--- a/k8s/setup.py
+++ b/k8s/setup.py
@@ -23,7 +23,7 @@ from setuptools import setup
 setup(
     name='k8splugin',
     description='Cloudify plugin for containerized components deployed using Kubernetes',
-    version="1.4.7",
+    version="1.4.8",
     author='J. F. Lucas, Michael Hwang, Tommy Carpenter',
     packages=['k8splugin','k8sclient','msb','configure'],
     zip_safe=False,
diff --git a/k8s/tests/test_tasks.py b/k8s/tests/test_tasks.py
index 69e866d..948489a 100644
--- a/k8s/tests/test_tasks.py
+++ b/k8s/tests/test_tasks.py
@@ -208,8 +208,7 @@ def test_verify_container(monkeypatch, mockconfig):
     monkeypatch.setattr(k8sclient, "is_available",
             fake_is_available_never_good)
 
-    with pytest.raises(DockerPluginDeploymentError):
-        tasks._verify_k8s_deployment("some-name", 2)
+    assert not tasks._verify_k8s_deployment("some-name", 2)
 
 
 def test_update_delivery_url(monkeypatch, mockconfig):