From 6a8304c51e129a7860d60ac562feac1f6056b5c3 Mon Sep 17 00:00:00 2001 From: Mahendra Raghuwanshi Date: Mon, 28 Jan 2019 09:49:30 +0000 Subject: Readiness check for the deployment/statefulsets Change-Id: I8b9d2bd2720060cab215107bc0031c243155e9f6 Issue-ID: OOM-1611 Signed-off-by: Mahendra Raghuwanshi --- kubernetes/readiness/docker/init/ready.py | 84 ++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 17 deletions(-) (limited to 'kubernetes/readiness/docker/init/ready.py') diff --git a/kubernetes/readiness/docker/init/ready.py b/kubernetes/readiness/docker/init/ready.py index f4a5e5da8f..87c09a444c 100644 --- a/kubernetes/readiness/docker/init/ready.py +++ b/kubernetes/readiness/docker/init/ready.py @@ -30,6 +30,62 @@ configuration.ssl_ca_cert = cert configuration.api_key['authorization'] = token configuration.api_key_prefix['authorization'] = 'Bearer' coreV1Api = client.CoreV1Api(client.ApiClient(configuration)) +api_instance=client.ExtensionsV1beta1Api(client.ApiClient(configuration)) +api = client.AppsV1beta1Api(client.ApiClient(configuration)) +batchV1Api = client.BatchV1Api(client.ApiClient(configuration)) + +def is_job_complete(job_name): + complete = False + log.info("Checking if " + job_name + " is complete") + response = "" + try: + response = batchV1Api.read_namespaced_job_status(job_name, namespace) + if response.status.succeeded == 1: + job_status_type = response.status.conditions[0].type + if job_status_type == "Complete": + complete = True + log.info(job_name + " is complete") + else: + log.info(job_name + " is not complete") + else: + log.info(job_name + " has not succeeded yet") + return complete + except Exception as e: + log.error("Exception when calling read_namespaced_job_status: %s\n" % e) + +def wait_for_statefulset_complete(statefulset_name): + try: + response = api.read_namespaced_stateful_set(statefulset_name, namespace) + s = response.status + if ( s.updated_replicas == response.spec.replicas and + s.replicas == response.spec.replicas and + s.ready_replicas == response.spec.replicas and + s.current_replicas == response.spec.replicas and + s.observed_generation == response.metadata.generation): + log.info("Statefulset " + statefulset_name + " is ready") + return True + else: + log.info("Statefulset " + statefulset_name + " is not ready") + return False + except Exception as e: + log.error("Exception when waiting for Statefulset status: %s\n" % e) + +def wait_for_deployment_complete(deployment_name): + try: + response = api.read_namespaced_deployment(deployment_name, namespace) + s = response.status + if ( s.unavailable_replicas == None and + s.updated_replicas == response.spec.replicas and + s.replicas == response.spec.replicas and + s.ready_replicas == response.spec.replicas and + s.observed_generation == response.metadata.generation): + log.info("Deployment " + deployment_name + " is ready") + return True + else: + log.info("Deployment " + deployment_name + " is not ready") + return False + except Exception as e: + log.error("Exception when waiting for deployment status: %s\n" % e) def is_ready(container_name): ready = False @@ -41,28 +97,23 @@ def is_ready(container_name): if i.status.container_statuses is None: continue for s in i.status.container_statuses: - if i.metadata.owner_references[0].kind == "StatefulSet": - if i.metadata.name == container_name: - ready = s.ready - if not ready: - log.info(container_name + " is not ready.") - else: - log.info(container_name + " is ready!") - else: - continue - elif s.name == container_name: - ready = s.ready - if not ready: - log.info(container_name + " is not ready.") - else: - log.info(container_name + " is ready!") + if s.name == container_name: + if i.metadata.owner_references[0].kind == "StatefulSet": + ready = wait_for_statefulset_complete(i.metadata.owner_references[0].name) + elif i.metadata.owner_references[0].kind == "ReplicaSet": + api_response = api_instance.read_namespaced_replica_set_status(i.metadata.owner_references[0].name, namespace) + ready = wait_for_deployment_complete(api_response.metadata.owner_references[0].name) + elif i.metadata.owner_references[0].kind == "Job": + ready = is_job_complete(i.metadata.owner_references[0].name) + + return ready + else: continue return ready except Exception as e: log.error("Exception when calling list_namespaced_pod: %s\n" % e) - DEF_TIMEOUT = 10 DESCRIPTION = "Kubernetes container readiness check utility" USAGE = "Usage: ready.py [-t ] -c [-c ...]\n" \ @@ -105,7 +156,6 @@ def main(argv): else: time.sleep(5) - if __name__ == "__main__": main(sys.argv[1:]) -- cgit 1.2.3-korg