diff options
author | 2023-05-18 21:04:51 +0000 | |
---|---|---|
committer | 2023-05-23 06:55:33 +0000 | |
commit | 6dc4f7277cd749ebe278ae0838c34ff3a2481ef9 (patch) | |
tree | 344acc25ef03dd20be9e49195759e5bd2001b4ed /src/onaptests/scenario | |
parent | deb99c7b3c7caaf9704e6316f7eba062a54b04fc (diff) |
Refactor status verification test
Issue-ID: INT-2239
Signed-off-by: pawel.denst <pawel.denst@external.t-mobile.pl>
Signed-off-by: Lukasz Rajewski <lukasz.rajewski@t-mobile.pl>
Change-Id: I6db4d1d46536b0e1d6aeac2d8c9a569ed007f590
Diffstat (limited to 'src/onaptests/scenario')
-rw-r--r-- | src/onaptests/scenario/resources.py | 155 | ||||
-rw-r--r-- | src/onaptests/scenario/status.py | 755 |
2 files changed, 36 insertions, 874 deletions
diff --git a/src/onaptests/scenario/resources.py b/src/onaptests/scenario/resources.py deleted file mode 100644 index 2b0352e..0000000 --- a/src/onaptests/scenario/resources.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Resources module.""" - - -class K8sResource(): - """K8sResource class.""" - - def __init__(self, k8s=None): - """Init the k8s resource.""" - self.k8s = k8s - self.name = "" - self.events = [] - if self.k8s: - self.name = self.k8s.metadata.name - self.specific_k8s_init() - - def specific_k8s_init(self): - """Do the specific part for k8s resource when k8s object is present.""" - pass - - def __repr__(self): - return self.name - - def __str__(self): - return self.name - - def __eq__(self, other): - if (isinstance(other, K8sResource)): - return self.name == other.name - else: - return False - -class K8sPodParentResource(K8sResource): - """K8sPodParentResource class.""" - - def __init__(self, k8s=None): - """Init the k8s pod parent resource.""" - self.pods = [] - self.failed_pods = 0 - super().__init__(k8s=k8s) - - -class Pod(K8sResource): - """Pod class.""" - - def __init__(self, k8s=None): - """Init the pod.""" - self.containers = [] - self.init_containers = [] - self.running_containers = 0 - self.runned_init_containers = 0 - self.volumes = {} - self.restart_count = 0 - self.init_restart_count = 0 - self.init_done = True - super().__init__(k8s=k8s) - - def specific_k8s_init(self): - """Specific k8s init.""" - self.set_volumes(self.k8s.spec.volumes) - - def set_volumes(self, volumes): - """Generate the volume list.""" - for volume in volumes: - volume_name = volume.name - self.volumes[volume_name] = {} - for volume_type in volume.attribute_map: - if volume_type != "name" and getattr(volume, volume_type): - self._parse_volume_type(volume, volume_name, volume_type) - - def _parse_volume_type(self, volume, name, volume_type): - """Parse volume type informations.""" - self.volumes[name][volume_type] = {} - infos = getattr(volume, volume_type) - for details in infos.attribute_map: - self.volumes[name][volume_type][details] = getattr(infos, details) - - def ready(self): - """Calculate if Pod is ready.""" - if self.init_done and self.running_containers == len(self.containers): - return True - return False - - -class Container(): - """Container class.""" - - def __init__(self, name=""): - """Init the container.""" - self.name = name - self.status = "" - self.ready = False - self.restart_count = 0 - self.image = "" - - def set_status(self, status): - """Generate status for container.""" - if status.running: - self.status = "Running" - else: - if status.terminated: - self.status = "Terminated ({})".format( - status.terminated.reason) - else: - if status.waiting: - self.status = "Waiting ({})".format( - status.waiting.reason) - else: - self.status = "Unknown" - - -class Service(K8sPodParentResource): - """Service class.""" - - def __init__(self, k8s=None): - """Init the service.""" - self.type = "" - super().__init__(k8s=k8s) - - def specific_k8s_init(self): - """Do the specific part for service when k8s object is present.""" - self.type = self.k8s.spec.type - - -class Job(K8sPodParentResource): - """Job class.""" - - -class Deployment(K8sPodParentResource): - """Deployment class.""" - -class ReplicaSet(K8sPodParentResource): - """ReplicaSet class.""" - -class StatefulSet(K8sPodParentResource): - """StatefulSet class.""" - - -class DaemonSet(K8sPodParentResource): - """DaemonSet class.""" - - -class Pvc(K8sResource): - """Pvc class.""" - - -class ConfigMap(K8sResource): - """ConfigMap class.""" - - -class Secret(K8sResource): - """Secret class.""" - - -class Ingress(K8sResource): - """Ingress class."""
\ No newline at end of file diff --git a/src/onaptests/scenario/status.py b/src/onaptests/scenario/status.py index c8aea74..79ce0c6 100644 --- a/src/onaptests/scenario/status.py +++ b/src/onaptests/scenario/status.py @@ -1,739 +1,56 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: Apache-2.0 -import json -import os import logging -import re import time -from natural.date import delta from xtesting.core import testcase -from kubernetes import client, config -from kubernetes.stream import stream -from urllib3.exceptions import MaxRetryError, NewConnectionError -from jinja2 import Environment, PackageLoader, select_autoescape +from onapsdk.configuration import settings +from onapsdk.exceptions import SDKException -from onaptests.scenario.resources import Pod, Container, Service, Job -from onaptests.scenario.resources import Deployment, StatefulSet, DaemonSet, Pvc, ReplicaSet -from onaptests.scenario.resources import ConfigMap, Secret, Ingress - -NAMESPACE = os.getenv('K8S_NAMESPACE', 'onap') -FULL_LOGS_CONTAINERS = [ - 'dcae-bootstrap', 'dcae-cloudify-manager', 'aai-resources', - 'aai-traversal', 'aai-modelloader', 'sdnc', 'so', 'so-bpmn-infra', - 'so-openstack-adapter', 'so-sdc-controller', 'mariadb-galera', 'sdc-be', - 'sdc-fe' -] - -# patterns to be excluded from the check -WAIVER_LIST = ['integration'] - -SPECIFIC_LOGS_CONTAINERS = { - 'sdc-be': ['/var/log/onap/sdc/sdc-be/error.log'], - 'sdc-onboarding-be': ['/var/log/onap/sdc/sdc-onboarding-be/error.log'], - 'aaf-cm': [ - '/opt/app/osaaf/logs/cm/cm-service.log', - '/opt/app/osaaf/logs/cm/cm-init.log' - ], - 'aaf-fs': [ - '/opt/app/osaaf/logs/fs/fs-service.log', - '/opt/app/osaaf/logs/fs/fs-init.log' - ], - 'aaf-locate': [ - '/opt/app/osaaf/logs/locate/locate-service.log', - '/opt/app/osaaf/logs/locate/locate-init.log' - ], - 'aaf-service': [ - '/opt/app/osaaf/logs/service/authz-service.log', - '/opt/app/osaaf/logs/service/authz-init.log' - ], - 'sdc-be': [ - '/var/log/onap/sdc/sdc-be/debug.log', - '/var/log/onap/sdc/sdc-be/error.log' - ], - 'sdc-fe': [ - '/var/log/onap/sdc/sdc-fe/debug.log', - '/var/log/onap/sdc/sdc-fe/error.log' - ], - 'vid': [ - '/var/log/onap/vid/audit.log', - '/var/log/onap/vid/application.log', - '/var/log/onap/vid/debug.log', - '/var/log/onap/vid/error.log' - ], -} - -DOCKER_REPOSITORIES = [ - 'nexus3.onap.org:10001', 'docker.elastic.co', 'docker.io', 'library', - 'registry.gitlab.com', 'registry.hub.docker.com', 'k8s.gcr.io', 'gcr.io' -] -DOCKER_REPOSITORIES_NICKNAMES = { - 'nexus3.onap.org:10001': 'onap', - 'docker.elastic.co': 'elastic', - 'docker.io': 'dockerHub (docker.io)', - 'registry.hub.docker.com': 'dockerHub (registry)', - 'registry.gitlab.com': 'gitlab', - 'library': 'dockerHub (library)', - 'default': 'dockerHub', - 'k8s.gcr.io': 'google (k8s.gcr)', - 'gcr.io': 'google (gcr)' -} - -GENERIC_NAMES = { - 'postgreSQL': ['crunchydata/crunchy-postgres', 'postgres'], - 'mariadb': ['adfinissygroup/k8s-mariadb-galera-centos', 'mariadb'], - 'elasticsearch': [ - 'bitnami/elasticsearch', 'elasticsearch/elasticsearch', - 'onap/clamp-dashboard-elasticsearch' - ], - 'nginx': ['bitnami/nginx', 'nginx'], - 'cassandra': [ - 'cassandra', 'onap/music/cassandra_3_11', 'onap/music/cassandra_music', - 'onap/aaf/aaf_cass' - ], - 'zookeeper': ['google_samples/k8szk', 'onap/dmaap/zookeeper', 'zookeeper'], - 'redis': [ - 'onap/vfc/db', - 'onap/org.onap.dcaegen2.deployments.redis-cluster-container' - ], - 'consul': ['consul', 'oomk8s/consul'], - 'rabbitmq': ['ansible/awx_rabbitmq', 'rabbitmq'] -} - -MAX_LOG_BYTES = 512000 +from onaptests.steps.cloud.check_status import CheckNamespaceStatusStep +from onaptests.utils.exceptions import OnapTestException class Status(testcase.TestCase): - """Retrieve status of Kubernetes resources.""" + """Retrieve status of Kubernetes resources in the nemaspace.""" __logger = logging.getLogger(__name__) - def __init__(self, kubeconfig=None, dir_result=None, **kwargs): + def __init__(self, **kwargs): """Init the testcase.""" if "case_name" not in kwargs: - kwargs["case_name"] = 'namespace_status' - super(Status, self).__init__(**kwargs) - if kubeconfig is not None: - config.load_kube_config(config_file=kubeconfig) - else: - config.load_kube_config() - self.core = client.CoreV1Api() - self.batch = client.BatchV1Api() - self.app = client.AppsV1Api() - self.networking = client.NetworkingV1Api() - if dir_result: - self.res_dir = f"{dir_result}/kubernetes-status" - else: - self.res_dir = f"{self.dir_results}/kubernetes-status" - - self.__logger.debug("namespace status init started") + kwargs["case_name"] = 'status' + super().__init__(**kwargs) + self.__logger.debug("Status check init started") + self.test = CheckNamespaceStatusStep( + cleanup=settings.CLEANUP_FLAG) self.start_time = None self.stop_time = None self.result = 0 - self.pods = [] - self.services = [] - self.jobs = [] - self.deployments = [] - self.replicasets =[] - self.statefulsets = [] - self.daemonsets = [] - self.pvcs = [] - self.configmaps = [] - self.secrets = [] - self.ingresses = [] - self.details = {} def run(self): - """Run tests.""" + """Status check test.""" self.start_time = time.time() - os.makedirs(self.res_dir, exist_ok=True) - self.__logger.debug("start test") + self.__logger.debug("start time") try: - self.k8s_pods = self.core.list_namespaced_pod(NAMESPACE).items - self.__logger.info("%4s Pods in the namespace", len(self.k8s_pods)) - - self.k8s_jobs = self.batch.list_namespaced_job(NAMESPACE).items - self.__logger.info("%4s Jobs in the namespace", len(self.k8s_jobs)) - - self.k8s_deployments = self.app.list_namespaced_deployment( - NAMESPACE).items - self.__logger.info("%4s Deployments in the namespace", - len(self.k8s_deployments)) - - self.k8s_replicasets = self.app.list_namespaced_replica_set( - NAMESPACE).items - self.__logger.info("%4s Replicasets in the namespace", - len(self.k8s_replicasets)) - - self.k8s_statefulsets = self.app.list_namespaced_stateful_set( - NAMESPACE).items - self.__logger.info("%4s StatefulSets in the namespace", - len(self.k8s_statefulsets)) - - self.k8s_daemonsets = self.app.list_namespaced_daemon_set( - NAMESPACE).items - self.__logger.info("%4s DaemonSets in the namespace", - len(self.k8s_daemonsets)) - - self.k8s_services = self.core.list_namespaced_service( - NAMESPACE).items - self.__logger.info("%4s Services in the namespace", - len(self.k8s_services)) - - self.k8s_pvcs = self.core.list_namespaced_persistent_volume_claim( - NAMESPACE).items - self.__logger.info("%4s PVCs in the namespace", len(self.pvcs)) - - self.k8s_configmaps = self.core.list_namespaced_config_map( - NAMESPACE).items - self.__logger.info("%4s ConfigMaps in the namespace", - len(self.configmaps)) - - self.k8s_secrets = self.core.list_namespaced_secret( - NAMESPACE).items - self.__logger.info("%4s Secrets in the namespace", - len(self.secrets)) - - self.k8s_ingresses = self.networking.list_namespaced_ingress( - NAMESPACE).items - self.__logger.info("%4s Ingresses in the namespace", - len(self.ingresses)) - except (ConnectionRefusedError, MaxRetryError, NewConnectionError): - self.__logger.error("namespace status test failed.") - self.__logger.error("cannot connect to Kubernetes.") - return testcase.TestCase.EX_TESTCASE_FAILED - - self.failing_statefulsets = [] - self.failing_jobs = [] - self.failing_deployments = [] - self.failing_replicasets = [] - self.failing_daemonsets = [] - self.failing_pvcs = [] - self.failing = False - - self.jinja_env = Environment(autoescape=select_autoescape(['html']), - loader=PackageLoader('onaptests.templates','status')) - self.parse_services() - jobs_pods = self.parse_jobs() - self.parse_pods(excluded_pods=jobs_pods) - self.parse_deployments() - self.parse_replicasets() - self.parse_statefulsets() - self.parse_daemonsets() - self.parse_pvcs() - self.parse_configmaps() - self.parse_secrets() - self.parse_ingresses() - self.parse_versions() - - self.jinja_env.get_template('index.html.j2').stream( - ns=self, - delta=delta).dump('{}/index.html'.format(self.res_dir)) - self.jinja_env.get_template('raw_output.txt.j2').stream( - ns=self, namespace=NAMESPACE).dump('{}/onap-k8s.log'.format( - self.res_dir)) - - self.stop_time = time.time() - if len(self.jobs) > 0: - self.details['jobs'] = { - 'number': len(self.jobs), - 'number_failing': len(self.failing_jobs), - 'failing': self.map_by_name(self.failing_jobs) - } - if len(self.deployments) > 0: - self.details['deployments'] = { - 'number': len(self.deployments), - 'number_failing': len(self.failing_deployments), - 'failing': self.map_by_name(self.failing_deployments) - } - if len(self.replicasets) > 0: - self.details['replicasets'] = { - 'number': len(self.replicasets), - 'number_failing': len(self.failing_replicasets), - 'failing': self.map_by_name(self.failing_replicasets) - } - if len(self.statefulsets) > 0: - self.details['statefulsets'] = { - 'number': len(self.statefulsets), - 'number_failing': len(self.failing_statefulsets), - 'failing': self.map_by_name(self.failing_statefulsets) - } - if len(self.daemonsets) > 0: - self.details['daemonsets'] = { - 'number': len(self.daemonsets), - 'number_failing': len(self.failing_daemonsets), - 'failing': self.map_by_name(self.failing_daemonsets) - } - if len(self.pvcs) > 0: - self.details['pvcs'] = { - 'number': len(self.pvcs), - 'number_failing': len(self.failing_pvcs), - 'failing': self.map_by_name(self.failing_pvcs) - } - if self.failing: - self.__logger.error("namespace status test failed.") - self.__logger.error("number of errored Jobs: %s", - len(self.failing_jobs)) - self.__logger.error("number of errored Deployments: %s", - len(self.failing_deployments)) - self.__logger.error("number of errored Replicasets: %s", - len(self.failing_replicasets)) - self.__logger.error("number of errored StatefulSets: %s", - len(self.failing_statefulsets)) - self.__logger.error("number of errored DaemonSets: %s", - len(self.failing_daemonsets)) - self.__logger.error("number of errored PVCs: %s", - len(self.failing_pvcs)) - return testcase.TestCase.EX_TESTCASE_FAILED - - self.result = 100 - return testcase.TestCase.EX_OK - - def parse_pods(self, excluded_pods=None): - """Parse the pods status.""" - self.__logger.info("%4s pods to parse", len(self.k8s_pods)) - for k8s in self.k8s_pods: - pod = Pod(k8s=k8s) - - if excluded_pods and pod in excluded_pods: - continue - - if k8s.status.init_container_statuses: - for k8s_container in k8s.status.init_container_statuses: - pod.runned_init_containers += self.parse_container( - pod, k8s_container, init=True) - if k8s.status.container_statuses: - for k8s_container in k8s.status.container_statuses: - pod.running_containers += self.parse_container( - pod, k8s_container) - pod.events = self.core.list_namespaced_event( - NAMESPACE, - field_selector="involvedObject.name={}".format(pod.name)).items - self.jinja_env.get_template('pod.html.j2').stream(pod=pod).dump( - '{}/pod-{}.html'.format(self.res_dir, pod.name)) - if any(waiver_elt in pod.name for waiver_elt in WAIVER_LIST): - self.__logger.warn("Waiver pattern found in pod, exclude %s", pod.name) - else: - self.pods.append(pod) - - def parse_container(self, pod, k8s_container, init=False): - """Get the logs of a container.""" - logs = "" - old_logs = "" - prefix = "" - containers_list = pod.containers - container = Container(name=k8s_container.name) - container.restart_count = k8s_container.restart_count - container.set_status(k8s_container.state) - container.ready = k8s_container.ready - container.image = k8s_container.image - if init: - prefix = "init " - containers_list = pod.init_containers - if container.restart_count > pod.init_restart_count: - pod.init_restart_count = container.restart_count - if not container.ready: - pod.init_done = False - else: - if container.restart_count > pod.restart_count: - pod.restart_count = container.restart_count - - try: - log_files = {} - logs = "" - try: - logs = self.core.read_namespaced_pod_log( - pod.name, - NAMESPACE, - container=container.name, - limit_bytes=MAX_LOG_BYTES, - ) - except UnicodeDecodeError: - logs= "{0} has an unicode decode error...".format(pod.name) - self.__logger.error( - "{0} has an unicode decode error in the logs...", pod.name, - ) - with open( - "{}/pod-{}-{}.log".format(self.res_dir, - pod.name, container.name), - 'w') as log_result: - log_result.write(logs) - if (not container.ready) and container.restart_count > 0: - old_logs = self.core.read_namespaced_pod_log( - pod.name, - NAMESPACE, - container=container.name, - previous=True) - with open( - "{}/pod-{}-{}.old.log".format(self.res_dir, - pod.name, - container.name), - 'w') as log_result: - log_result.write(old_logs) - if (container.name in FULL_LOGS_CONTAINERS): - logs = self.core.read_namespaced_pod_log( - pod.name, NAMESPACE, container=container.name) - with open( - "{}/pod-{}-{}.log".format(self.res_dir, - pod.name, container.name), - 'w') as log_result: - log_result.write(logs) - if (container.name in SPECIFIC_LOGS_CONTAINERS): - for log_file in SPECIFIC_LOGS_CONTAINERS[container.name]: - exec_command = ['/bin/sh', '-c', "cat {}".format(log_file)] - log_files[log_file] = stream( - self.core.connect_get_namespaced_pod_exec, - pod.name, - NAMESPACE, - container=container.name, - command=exec_command, - stderr=True, - stdin=False, - stdout=True, - tty=False) - log_file_slug = log_file.split('.')[0].split('/')[-1] - with open( - "{}/pod-{}-{}-{}.log".format( - self.res_dir, pod.name, - container.name, log_file_slug), - 'w') as log_result: - log_result.write(log_files[log_file]) - except client.rest.ApiException as exc: - self.__logger.warning("%scontainer %s of pod %s has an exception: %s", - prefix, container.name, pod.name, exc.reason) - self.jinja_env.get_template('container_log.html.j2').stream( - container=container, - pod_name=pod.name, - logs=logs, - old_logs=old_logs, - log_files=log_files).dump('{}/pod-{}-{}-logs.html'.format( - self.res_dir, pod.name, container.name)) - if any(waiver_elt in container.name for waiver_elt in WAIVER_LIST): - self.__logger.warn( - "Waiver pattern found in container, exclude %s", container.name) - else: - containers_list.append(container) - if k8s_container.ready: - return 1 - return 0 - - def parse_services(self): - """Parse the services.""" - self.__logger.info("%4s services to parse", len(self.k8s_services)) - for k8s in self.k8s_services: - service = Service(k8s=k8s) - - (service.pods, - service.failed_pods) = self._find_child_pods(k8s.spec.selector) - - self.jinja_env.get_template('service.html.j2').stream( - service=service).dump('{}/service-{}.html'.format( - self.res_dir, service.name)) - self.services.append(service) - - def parse_jobs(self): - """Parse the jobs. - Return a list of Pods that were created to perform jobs. - """ - self.__logger.info("%4s jobs to parse", len(self.k8s_jobs)) - jobs_pods = [] - for i in range(len(self.k8s_jobs)): - k8s = self.k8s_jobs[i] - job = Job(k8s=k8s) - job_pods = [] - - if k8s.spec.selector and k8s.spec.selector.match_labels: - (job.pods, job.failed_pods) = self._find_child_pods( - k8s.spec.selector.match_labels) - job_pods += job.pods - field_selector = "involvedObject.name={}".format(job.name) - field_selector += ",involvedObject.kind=Job" - job.events = self.core.list_namespaced_event( - NAMESPACE, - field_selector=field_selector).items - - self.jinja_env.get_template('job.html.j2').stream(job=job).dump( - '{}/job-{}.html'.format(self.res_dir, job.name)) - - # timemout job - if not k8s.status.completion_time: - self.__logger.warning("a Job is in error: {}".format(job.name)) - if any( - waiver_elt not in job.name for waiver_elt in WAIVER_LIST): - self.failing_jobs.append(job) - self.failing = True - # completed job - if any(waiver_elt not in job.name for waiver_elt in WAIVER_LIST): - self.jobs.append(job) - jobs_pods += job_pods - return jobs_pods - - def parse_deployments(self): - """Parse the deployments.""" - self.__logger.info("%4s deployments to parse", - len(self.k8s_deployments)) - for i in range(len(self.k8s_deployments)): - k8s = self.k8s_deployments[i] - deployment = Deployment(k8s=k8s) - - if k8s.spec.selector and k8s.spec.selector.match_labels: - (deployment.pods, - deployment.failed_pods) = self._find_child_pods( - k8s.spec.selector.match_labels) - field_selector = "involvedObject.name={}".format(deployment.name) - field_selector += ",involvedObject.kind=Deployment" - deployment.events = self.core.list_namespaced_event( - NAMESPACE, - field_selector=field_selector).items - - self.jinja_env.get_template('deployment.html.j2').stream( - deployment=deployment).dump('{}/deployment-{}.html'.format( - self.res_dir, deployment.name)) - - if k8s.status.unavailable_replicas: - self.__logger.warning("a Deployment is in error: {}".format(deployment.name)) - self.failing_deployments.append(deployment) - self.failing = True - - self.deployments.append(deployment) - - def parse_replicasets(self): - """Parse the replicasets.""" - self.__logger.info("%4s replicasets to parse", - len(self.k8s_replicasets)) - for i in range(len(self.k8s_replicasets)): - k8s = self.k8s_replicasets[i] - replicaset = ReplicaSet(k8s=k8s) - - if k8s.spec.selector and k8s.spec.selector.match_labels: - (replicaset.pods, - replicaset.failed_pods) = self._find_child_pods( - k8s.spec.selector.match_labels) - field_selector = "involvedObject.name={}".format(replicaset.name) - field_selector += ",involvedObject.kind=ReplicaSet" - replicaset.events = self.core.list_namespaced_event( - NAMESPACE, - field_selector=field_selector).items - - self.jinja_env.get_template('replicaset.html.j2').stream( - replicaset=replicaset).dump('{}/replicaset-{}.html'.format( - self.res_dir, replicaset.name)) - - if (not k8s.status.ready_replicas - or (k8s.status.ready_replicas < k8s.status.replicas)): - self.__logger.warning("a ReplicaSet is in error: {}".format(replicaset.name)) - self.failing_replicasets.append(replicaset) - self.failing = True - - self.replicasets.append(replicaset) - - def parse_statefulsets(self): - """Parse the statefulsets.""" - self.__logger.info("%4s statefulsets to parse", - len(self.k8s_statefulsets)) - for i in range(len(self.k8s_statefulsets)): - k8s = self.k8s_statefulsets[i] - statefulset = StatefulSet(k8s=k8s) - - if k8s.spec.selector and k8s.spec.selector.match_labels: - (statefulset.pods, - statefulset.failed_pods) = self._find_child_pods( - k8s.spec.selector.match_labels) - field_selector = "involvedObject.name={}".format(statefulset.name) - field_selector += ",involvedObject.kind=StatefulSet" - statefulset.events = self.core.list_namespaced_event( - NAMESPACE, - field_selector=field_selector).items - - self.jinja_env.get_template('statefulset.html.j2').stream( - statefulset=statefulset).dump('{}/statefulset-{}.html'.format( - self.res_dir, statefulset.name)) - - if ((not k8s.status.ready_replicas) - or (k8s.status.ready_replicas < k8s.status.replicas)): - self.__logger.warning("a StatefulSet is in error: {}".format(statefulset.name)) - self.failing_statefulsets.append(statefulset) - self.failing = True - - self.statefulsets.append(statefulset) - - def parse_daemonsets(self): - """Parse the daemonsets.""" - self.__logger.info("%4s daemonsets to parse", len(self.k8s_daemonsets)) - for i in range(len(self.k8s_daemonsets)): - k8s = self.k8s_daemonsets[i] - daemonset = DaemonSet(k8s=k8s) - - if k8s.spec.selector and k8s.spec.selector.match_labels: - (daemonset.pods, - daemonset.failed_pods) = self._find_child_pods( - k8s.spec.selector.match_labels) - field_selector = "involvedObject.name={}".format(daemonset.name) - field_selector += ",involvedObject.kind=DaemonSet" - daemonset.events = self.core.list_namespaced_event( - NAMESPACE, - field_selector=field_selector).items - - self.jinja_env.get_template('daemonset.html.j2').stream( - daemonset=daemonset).dump('{}/daemonset-{}.html'.format( - self.res_dir, daemonset.name)) - - if (k8s.status.number_ready < k8s.status.desired_number_scheduled): - self.__logger.warning("a DaemonSet is in error: {}".format(daemonset.name)) - self.failing_daemonsets.append(daemonset) - self.failing = True - - self.daemonsets.append(daemonset) - - def parse_pvcs(self): - """Parse the persistent volume claims.""" - self.__logger.info("%4s pvcs to parse", len(self.k8s_pvcs)) - for k8s in self.k8s_pvcs: - pvc = Pvc(k8s=k8s) - field_selector = f"involvedObject.name={pvc.name},involvedObject.kind=PersistentVolumeClaim" - pvc.events = self.core.list_namespaced_event( - NAMESPACE, - field_selector=field_selector).items - - if k8s.status.phase != "Bound": - self.__logger.warning("a PVC is in error: {}".format(pvc.name)) - self.failing_pvcs.append(pvc) - self.failing = True - - self.pvcs.append(pvc) - - def parse_configmaps(self): - """Parse the config maps.""" - self.__logger.info("%4s config maps to parse", - len(self.k8s_configmaps)) - for k8s in self.k8s_configmaps: - configmap = ConfigMap(k8s=k8s) - self.configmaps.append(configmap) - - def parse_secrets(self): - """Parse the secrets.""" - self.__logger.info("%4s secrets to parse", len(self.k8s_secrets)) - for k8s in self.k8s_secrets: - secret = Secret(k8s=k8s) - self.secrets.append(secret) - - def parse_ingresses(self): - """Parse the ingresses.""" - self.__logger.info("%4s ingresses to parse", len(self.k8s_ingresses)) - for k8s in self.k8s_secrets: - ingress = Ingress(k8s=k8s) - self.ingresses.append(ingress) - - def parse_versions(self): - """Parse the versions of the pods.""" - self.__logger.info("%4s pods to parse", len(self.k8s_pods)) - pod_versions = [] - containers = {} - for pod in self.k8s_pods: - pod_component = pod.metadata.name - if 'app' in pod.metadata.labels: - pod_component = pod.metadata.labels['app'] + self.test.execute() + self.__logger.info("Status check successfully completed") + # The cleanup is part of the test, not only a teardown action + if settings.CLEANUP_FLAG: + self.__logger.info("Status check cleanup called") + time.sleep(settings.CLEANUP_ACTIVITY_TIMER) + self.test.cleanup() + self.result = 100 else: - if 'app.kubernetes.io/name' in pod.metadata.labels: - pod_component = pod.metadata.labels[ - 'app.kubernetes.io/name'] - else: - self.__logger.error("pod %s has no 'app' or 'app.kubernetes.io/name' in metadata: %s", pod_component, pod.metadata.labels) - - # looks for docker version - for container in pod.spec.containers: - pod_version = {} - pod_container_version = container.image.rsplit(":", 1) - pod_container_image = pod_container_version[0] - pod_container_tag = "latest" - if len(pod_container_version) > 1: - pod_container_tag = pod_container_version[1] - - pod_version.update({ - 'container': container.name, - 'component': pod_component, - 'image': pod_container_image, - 'version': pod_container_tag - }) - pod_versions.append(pod_version) - - search_rule = "^(?P<source>[^/]*)/*(?P<container>[^:]*):*(?P<version>.*)$" - search = re.search(search_rule, container.image) - name = "{}/{}".format(search.group('source'), - search.group('container')) - version = search.group('version') - if name[-1] == '/': - name = name[0:-1] - source = "default" - if search.group('source') in DOCKER_REPOSITORIES: - source = search.group('source') - name = search.group('container') - container_search_rule = "^library/(?P<real_container>[^:]*)$" - container_search = re.search(container_search_rule, name) - if container_search: - name = container_search.group('real_container') - for common_component in GENERIC_NAMES.keys(): - if name in GENERIC_NAMES[common_component]: - version = "{}:{}".format(name, version) - name = common_component - break - - repository = DOCKER_REPOSITORIES_NICKNAMES[source] - if name in containers: - if version in containers[name]['versions']: - if not (pod_component in containers[name]['versions'] - [version]['components']): - containers[name]['versions'][version][ - 'components'].append(pod_component) - containers[name]['number_components'] += 1 - if not (repository in containers[name]['versions'] - [version]['repositories']): - containers[name]['versions'][version][ - 'repositories'].append(repository) - else: - containers[name]['versions'][version] = { - 'repositories': [repository], - 'components': [pod_component] - } - containers[name]['number_components'] += 1 - else: - containers[name] = { - 'versions': { - version: { - 'repositories': [repository], - 'components': [pod_component] - } - }, - 'number_components': 1 - } - - self.jinja_env.get_template('version.html.j2').stream( - pod_versions=pod_versions).dump('{}/versions.html'.format( - self.res_dir)) - self.jinja_env.get_template('container_versions.html.j2').stream( - containers=containers).dump('{}/container_versions.html'.format( - self.res_dir)) - # create a json file for version tracking - with open(self.res_dir + "/onap_versions.json", "w") as write_file: - json.dump(pod_versions, write_file) - - def _find_child_pods(self, selector): - pods_list = [] - failed_pods = 0 - if selector: - raw_selector = '' - for key, value in selector.items(): - raw_selector += key + '=' + value + ',' - raw_selector = raw_selector[:-1] - pods = self.core.list_namespaced_pod( - NAMESPACE, label_selector=raw_selector).items - for pod in pods: - for known_pod in self.pods: - if known_pod.name == pod.metadata.name: - pods_list.append(known_pod) - if not known_pod.ready(): - failed_pods += 1 - return (pods_list, failed_pods) - - def map_by_name(self, resources): - return list(map(lambda resource: resource.name, resources)) + self.__logger.info("No cleanup requested. Test completed.") + self.result = 100 + except OnapTestException as exc: + self.result = 0 + self.__logger.exception(exc.error_message) + except SDKException: + self.result = 0 + self.__logger.exception("SDK Exception") + finally: + self.stop_time = time.time() + + def clean(self): + """Clean Additional resources if needed.""" + self.__logger.info("Generate Test report") + self.test.reports_collection.generate_report() |