From ed20e3771ad69381962b6bd8bfc2ae95bacf76ca Mon Sep 17 00:00:00 2001 From: Sylvain Desbureaux Date: Mon, 14 Sep 2020 11:12:15 +0200 Subject: [INFRA] new helm test Instead of parsing only helm status, this new test is also taking a look at specific helm deploy plugin logs. Also, main pages is now "html" based instead of raw based. Issue-ID: INT-1718 Signed-off-by: Sylvain Desbureaux Change-Id: Ia851db5c8cfd866394ed22e3cd3b7ce183ef0fb9 --- infra-healthcheck/README.md | 10 ++- infra-healthcheck/docker/Dockerfile | 2 - infra-healthcheck/infra_healthcheck/k8stest.py | 49 ++--------- infra-healthcheck/requirements.txt | 1 + infra-healthcheck/scripts/check_onap_helm.sh | 36 --------- infra-healthcheck/scripts/check_onap_k8s.sh | 108 ------------------------- infra-healthcheck/setup.cfg | 2 +- infra-healthcheck/upper-constraints.txt | 2 + 8 files changed, 18 insertions(+), 192 deletions(-) delete mode 100644 infra-healthcheck/scripts/check_onap_helm.sh delete mode 100644 infra-healthcheck/scripts/check_onap_k8s.sh diff --git a/infra-healthcheck/README.md b/infra-healthcheck/README.md index 0f7001b..36628a1 100644 --- a/infra-healthcheck/README.md +++ b/infra-healthcheck/README.md @@ -54,19 +54,25 @@ environment variables: - TEST_DB_URL: the url of the target Database with the env variable . - NODE_NAME: the name of your test environement. It must be declared in the test database (e.g. windriver-SB00) -- Optionnal +- Optional: - INSTALLER_TYPE: precise how your ONAP has been installed (e.g. kubespray-oom, rke-oom) - BUILD_TAG: a unique tag of your CI system. It can be usefull to get all the - tests of one CI run. It uses the regex (dai|week)ly-(.+?)-[0-9]\* to find the + tests of one CI run. It uses the regex (dai|week)ly-(.+?)-[0-9]\* to find the version (e.g. daily-elalto-123456789). - DEPLOY_SCENARIO: your scenario deployment. ingress test run only if the scenario includes 'ingress' + - ONAP_RELEASE: the name of the onap release in Helm. Default is "onap". + - ONAP_HELM_LOG_PATH: the path where to retrieve specific logs that helm + deploy has captured. you should add a volume if you want to retrieve them: + `-v /.helm/plugins/deploy/cache/onap/logs:/onap_helm_logs`. + `/onap_helm_logs` is the default value. The command becomes: ``` docker run -v :/root/.kube/config -v +-v /.helm/plugins/deploy/cache/onap/logs:/onap_helm_logs :/var/lib/xtesting/results nexus3.onap.org:10001/onap/xtesting-infra-healthcheck:latest:latest /bin/bash -c "run_tests -r -t all" diff --git a/infra-healthcheck/docker/Dockerfile b/infra-healthcheck/docker/Dockerfile index 5e32672..f84f976 100644 --- a/infra-healthcheck/docker/Dockerfile +++ b/infra-healthcheck/docker/Dockerfile @@ -11,7 +11,6 @@ ARG ONAP_TAG=master ADD https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl /usr/local/bin/kubectl -COPY scripts/check_onap_helm.sh /check_onap_helm.sh COPY upper-constraints.txt . RUN set -x && \ @@ -23,7 +22,6 @@ RUN set -x && \ wget https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz -O - | tar -xzO linux-amd64/helm > /usr/local/bin/helm && \ wget -O /check_for_ingress_and_nodeports.py https://git.onap.org/integration/plain/test/security/check_for_ingress_and_nodeports.py?h=$ONAP_TAG &&\ chmod +x /usr/local/bin/helm && \ - chmod +x /check_onap_*.sh && \ pip3 install --upgrade pip && \ pip3 install --no-cache-dir -r upper-constraints.txt && \ pip3 install --no-cache-dir \ diff --git a/infra-healthcheck/infra_healthcheck/k8stest.py b/infra-healthcheck/infra_healthcheck/k8stest.py index deab627..e1c4213 100644 --- a/infra-healthcheck/infra_healthcheck/k8stest.py +++ b/infra-healthcheck/infra_healthcheck/k8stest.py @@ -48,9 +48,10 @@ class K8sTesting(testcase.TestCase): raise Exception(output) # create a log file - result_folder = "/var/lib/xtesting/results/" + self.case_name + "/" - file_name = result_folder + self.case_name + ".log" + result_folder = "{}/{}".format(self.dir_results, self.case_name) + file_name = "{}/{}.log".format(result_folder, self.case_name) os.makedirs(result_folder, exist_ok=True) + log_file = open(file_name, "w") log_file.write(output) log_file.close() @@ -66,35 +67,7 @@ class K8sTesting(testcase.TestCase): remarks.append(log.replace('>', '')) else: remarks.append(log) - - if self.case_name == 'onap-helm': - for remark in remarks: - if ':' in remark: - # 2 possible Results - # * numeric nb pods, failed, duration - # * list of pods, charts,... - # split and replace can be hazardous, depending - # on result format change.. - try: - if '[' in remark: - # it is a list - str1 = remark.split( - ":", 1)[1].strip().replace( - ']', '').replace('[', '') - details[remark.split( - ":", 1)[0].strip()] = str1.split(",") - else: - details[remark.split(":", 1)[0].strip()] = int( - remark.split(":", 1)[1].strip()) - except: - pass - - # if 1 pod/helm chart if Failed, the testcase is failed - if int(details[self.criteria_string]) < 1: - success = True - elif("failed" not in str_remarks.join(remarks).lower()): - success = True - elif 'PASS' in remarks: + if 'PASS' in remarks: success = True self.details = details @@ -106,7 +79,6 @@ class K8sTesting(testcase.TestCase): self.result = 0 def run(self, **kwargs): - self.start_time = time.time() try: self.run_kubetest() @@ -119,25 +91,16 @@ class K8sTesting(testcase.TestCase): return res -class OnapHelmTest(K8sTesting): - """Kubernetes conformance test suite""" - def __init__(self, **kwargs): - super(OnapHelmTest, self).__init__(**kwargs) - self.cmd = ['/check_onap_helm.sh'] - self.criteria_string = "Nb Failed Helm Charts" - - class OnapSecurityNodePortsIngress(K8sTesting): """Check that there is no NodePort without corresponding Ingress port.""" def __init__(self, **kwargs): super(OnapSecurityNodePortsIngress, self).__init__(**kwargs) self.cmd = ['python3', '/check_for_ingress_and_nodeports.py', '--conf', '/root/.kube/config'] - self.criteria_string = "NodePort without corresponding Ingress found" - + self.error_string = "NodePort without corresponding Ingress found" class OnapSecurityNodePortsCerts(K8sTesting): - """Check the cerfificates fot he nodeports.""" + """Check the cerfificates for the nodeports.""" def __init__(self, **kwargs): super(OnapSecurityNodePortsCerts, self).__init__(**kwargs) os.chdir('/usr/lib/python3.8/site-packages/check_certificates') diff --git a/infra-healthcheck/requirements.txt b/infra-healthcheck/requirements.txt index 91babf8..cacbc00 100644 --- a/infra-healthcheck/requirements.txt +++ b/infra-healthcheck/requirements.txt @@ -12,3 +12,4 @@ xtesting kubernetes # Apache-2.0 colorama # BSD kubernetes_status +helm_onap_status diff --git a/infra-healthcheck/scripts/check_onap_helm.sh b/infra-healthcheck/scripts/check_onap_helm.sh deleted file mode 100644 index 27f8094..0000000 --- a/infra-healthcheck/scripts/check_onap_helm.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -echo "------------------------------------------------------------------------" -echo "------------------- ONAP Check helm charts ----------------------------" -echo "------------------------------------------------------------------------" -code=0 -nb_charts=$(helm ls |awk {'print $1'}|grep -v NAME |wc -l) -nb_failed_charts=0 -list_failed_charts="[$(helm ls |grep -v DEPLOYED |grep -v NAME |awk '{print $1}')]" -nice_list=$(echo $list_failed_charts |sed -e "s/ /,/g") - -# List Helm chart and get their status -for i in $(helm ls |awk {'print $1'}|grep -v NAME);do - echo "Chart $i" - status=$(helm status $i |grep STATUS:) - echo ${status} - if [ "${status}" != "STATUS: DEPLOYED" ]; then - echo "Chart problem" - helm status $i -o yaml - code=1 - let "nb_failed_charts++" - fi - echo "--------------------------------------------------------------------" -done - -echo "------------------------------------------------" -echo "------- ONAP Helm tests ------------------------" -echo "------------------------------------------------" -echo ">>> Nb Helm Charts: ${nb_charts}" -echo ">>> Nb Failed Helm Charts: ${nb_failed_charts}" -echo ">>> List of Failed Helm Charts: ${nice_list}" -echo "------------------------------------------------" -echo "------------------------------------------------" -echo "------------------------------------------------" - -exit $code diff --git a/infra-healthcheck/scripts/check_onap_k8s.sh b/infra-healthcheck/scripts/check_onap_k8s.sh deleted file mode 100644 index 2dffd7e..0000000 --- a/infra-healthcheck/scripts/check_onap_k8s.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -echo "------------------------------------------------------------------------" -echo "-------------------- ONAP Check kubernetes ----------------------------" -echo "------------------------------------------------------------------------" - -code=0 - -# get the pod list -echo "List of ONAP pods" -echo "*****************" -kubectl get pods -n onap - -# show deployments -echo "Show ONAP kubernetes deployments" -echo "********************************" -kubectl get deployments -n onap -echo "------------------------------------------------------------------------" - -# show SVC -echo "Show ONAP kubernetes SVC" -echo "************************" -kubectl get svc -n onap -echo "------------------------------------------------------------------------" - -# show ONAP events -echo "Show ONAP kubernetes events" -echo "***************************" -kubectl get events -n onap -echo "------------------------------------------------------------------------" - -# show ONAP config maps -echo "Show ONAP kubernetes config maps" -echo "***************************" -kubectl get cm -n onap -echo "------------------------------------------------------------------------" - -# show ONAP jobs -echo "Show ONAP kubernetes jobs" -echo "***************************" -kubectl get jobs -n onap -echo "------------------------------------------------------------------------" - -# show ONAP statefulsets -echo "Show ONAP kubernetes statefulset" -echo "***************************" -kubectl get sts -n onap -echo "------------------------------------------------------------------------" - -# if all pods in RUNNING state exit 0, else exit 1 -nb_pods=$((`kubectl get pods -n onap | grep Running | grep -v functest | wc -l` -1)) -list_failed_pods=$(kubectl get pods -n onap |grep -v Running |grep -v functest |grep -v NAME | grep -v Completed | awk '{print $1}') -list_filtered_failed_pods=() - -for i in $list_failed_pods;do - status=$(kubectl get pods -n onap $i | grep -v NAME | awk '{print $3'}) - # in case of Error or Init:Error - # we check that another instance is not already Completed or Running - if [ $status = "Error" ] || [ $status = "Init:Error" ];then - echo "$i in Status Error or Init Error found for the pods, is is really true...." - # By default pod naming is similar, keep only the root to check - root_name=${i::-6} - kubectl get pods -n onap | grep $root_name | grep Completed - if [ $? ];then - echo "Instance Completed found." - else - echo "No Completed instance found." - list_filtered_failed_pods+=$i, - fi - else - # Other status are not running/not completed pods - list_filtered_failed_pods+=$i, - fi -done - -nice_list=${list_filtered_failed_pods::-1} - -IFS=, -nb_pods_not_running=$(echo "$list_filtered_failed_pods" | tr -cd , | wc -c) - -if [ $nb_pods_not_running -ne 0 ]; then -echo "$nb_pods_not_running pods (on $nb_pods) are not in Running state" -echo "---------------------------------------------------------------------" - kubectl get pods -n onap | grep -v Running | grep -v functest | grep -v Completed - echo "--------------------------------------------------------------------" - echo "Describe non running pods" - echo "*************************" - for i in $nice_list;do - echo "****************************************************************" - kubectl describe pod $i -n onap - kubectl logs --all-containers=true -n onap $i - done - code=1 -else - echo "all pods ($nb_pods) are running well" -fi - -echo "------------------------------------------------" -echo "------- ONAP kubernetes tests ------------------" -echo "------------------------------------------------" -echo ">>> Nb Pods: $nb_pods" -echo ">>> Nb Failed Pods: $nb_pods_not_running" -echo ">>> List of Failed Pods: [$nice_list]" -echo "------------------------------------------------" -echo "------------------------------------------------" -echo "------------------------------------------------" - -exit $code diff --git a/infra-healthcheck/setup.cfg b/infra-healthcheck/setup.cfg index 4664552..f06fd45 100644 --- a/infra-healthcheck/setup.cfg +++ b/infra-healthcheck/setup.cfg @@ -8,7 +8,7 @@ packages = infra_healthcheck [entry_points] xtesting.testcase = onap_k8s = kubernetes_status.status:Status - onap_helm = infra_healthcheck.k8stest:OnapHelmTest + onap_helm = helm_onap_status.status:Status nodeport_ingress = infra_healthcheck.k8stest:OnapSecurityNodePortsIngress nodeport_check_certs = infra_healthcheck.k8stest:OnapSecurityNodePortsCerts diff --git a/infra-healthcheck/upper-constraints.txt b/infra-healthcheck/upper-constraints.txt index d37d29d..527e677 100644 --- a/infra-healthcheck/upper-constraints.txt +++ b/infra-healthcheck/upper-constraints.txt @@ -1 +1,3 @@ git+https://gitlab.com/Orange-OpenSource/lfn/tools/kubernetes-status.git#egg=kubernetes_status +git+https://gitlab.com/Orange-OpenSource/lfn/tools/helm-onap-status.git#egg=helm_onap_status + -- cgit 1.2.3-korg