[INFRA] new helm test

Instead of parsing only helm status, this new test is also taking a look
at specific helm deploy plugin logs.
Also, main pages is now "html" based instead of raw based.

Issue-ID: INT-1718
Signed-off-by: Sylvain Desbureaux <sylvain.desbureaux@orange.com>
Change-Id: Ia851db5c8cfd866394ed22e3cd3b7ce183ef0fb9

8 files changed, 18 insertions, 192 deletions

diff --git a/infra-healthcheck/README.md b/infra-healthcheck/README.md
index 0f7001b..36628a1 100644
--- a/infra-healthcheck/README.md
+++ b/infra-healthcheck/README.md
@@ -54,19 +54,25 @@ environment variables:
   - TEST_DB_URL: the url of the target Database with the env variable .
   - NODE_NAME: the name of your test environement. It must be declared in the
     test database (e.g. windriver-SB00)
-- Optionnal
+- Optional:
   - INSTALLER_TYPE: precise how your ONAP has been installed (e.g. kubespray-oom,
     rke-oom)
   - BUILD_TAG: a unique tag of your CI system. It can be usefull to get all the
-    tests of one CI run. It uses the regex (dai|week)ly-(.+?)-[0-9]\* to find the
+    tests of one CI run. It uses the regex (dai|week)ly-(.+?)-[0-9]\* to find the 
     version (e.g. daily-elalto-123456789).
   - DEPLOY_SCENARIO: your scenario deployment. ingress test run only if the
     scenario includes 'ingress'
+  - ONAP_RELEASE: the name of the onap release in Helm. Default is "onap".
+  - ONAP_HELM_LOG_PATH: the path where to retrieve specific logs that helm
+    deploy has captured. you should add a volume if you want to retrieve them:
+    `-v <the user home dir>/.helm/plugins/deploy/cache/onap/logs:/onap_helm_logs`.
+    `/onap_helm_logs` is the default value.
 
 The command becomes:
 
 ```
 docker run -v <the kube config>:/root/.kube/config -v
+-v <the user home dir>/.helm/plugins/deploy/cache/onap/logs:/onap_helm_logs
 <result directory>:/var/lib/xtesting/results
 nexus3.onap.org:10001/onap/xtesting-infra-healthcheck:latest:latest
 /bin/bash -c "run_tests -r -t all"
diff --git a/infra-healthcheck/docker/Dockerfile b/infra-healthcheck/docker/Dockerfile
index 5e32672..f84f976 100644
--- a/infra-healthcheck/docker/Dockerfile
+++ b/infra-healthcheck/docker/Dockerfile
@@ -11,7 +11,6 @@ ARG ONAP_TAG=master
 
 ADD https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl /usr/local/bin/kubectl
 
-COPY scripts/check_onap_helm.sh /check_onap_helm.sh
 COPY upper-constraints.txt .
 
 RUN set -x && \
@@ -23,7 +22,6 @@ RUN set -x && \
     wget https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz -O - | tar -xzO linux-amd64/helm > /usr/local/bin/helm && \
     wget -O /check_for_ingress_and_nodeports.py https://git.onap.org/integration/plain/test/security/check_for_ingress_and_nodeports.py?h=$ONAP_TAG &&\
     chmod +x /usr/local/bin/helm && \
-    chmod +x /check_onap_*.sh && \
     pip3 install --upgrade pip && \
     pip3 install --no-cache-dir -r upper-constraints.txt && \
     pip3 install --no-cache-dir \
diff --git a/infra-healthcheck/infra_healthcheck/k8stest.py b/infra-healthcheck/infra_healthcheck/k8stest.py
index deab627..e1c4213 100644
--- a/infra-healthcheck/infra_healthcheck/k8stest.py
+++ b/infra-healthcheck/infra_healthcheck/k8stest.py
@@ -48,9 +48,10 @@ class K8sTesting(testcase.TestCase):
             raise Exception(output)
 
         # create a log file
-        result_folder = "/var/lib/xtesting/results/" + self.case_name + "/"
-        file_name = result_folder + self.case_name + ".log"
+        result_folder = "{}/{}".format(self.dir_results, self.case_name)
+        file_name = "{}/{}.log".format(result_folder, self.case_name)
         os.makedirs(result_folder, exist_ok=True)
+
         log_file = open(file_name, "w")
         log_file.write(output)
         log_file.close()
@@ -66,35 +67,7 @@ class K8sTesting(testcase.TestCase):
                 remarks.append(log.replace('>', ''))
             else:
                 remarks.append(log)
-
-        if self.case_name == 'onap-helm':
-            for remark in remarks:
-                if ':' in remark:
-                    # 2 possible Results
-                    # * numeric nb pods, failed, duration
-                    # * list of pods, charts,...
-                    # split and replace can be hazardous, depending
-                    # on result format change..
-                    try:
-                        if '[' in remark:
-                        # it is a list
-                            str1 = remark.split(
-                                ":", 1)[1].strip().replace(
-                                    ']', '').replace('[', '')
-                            details[remark.split(
-                                ":", 1)[0].strip()] = str1.split(",")
-                        else:
-                            details[remark.split(":", 1)[0].strip()] = int(
-                                remark.split(":", 1)[1].strip())
-                    except:
-                        pass
-
-            # if 1 pod/helm chart if Failed, the testcase is failed
-            if int(details[self.criteria_string]) < 1:
-                success = True
-            elif("failed" not in str_remarks.join(remarks).lower()):
-                success = True
-        elif 'PASS' in remarks:
+        if 'PASS' in remarks:
             success = True
 
         self.details = details
@@ -106,7 +79,6 @@ class K8sTesting(testcase.TestCase):
             self.result = 0
 
     def run(self, **kwargs):
-
         self.start_time = time.time()
         try:
             self.run_kubetest()
@@ -119,25 +91,16 @@ class K8sTesting(testcase.TestCase):
         return res
 
 
-class OnapHelmTest(K8sTesting):
-    """Kubernetes conformance test suite"""
-    def __init__(self, **kwargs):
-        super(OnapHelmTest, self).__init__(**kwargs)
-        self.cmd = ['/check_onap_helm.sh']
-        self.criteria_string = "Nb Failed Helm Charts"
-
-
 class OnapSecurityNodePortsIngress(K8sTesting):
     """Check that there is no NodePort without corresponding Ingress port."""
     def __init__(self, **kwargs):
         super(OnapSecurityNodePortsIngress, self).__init__(**kwargs)
         self.cmd = ['python3', '/check_for_ingress_and_nodeports.py',
                     '--conf', '/root/.kube/config']
-        self.criteria_string = "NodePort without corresponding Ingress found"
-
+        self.error_string = "NodePort without corresponding Ingress found"
 
 class OnapSecurityNodePortsCerts(K8sTesting):
-    """Check the cerfificates fot he nodeports."""
+    """Check the cerfificates for the nodeports."""
     def __init__(self, **kwargs):
         super(OnapSecurityNodePortsCerts, self).__init__(**kwargs)
         os.chdir('/usr/lib/python3.8/site-packages/check_certificates')
diff --git a/infra-healthcheck/requirements.txt b/infra-healthcheck/requirements.txt
index 91babf8..cacbc00 100644
--- a/infra-healthcheck/requirements.txt
+++ b/infra-healthcheck/requirements.txt
@@ -12,3 +12,4 @@ xtesting
 kubernetes # Apache-2.0
 colorama # BSD
 kubernetes_status
+helm_onap_status
diff --git a/infra-healthcheck/scripts/check_onap_helm.sh b/infra-healthcheck/scripts/check_onap_helm.sh
deleted file mode 100644
index 27f8094..0000000
--- a/infra-healthcheck/scripts/check_onap_helm.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-echo "------------------------------------------------------------------------"
-echo "-------------------  ONAP Check helm charts ----------------------------"
-echo "------------------------------------------------------------------------"
-code=0
-nb_charts=$(helm ls |awk {'print $1'}|grep -v NAME |wc -l)
-nb_failed_charts=0
-list_failed_charts="[$(helm ls |grep -v DEPLOYED |grep -v NAME |awk '{print $1}')]"
-nice_list=$(echo $list_failed_charts |sed  -e "s/ /,/g")
-
-# List Helm chart and get their status
-for i in $(helm ls |awk {'print $1'}|grep -v NAME);do
-    echo "Chart $i"
-    status=$(helm status $i |grep STATUS:)
-    echo ${status}
-    if [ "${status}" != "STATUS: DEPLOYED" ]; then
-        echo "Chart problem"
-        helm status $i -o yaml
-        code=1
-        let "nb_failed_charts++"
-    fi
-    echo "--------------------------------------------------------------------"
-done
-
-echo "------------------------------------------------"
-echo "------- ONAP Helm tests ------------------------"
-echo "------------------------------------------------"
-echo ">>> Nb Helm Charts: ${nb_charts}"
-echo ">>> Nb Failed Helm Charts: ${nb_failed_charts}"
-echo ">>> List of Failed Helm Charts: ${nice_list}"
-echo "------------------------------------------------"
-echo "------------------------------------------------"
-echo "------------------------------------------------"
-
-exit $code
diff --git a/infra-healthcheck/scripts/check_onap_k8s.sh b/infra-healthcheck/scripts/check_onap_k8s.sh
deleted file mode 100644
index 2dffd7e..0000000
--- a/infra-healthcheck/scripts/check_onap_k8s.sh
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/bin/bash
-
-echo "------------------------------------------------------------------------"
-echo "--------------------  ONAP Check kubernetes ----------------------------"
-echo "------------------------------------------------------------------------"
-
-code=0
-
-# get the pod list
-echo "List of ONAP pods"
-echo "*****************"
-kubectl get pods -n onap
-
-# show deployments
-echo "Show ONAP kubernetes deployments"
-echo "********************************"
-kubectl get deployments -n onap
-echo "------------------------------------------------------------------------"
-
-# show SVC
-echo "Show ONAP kubernetes SVC"
-echo "************************"
-kubectl get svc -n onap
-echo "------------------------------------------------------------------------"
-
-# show ONAP events
-echo "Show ONAP kubernetes events"
-echo "***************************"
-kubectl get events -n onap
-echo "------------------------------------------------------------------------"
-
-# show ONAP config maps
-echo "Show ONAP kubernetes config maps"
-echo "***************************"
-kubectl get cm -n onap
-echo "------------------------------------------------------------------------"
-
-# show ONAP jobs
-echo "Show ONAP kubernetes jobs"
-echo "***************************"
-kubectl get jobs -n onap
-echo "------------------------------------------------------------------------"
-
-# show ONAP statefulsets
-echo "Show ONAP kubernetes statefulset"
-echo "***************************"
-kubectl get sts -n onap
-echo "------------------------------------------------------------------------"
-
-# if all pods in RUNNING state exit 0, else exit 1
-nb_pods=$((`kubectl get pods -n onap | grep Running | grep -v functest | wc -l` -1))
-list_failed_pods=$(kubectl get pods -n onap |grep -v Running |grep -v functest |grep -v NAME | grep -v Completed | awk '{print $1}')
-list_filtered_failed_pods=()
-
-for i in $list_failed_pods;do
-    status=$(kubectl get pods -n onap $i | grep -v NAME | awk '{print $3'})
-	  # in case of Error or Init:Error
-    # we check that another instance is not already Completed or Running
-    if [ $status = "Error" ]  || [ $status = "Init:Error" ];then
-        echo "$i in Status Error or Init Error found for the pods, is is really true...."
-        # By default pod naming is similar, keep only the root to check
-        root_name=${i::-6}
-        kubectl get pods -n onap | grep $root_name | grep Completed
-        if [ $? ];then
-            echo "Instance Completed found."
-        else
-            echo "No Completed instance found."
-            list_filtered_failed_pods+=$i,
-        fi
-    else
-        # Other status are not running/not completed pods
-        list_filtered_failed_pods+=$i,
-    fi
-done
-
-nice_list=${list_filtered_failed_pods::-1}
-
-IFS=,
-nb_pods_not_running=$(echo "$list_filtered_failed_pods" | tr -cd , | wc -c)
-
-if [ $nb_pods_not_running -ne 0 ]; then
-echo "$nb_pods_not_running pods (on $nb_pods) are not in Running state"
-echo "---------------------------------------------------------------------"
-    kubectl get pods -n onap | grep -v Running | grep -v functest | grep -v Completed
-    echo "--------------------------------------------------------------------"
-    echo "Describe non running pods"
-    echo "*************************"
-    for i in $nice_list;do
-        echo "****************************************************************"
-        kubectl describe pod $i -n onap
-        kubectl logs --all-containers=true -n onap $i
-    done
-    code=1
-else
-    echo "all pods ($nb_pods) are running well"
-fi
-
-echo "------------------------------------------------"
-echo "------- ONAP kubernetes tests ------------------"
-echo "------------------------------------------------"
-echo ">>> Nb Pods: $nb_pods"
-echo ">>> Nb Failed Pods: $nb_pods_not_running"
-echo ">>> List of Failed Pods: [$nice_list]"
-echo "------------------------------------------------"
-echo "------------------------------------------------"
-echo "------------------------------------------------"
-
-exit $code
diff --git a/infra-healthcheck/setup.cfg b/infra-healthcheck/setup.cfg
index 4664552..f06fd45 100644
--- a/infra-healthcheck/setup.cfg
+++ b/infra-healthcheck/setup.cfg
@@ -8,7 +8,7 @@ packages = infra_healthcheck
 [entry_points]
 xtesting.testcase =
     onap_k8s = kubernetes_status.status:Status
-    onap_helm = infra_healthcheck.k8stest:OnapHelmTest
+    onap_helm = helm_onap_status.status:Status
     nodeport_ingress = infra_healthcheck.k8stest:OnapSecurityNodePortsIngress
     nodeport_check_certs = infra_healthcheck.k8stest:OnapSecurityNodePortsCerts
 
diff --git a/infra-healthcheck/upper-constraints.txt b/infra-healthcheck/upper-constraints.txt
index d37d29d..527e677 100644
--- a/infra-healthcheck/upper-constraints.txt
+++ b/infra-healthcheck/upper-constraints.txt
@@ -1 +1,3 @@
 git+https://gitlab.com/Orange-OpenSource/lfn/tools/kubernetes-status.git#egg=kubernetes_status
+git+https://gitlab.com/Orange-OpenSource/lfn/tools/helm-onap-status.git#egg=helm_onap_status
+