Refactore SDNC/R robot test suite for K-Release

add improvements SDNC ready state detection

Issue-ID: SDNC-1682
Signed-off-by: Alexander Dehn <alexander.dehn@highstreet-technologies.com>
Change-Id: I6b9829b5187f9c27ff9c2fab6b9a7381ce563b25

Former-commit-id: 83da3b4380d992d5891ed8e35b305358db9af4ff

5 files changed, 63 insertions, 38 deletions

diff --git a/csit/scripts/sdnr/docker-compose/.env b/csit/scripts/sdnr/docker-compose/.env
index be8448bc..a2bea441 100644
--- a/csit/scripts/sdnr/docker-compose/.env
+++ b/csit/scripts/sdnr/docker-compose/.env
@@ -47,4 +47,4 @@ VESCOLLECTOR_IP=172.40.0.90
 NTS_NF_SDN_CONTROLLER_IP=127.0.0.1
 NTS_NF_SDN_CONTROLLER_PORT=8181
 NTS_HOST_IP=172.40.0.1
-
+NTS_NF_MOUNT_POINT_ADDRESSING_METHOD=docker-mapping
diff --git a/csit/scripts/sdnr/docker-compose/docker-compose-nts-networkfunction.yaml b/csit/scripts/sdnr/docker-compose/docker-compose-nts-networkfunction.yaml
index 05ab6a98..a4ae7feb 100644
--- a/csit/scripts/sdnr/docker-compose/docker-compose-nts-networkfunction.yaml
+++ b/csit/scripts/sdnr/docker-compose/docker-compose-nts-networkfunction.yaml
@@ -1,18 +1,34 @@
+# *******************************************************************************
+# * ============LICENSE_START========================================================================
+# * Copyright (C) 2021 highstreet technologies GmbH Intellectual Property. All rights reserved.
+# * =================================================================================================
+# * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# * in compliance with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software distributed under the License
+# * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# * or implied. See the License for the specific language governing permissions and limitations under
+# * the License.
+# * ============LICENSE_END==========================================================================
 version: "3"
 services:
   nts-function:
     image: ${NTS_NF_DOCKER_REPOSITORY}${NTS_NF_IMAGE_NAME}:${NTS_NF_IMAGE_TAG}
     container_name: ${NTS_NF_IMAGE_NAME}
     stop_grace_period: 5m
+    cap_add:
+      - NET_ADMIN
     ports:
       - "::${NTS_HOST_NETCONF_SSH_BASE_PORT}-${NTS_HOST_NETCONF_SSH_BASE_PORT_PLUS_SSH_CON}:${EXPOSE_PORT_SSH}-${EXPOSE_PORT_SSH_PLUS_CON}"
       - "::${NTS_HOST_NETCONF_TLS_BASE_PORT}-${NTS_HOST_NETCONF_TLS_BASE_PORT_PLUS_TLS_CON}:${EXPOSE_PORT_TLS}-${EXPOSE_PORT_TLS_PLUS_CON}"
     environment:
-      NTS_NF_STANDALONE_START_FEATURES: "datastore-populate ves-heartbeat ves-pnf-registration web-cut-through manual-notification-generation netconf-call-home"
-      NTS_NF_MOUNT_POINT_ADDRESSING_METHOD: "host-mapping"
+      NTS_NF_STANDALONE_START_FEATURES: "datastore-populate ves-heartbeat ves-pnf-registration web-cut-through manual-notification-generation"
+      NTS_NF_MOUNT_POINT_ADDRESSING_METHOD: ${NTS_NF_MOUNT_POINT_ADDRESSING_METHOD} # mapping types: host-mapping, docker-mapping
       NTS_HOST_IP: ${NTS_HOST_IP}
       HOSTNAME: ${NTS_NF_CONTAINER_NAME}
-      IPv6Enabled: ${IPV6_ENABLED}
+      IPv6_ENABLED: ${IPV6_ENABLED}
 
       SSH_CONNECTIONS: ${NTS_NF_SSH_CONNECTIONS}
       TLS_CONNECTIONS: ${NTS_NF_TLS_CONNECTIONS}
diff --git a/csit/scripts/sdnr/docker-compose/nts-networkfunctions-launch.sh b/csit/scripts/sdnr/docker-compose/nts-networkfunctions-launch.sh
index 1f0939c6..3b1e8265 100755
--- a/csit/scripts/sdnr/docker-compose/nts-networkfunctions-launch.sh
+++ b/csit/scripts/sdnr/docker-compose/nts-networkfunctions-launch.sh
@@ -15,6 +15,7 @@
 # * ============LICENSE_END==========================================================================
 
 set -o xtrace
+set +e
 csvfile=$1
 export DOCKER_ENGINE_VERSION=$(docker version --format '{{.Server.APIVersion}}')
 
@@ -34,7 +35,9 @@ fi
 
 firstline=0
 # read each line of nts-networkfunctions.csv and put in into the corresponding variables
-while IFS=',' read NAME NTS_NF_DOCKER_REPOSITORY NTS_NF_IMAGE_NAME NTS_NF_IMAGE_TAG NTS_NF_IP NTS_HOST_NETCONF_SSH_BASE_PORT NTS_HOST_NETCONF_TLS_BASE_PORT NTS_NF_SSH_CONNECTIONS NTS_NF_TLS_CONNECTIONS; do
+while IFS=',' read NAME NTS_NF_DOCKER_REPOSITORY NTS_NF_IMAGE_NAME NTS_NF_IMAGE_TAG NTS_NF_IP NTS_NF_IPv6 \
+                   NTS_HOST_NETCONF_SSH_BASE_PORT NTS_HOST_NETCONF_TLS_BASE_PORT NTS_NF_SSH_CONNECTIONS NTS_NF_TLS_CONNECTIONS \
+                   PORT NETCONF_HOST USER PASSWORD NTS_FUNCTION_TYPE; do
     if [ $firstline -eq 0 ]; then
         firstline=1
         continue
@@ -45,9 +48,11 @@ while IFS=',' read NAME NTS_NF_DOCKER_REPOSITORY NTS_NF_IMAGE_NAME NTS_NF_IMAGE_
         if [[ -z ${USE_DEFAULT_REPO} ]]; then
         export NTS_NF_DOCKER_REPOSITORY=$NTS_NF_DOCKER_REPOSITORY
     fi
+    export NAME=$NAME
     export NTS_NF_IMAGE_NAME=$NTS_NF_IMAGE_NAME
     export NTS_NF_IMAGE_TAG=$NTS_NF_IMAGE_TAG
     export NTS_NF_IP=$NTS_NF_IP
+    export NTS_NF_IPv6=$NTS_NF_IPv6
     export NTS_HOST_NETCONF_SSH_BASE_PORT=$NTS_HOST_NETCONF_SSH_BASE_PORT
     export NTS_HOST_NETCONF_TLS_BASE_PORT=$NTS_HOST_NETCONF_TLS_BASE_PORT
     export NTS_HOST_NETCONF_SSH_BASE_PORT_PLUS_SSH_CON=$(expr $NTS_HOST_NETCONF_SSH_BASE_PORT + $NTS_NF_SSH_CONNECTIONS - 1)
@@ -71,3 +76,4 @@ while IFS=',' read NAME NTS_NF_DOCKER_REPOSITORY NTS_NF_IMAGE_NAME NTS_NF_IMAGE_
     docker-compose -p ${NAME} --env-file $CUR_PATH/.env -f $CUR_PATH/docker-compose-nts-networkfunction.yaml up -d
 done <$csvfile
 docker ps -a --format "table |{{.Names}}\t|{{.Image}}\t|{{printf \"%.70s\" .Ports}}|"| { head -1; sort --field-separator='|' -k 4;}
+set -e
diff --git a/csit/scripts/sdnr/sdnr-launch.sh b/csit/scripts/sdnr/sdnr-launch.sh
index 651b8f44..39ca8081 100755
--- a/csit/scripts/sdnr/sdnr-launch.sh
+++ b/csit/scripts/sdnr/sdnr-launch.sh
@@ -47,6 +47,7 @@ fi
 
 HOST_IP=$(ip route get 8.8.8.8 | awk '/8.8.8.8/ {print $7}')
 SDNC_WEB_PORT=${SDNC_WEB_PORT:-8282}
+SDNC_PORT=${SDNC_PORT:-8181}
 
 env_file="--env-file ${SCRIPTS}/sdnr/docker-compose/.env"
 echo $env_file
@@ -62,6 +63,29 @@ else
 fi
 docker ps -a
 
+# Call function with port like:
+# check_for_ready_state 1234
+# export SDNC_READY_RETRY_PERIOD and SDNC_READY_TIMEOUT to overwrite default values
+function check_for_ready_state() {
+  sdnc_port=${1}
+  SDNC_READY_RETRY_PERIOD=${SDNC_READY_RETRY_PERIOD:-15}
+  SDNC_READY_TIMEOUT=${SDNC_READY_TIMEOUT:-450}
+  max_retry=$(expr $SDNC_READY_TIMEOUT / $SDNC_READY_RETRY_PERIOD)
+  for ((i=1; i<=$max_retry; i++)); do
+    wait_time=$(expr $i \* $SDNC_READY_RETRY_PERIOD)
+    echo "[INFO] SDNC/R container not yet ready after: $wait_time seconds, Timeout: $SDNC_READY_TIMEOUT seconds"
+    curl -sS -m 1 -k -D - ${HTTPS_PREFIX}${HOST_IP}:${sdnc_port}/ready | grep 200 && break
+    if [ $i == $max_retry ]; then
+        echo "[ERROR] SDNC/R container not ready after ${SDNC_READY_TIMEOUT} seconds!"
+        export SDNC_READY_STATE_TIME_OUT=true
+        return
+    fi
+    echo sleep ${SDNC_READY_RETRY_PERIOD}
+    sleep ${SDNC_READY_RETRY_PERIOD}
+  done
+  echo "[INFO] SDNC/R container ready after: $wait_time seconds"
+}
+
 function onap_dependent_components_launch() {
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-onap-addons.yaml pull
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-onap-addons.yaml up -d
@@ -92,8 +116,6 @@ function sdnr_launch() {
     else
         sdnr_launch_single_node $1
     fi
-    cd $WORKSPACE
-    ./getAllInfo.sh -c sdnr -kp
 }
 
 
@@ -103,38 +125,23 @@ function sdnr_launch_single_node() {
     #docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
     #                         -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
     #                         pull
-    docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
-                             pull
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
                              -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
                              up -d
-         for i in {1..50}; do
-             curl -sS -m 1 -D - ${HOST_IP}:8181/ready | grep 200 && break
-             echo sleep $i
-             sleep $i
-             if [ $i == 50 ]; then
-                echo "[ERROR] SDNC/R container not ready"
-                docker ps -a
-                docker logs sdnr
-                # exit 1
-             fi
-         done
+    check_for_ready_state ${SDNC_PORT}
 }
 
 function sdnr_web_launch() {
-    docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
-                             -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr-web.override.yaml \
-                             -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
-                             pull
+    # Use locally build sdnr, sdnc-web .. no need to pull
+    #docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
+    #                         -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr-web.override.yaml \
+    #                         -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
+    #                         pull
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
                              -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr-web.override.yaml \
                              -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
                              up -d
-         for i in {1..50}; do
-             curl -sS -m 1 -D - ${HOST_IP}:${SDNC_WEB_PORT}/ready | grep 200 && break
-             echo sleep $i
-             sleep $i
-    done
+    check_for_ready_state ${SDNC_WEB_PORT}
 }
 
 function sdnr_launch_cluster() {
@@ -142,13 +149,7 @@ function sdnr_launch_cluster() {
     SDNRDM="false"
     [[ -n "$1" ]]  && SDNRDM="true" && echo "SDNRDM arg detected - running in headless mode"
     echo "SDNR being launched in Cluster mode"
-    docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose/cluster-sdnr.yaml pull
+    #docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose/cluster-sdnr.yaml pull
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose/cluster-sdnr.yaml up -d
-    # Wait for initialization of docker services. At the moment its the master SDNR node
-         HOST_IP=$(ip route get 8.8.8.8 | awk '/8.8.8.8/ {print $7}')
-         for i in {1..50}; do
-             curl -sS -m 1 -D - ${HOST_IP}:${ODLUXPORT}/ready | grep 200 && break
-             echo sleep $i
-             sleep $i
-         done
+    check_for_ready_state ${SDNC_WEB_PORT}
 }
diff --git a/csit/scripts/sdnr/sdnr-teardown.sh b/csit/scripts/sdnr/sdnr-teardown.sh
index eeb729e0..02e8beeb 100755
--- a/csit/scripts/sdnr/sdnr-teardown.sh
+++ b/csit/scripts/sdnr/sdnr-teardown.sh
@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 # Modifications copyright (c) 2018 AT&T Intellectual Property
+# Modifications copyright (C) 2022 highstreet technologies GmbH Intellectual Property
 #
 
 if [[ -z $WORKSPACE ]]; then
@@ -37,7 +38,7 @@ source ${SCRIPTS}/sdnr/sdnrEnv_Common.sh
 env_file="--env-file ${SCRIPTS}/sdnr/docker-compose/.env"
 
 function sdnr_teardown() {
-	running_containers=$(docker ps -aq)
+	running_containers=$(docker ps -a --format "{{.Names}}")
 	if [ -z "$running_containers" ]
 	then
 		echo "No containers to get logs from!"
@@ -50,6 +51,7 @@ function sdnr_teardown() {
 			echo "Getting logs from container $i"
 			docker logs $i >> ${WORKSPACE}/archives/getallinfo/$i.log 2>&1
 		done
+        docker cp sdnr:/opt/opendaylight/data/log/karaf.log ${WORKSPACE}/archives/getallinfo/sdnr_karaf.log
 	fi
 	echo "Starting teardown!"
 	# removes sdnrdb, sdnr AND all of the rest of the containers (--remove-orphans)