Refactore SDNC/R robot test suite for K-Release

add improvements SDNC ready state detection

Issue-ID: SDNC-1682
Signed-off-by: Alexander Dehn <alexander.dehn@highstreet-technologies.com>
Change-Id: I6b9829b5187f9c27ff9c2fab6b9a7381ce563b25

Former-commit-id: 83da3b4380d992d5891ed8e35b305358db9af4ff

14 files changed, 190 insertions, 129 deletions

diff --git a/csit/plans/sdnr/setup.sh b/csit/plans/sdnr/setup.sh
index 5fa5f7ba..8f0d2963 100755
--- a/csit/plans/sdnr/setup.sh
+++ b/csit/plans/sdnr/setup.sh
@@ -36,6 +36,15 @@ onap_dependent_components_launch
 nts_networkfunctions_launch ${WORKSPACE}/plans/sdnr/testdata/nts-networkfunctions.csv
 sdnr_launch
 
+## environment for SDNC/R specific robot test runs
+## Ready state will be checked every SDNC_READY_RETRY_PERIOD seconds
+# SDNC_READY_RETRY_PERIOD=15
+## SDNC ready state will be checked max SDNC_READY_TIMEOUT seconds
+# SDNC_READY_TIMEOUT=450
+
+# if jenkins should be ok without running robots TCS's
+# SDNC_RELEASE_WITHOUT_ROBOT=true
+
 #Pass any variables required by Robot test suites in ROBOT_VARIABLES
 ROBOT_VARIABLES="--variablefile=${WORKSPACE}/plans/sdnr/testdata/localhost.py"
 ROBOT_IMAGE="hightec/sdnc-test-lib:v0.10.0"
diff --git a/csit/plans/sdnr/testdata/nts-networkfunctions.csv b/csit/plans/sdnr/testdata/nts-networkfunctions.csv
index c2ccb2d6..0b0970d0 100644
--- a/csit/plans/sdnr/testdata/nts-networkfunctions.csv
+++ b/csit/plans/sdnr/testdata/nts-networkfunctions.csv
@@ -1,7 +1,6 @@
-NAME,NTS_NF_DOCKER_REPOSITORY,NTS_NF_IMAGE_NAME,NTS_NF_IMAGE_TAG,NTSFUNC-IP,NTS_HOST_NETCONF_SSH_BASE_PORT,NTS_HOST_NETCONF_TLS_BASE_PORT,NTS_NF_SSH_CONNECTIONS,NTS_NF_TLS_CONNECTIONS
-ONF-CORE-1-2,docker.io/hightec/,nts-ng-onf-core-1-2,1.3.4,172.40.0.30,30000,30500,1,1
-ONF-CORE-1-4,docker.io/hightec/,nts-ng-onf-core-1-4,1.3.4,172.40.0.31,31000,31500,1,1
-OPENROADM-6-1-0,docker.io/hightec/,nts-ng-openroadm-6.1.0,1.3.4,172.40.0.36,36000,36500,1,1
-O-RAN-FH,docker.io/hightec/,nts-ng-o-ran-fh,1.3.4,172.40.0.40,40000,40500,1,1
-X-RAN,docker.io/hightec/,nts-ng-x-ran,1.3.4,172.40.0.42,42000,42500,1,1
-
+NAME,NTS_NF_DOCKER_REPOSITORY,NTS_NF_IMAGE_NAME,NTS_NF_IMAGE_TAG,NTSFUNC-IP,NTSFUNC-IPv6,NTS_HOST_NETCONF_SSH_BASE_PORT,NTS_HOST_NETCONF_TLS_BASE_PORT,NTS_NF_SSH_CONNECTIONS,NTS_NF_TLS_CONNECTIONS,PORT,NETCONF_HOST,USER,PASSWORD,NTS_FUNCTION_TYPE
+ONF-CORE-1-2,docker.io/hightec/,nts-ng-onf-core-1-2,1.3.4,172.40.0.30,2001:db8:1:50:172:50:0:30,30000,30500,1,1,830,172.40.0.1,netconf,netconf!,nts-onf-pac:NTS_FUNCTION_TYPE_ONF_CORE_MODEL_1_2
+ONF-CORE-1-4,docker.io/hightec/,nts-ng-onf-core-1-4,1.3.4,172.40.0.31,2001:db8:1:50:172:50:0:31,31000,31500,1,1,830,172.40.0.1,netconf,netconf!,nts-onf-pac:NTS_FUNCTION_TYPE_ONF_CORE_MODEL_1_4
+OPENROADM-6-1-0,docker.io/hightec/,nts-ng-openroadm-6.1.0,1.3.4,172.40.0.36,2001:db8:1:50:172:50:0:36,36000,36500,1,1,830,172.40.0.1,netconf,netconf!,nts-openroadm-pac:NTS_FUNCTION_TYPE_OPENROADM_6_1_0
+O-RAN-FH,docker.io/hightec/,nts-ng-o-ran-fh,1.3.4,172.40.0.40,2001:db8:1:50:172:50:0:40,40000,40500,1,1,830,172.40.0.1,netconf,netconf!,nts-o-ran-fh-pac:NTS_FUNCTION_TYPE_O_RAN_FH
+X-RAN,docker.io/hightec/,nts-ng-x-ran,1.3.4,172.40.0.42,2001:db8:1:50:172:50:0:42,42000,42500,1,1,830,172.40.0.1,netconf,netconf!,nts-x-ran-pac:NTS_FUNCTION_TYPE_X_RAN
diff --git a/csit/plans/sdnr/testplan.txt b/csit/plans/sdnr/testplan.txt
index 08ec3550..dcbd829a 100644
--- a/csit/plans/sdnr/testplan.txt
+++ b/csit/plans/sdnr/testplan.txt
@@ -1,9 +1,8 @@
 # Test suites are relative paths under [integration/csit.git]/tests/.
 # Place the suites in run order.
 
-## To skip sdnr robot test suite uncomment this test case
-#sdnr/debug/10_dummy.robot
 
-## To skip sdnr robot test suite comment this test case
+## To skip sdnr robot test suite 
+## set in .setup.sh SDNC_RELEASE_WITHOUT_ROBOT=true
 sdnr/healthcheck
-#sdnr/functional
+sdnr/functional
diff --git a/csit/run-csit.sh b/csit/run-csit.sh
index 32b0fa54..c4c4ab71 100755
--- a/csit/run-csit.sh
+++ b/csit/run-csit.sh
@@ -199,18 +199,29 @@ echo ROBOT_VARIABLES="${ROBOT_VARIABLES}"
 echo "Starting Robot test suites ${SUITES} ..."
 relax_set
 
-# Runs an alternative robotframework setup as docker image in $ROBOT_IMAGE
-# test suites will be executed within this docker container
-# and results are stored as usual
-if [[ -z $ROBOT_IMAGE ]]; then
-    echo "*** TRACE **** python is $(which python) [version $(python --version)]"
-    env
-    python -m robot.run -N ${TESTPLAN} -v WORKSPACE:/tmp ${ROBOT_VARIABLES} ${TESTOPTIONS} ${SUITES}
+if [[ -z $SDNC_RELEASE_WITHOUT_ROBOT ]] ; then
+    if [[ -z $SDNC_READY_STATE_TIME_OUT ]] ; then
+        # Runs an alternative robotframework setup as docker image in $ROBOT_IMAGE
+        # test suites will be executed within this docker container
+        # and results are stored as usual
+        if [[ -z $ROBOT_IMAGE ]]; then
+            echo "*** TRACE **** python is $(which python) [version $(python --version)]"
+            env
+            python -m robot.run -N ${TESTPLAN} -v WORKSPACE:/tmp ${ROBOT_VARIABLES} ${TESTOPTIONS} ${SUITES}
+        else
+            echo "*** TRACE **** python is running in a container"
+            docker run --net="host" \
+            -v ${WORKSPACE}:${WORKSPACE} -v ${WORKDIR}:${WORKDIR} $ROBOT_IMAGE  \
+            python3 -B -m robot.run -N ${TESTPLAN} -v WORKSPACE:/tmp --outputdir ${WORKDIR} ${ROBOT_VARIABLES} ${TESTOPTIONS} ${SUITES}
+        fi
+    else
+            echo "[INFO] Skip Robot test suite, because SDNC is not in ready state"
+            echo "[ERROR] SDNC is not in ready state, check karaf.log!"
+            false
+    fi
 else
-    echo "*** TRACE **** python is running in a container"
-    docker run --net="host" \
-    -v ${WORKSPACE}:${WORKSPACE} -v ${WORKDIR}:${WORKDIR} $ROBOT_IMAGE  \
-    python3 -B -m robot.run -N ${TESTPLAN} -v WORKSPACE:/tmp --outputdir ${WORKDIR} ${ROBOT_VARIABLES} ${TESTOPTIONS} ${SUITES}
+   echo "[WARNING] Robot test suite is skipped, but job remains ok. "
+   true
 fi
 RESULT=$?
 load_set
diff --git a/csit/scripts/sdnr/docker-compose/.env b/csit/scripts/sdnr/docker-compose/.env
index be8448bc..a2bea441 100644
--- a/csit/scripts/sdnr/docker-compose/.env
+++ b/csit/scripts/sdnr/docker-compose/.env
@@ -47,4 +47,4 @@ VESCOLLECTOR_IP=172.40.0.90
 NTS_NF_SDN_CONTROLLER_IP=127.0.0.1
 NTS_NF_SDN_CONTROLLER_PORT=8181
 NTS_HOST_IP=172.40.0.1
-
+NTS_NF_MOUNT_POINT_ADDRESSING_METHOD=docker-mapping
diff --git a/csit/scripts/sdnr/docker-compose/docker-compose-nts-networkfunction.yaml b/csit/scripts/sdnr/docker-compose/docker-compose-nts-networkfunction.yaml
index 05ab6a98..a4ae7feb 100644
--- a/csit/scripts/sdnr/docker-compose/docker-compose-nts-networkfunction.yaml
+++ b/csit/scripts/sdnr/docker-compose/docker-compose-nts-networkfunction.yaml
@@ -1,18 +1,34 @@
+# *******************************************************************************
+# * ============LICENSE_START========================================================================
+# * Copyright (C) 2021 highstreet technologies GmbH Intellectual Property. All rights reserved.
+# * =================================================================================================
+# * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# * in compliance with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software distributed under the License
+# * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# * or implied. See the License for the specific language governing permissions and limitations under
+# * the License.
+# * ============LICENSE_END==========================================================================
 version: "3"
 services:
   nts-function:
     image: ${NTS_NF_DOCKER_REPOSITORY}${NTS_NF_IMAGE_NAME}:${NTS_NF_IMAGE_TAG}
     container_name: ${NTS_NF_IMAGE_NAME}
     stop_grace_period: 5m
+    cap_add:
+      - NET_ADMIN
     ports:
       - "::${NTS_HOST_NETCONF_SSH_BASE_PORT}-${NTS_HOST_NETCONF_SSH_BASE_PORT_PLUS_SSH_CON}:${EXPOSE_PORT_SSH}-${EXPOSE_PORT_SSH_PLUS_CON}"
       - "::${NTS_HOST_NETCONF_TLS_BASE_PORT}-${NTS_HOST_NETCONF_TLS_BASE_PORT_PLUS_TLS_CON}:${EXPOSE_PORT_TLS}-${EXPOSE_PORT_TLS_PLUS_CON}"
     environment:
-      NTS_NF_STANDALONE_START_FEATURES: "datastore-populate ves-heartbeat ves-pnf-registration web-cut-through manual-notification-generation netconf-call-home"
-      NTS_NF_MOUNT_POINT_ADDRESSING_METHOD: "host-mapping"
+      NTS_NF_STANDALONE_START_FEATURES: "datastore-populate ves-heartbeat ves-pnf-registration web-cut-through manual-notification-generation"
+      NTS_NF_MOUNT_POINT_ADDRESSING_METHOD: ${NTS_NF_MOUNT_POINT_ADDRESSING_METHOD} # mapping types: host-mapping, docker-mapping
       NTS_HOST_IP: ${NTS_HOST_IP}
       HOSTNAME: ${NTS_NF_CONTAINER_NAME}
-      IPv6Enabled: ${IPV6_ENABLED}
+      IPv6_ENABLED: ${IPV6_ENABLED}
 
       SSH_CONNECTIONS: ${NTS_NF_SSH_CONNECTIONS}
       TLS_CONNECTIONS: ${NTS_NF_TLS_CONNECTIONS}
diff --git a/csit/scripts/sdnr/docker-compose/nts-networkfunctions-launch.sh b/csit/scripts/sdnr/docker-compose/nts-networkfunctions-launch.sh
index 1f0939c6..3b1e8265 100755
--- a/csit/scripts/sdnr/docker-compose/nts-networkfunctions-launch.sh
+++ b/csit/scripts/sdnr/docker-compose/nts-networkfunctions-launch.sh
@@ -15,6 +15,7 @@
 # * ============LICENSE_END==========================================================================
 
 set -o xtrace
+set +e
 csvfile=$1
 export DOCKER_ENGINE_VERSION=$(docker version --format '{{.Server.APIVersion}}')
 
@@ -34,7 +35,9 @@ fi
 
 firstline=0
 # read each line of nts-networkfunctions.csv and put in into the corresponding variables
-while IFS=',' read NAME NTS_NF_DOCKER_REPOSITORY NTS_NF_IMAGE_NAME NTS_NF_IMAGE_TAG NTS_NF_IP NTS_HOST_NETCONF_SSH_BASE_PORT NTS_HOST_NETCONF_TLS_BASE_PORT NTS_NF_SSH_CONNECTIONS NTS_NF_TLS_CONNECTIONS; do
+while IFS=',' read NAME NTS_NF_DOCKER_REPOSITORY NTS_NF_IMAGE_NAME NTS_NF_IMAGE_TAG NTS_NF_IP NTS_NF_IPv6 \
+                   NTS_HOST_NETCONF_SSH_BASE_PORT NTS_HOST_NETCONF_TLS_BASE_PORT NTS_NF_SSH_CONNECTIONS NTS_NF_TLS_CONNECTIONS \
+                   PORT NETCONF_HOST USER PASSWORD NTS_FUNCTION_TYPE; do
     if [ $firstline -eq 0 ]; then
         firstline=1
         continue
@@ -45,9 +48,11 @@ while IFS=',' read NAME NTS_NF_DOCKER_REPOSITORY NTS_NF_IMAGE_NAME NTS_NF_IMAGE_
         if [[ -z ${USE_DEFAULT_REPO} ]]; then
         export NTS_NF_DOCKER_REPOSITORY=$NTS_NF_DOCKER_REPOSITORY
     fi
+    export NAME=$NAME
     export NTS_NF_IMAGE_NAME=$NTS_NF_IMAGE_NAME
     export NTS_NF_IMAGE_TAG=$NTS_NF_IMAGE_TAG
     export NTS_NF_IP=$NTS_NF_IP
+    export NTS_NF_IPv6=$NTS_NF_IPv6
     export NTS_HOST_NETCONF_SSH_BASE_PORT=$NTS_HOST_NETCONF_SSH_BASE_PORT
     export NTS_HOST_NETCONF_TLS_BASE_PORT=$NTS_HOST_NETCONF_TLS_BASE_PORT
     export NTS_HOST_NETCONF_SSH_BASE_PORT_PLUS_SSH_CON=$(expr $NTS_HOST_NETCONF_SSH_BASE_PORT + $NTS_NF_SSH_CONNECTIONS - 1)
@@ -71,3 +76,4 @@ while IFS=',' read NAME NTS_NF_DOCKER_REPOSITORY NTS_NF_IMAGE_NAME NTS_NF_IMAGE_
     docker-compose -p ${NAME} --env-file $CUR_PATH/.env -f $CUR_PATH/docker-compose-nts-networkfunction.yaml up -d
 done <$csvfile
 docker ps -a --format "table |{{.Names}}\t|{{.Image}}\t|{{printf \"%.70s\" .Ports}}|"| { head -1; sort --field-separator='|' -k 4;}
+set -e
diff --git a/csit/scripts/sdnr/sdnr-launch.sh b/csit/scripts/sdnr/sdnr-launch.sh
index 651b8f44..39ca8081 100755
--- a/csit/scripts/sdnr/sdnr-launch.sh
+++ b/csit/scripts/sdnr/sdnr-launch.sh
@@ -47,6 +47,7 @@ fi
 
 HOST_IP=$(ip route get 8.8.8.8 | awk '/8.8.8.8/ {print $7}')
 SDNC_WEB_PORT=${SDNC_WEB_PORT:-8282}
+SDNC_PORT=${SDNC_PORT:-8181}
 
 env_file="--env-file ${SCRIPTS}/sdnr/docker-compose/.env"
 echo $env_file
@@ -62,6 +63,29 @@ else
 fi
 docker ps -a
 
+# Call function with port like:
+# check_for_ready_state 1234
+# export SDNC_READY_RETRY_PERIOD and SDNC_READY_TIMEOUT to overwrite default values
+function check_for_ready_state() {
+  sdnc_port=${1}
+  SDNC_READY_RETRY_PERIOD=${SDNC_READY_RETRY_PERIOD:-15}
+  SDNC_READY_TIMEOUT=${SDNC_READY_TIMEOUT:-450}
+  max_retry=$(expr $SDNC_READY_TIMEOUT / $SDNC_READY_RETRY_PERIOD)
+  for ((i=1; i<=$max_retry; i++)); do
+    wait_time=$(expr $i \* $SDNC_READY_RETRY_PERIOD)
+    echo "[INFO] SDNC/R container not yet ready after: $wait_time seconds, Timeout: $SDNC_READY_TIMEOUT seconds"
+    curl -sS -m 1 -k -D - ${HTTPS_PREFIX}${HOST_IP}:${sdnc_port}/ready | grep 200 && break
+    if [ $i == $max_retry ]; then
+        echo "[ERROR] SDNC/R container not ready after ${SDNC_READY_TIMEOUT} seconds!"
+        export SDNC_READY_STATE_TIME_OUT=true
+        return
+    fi
+    echo sleep ${SDNC_READY_RETRY_PERIOD}
+    sleep ${SDNC_READY_RETRY_PERIOD}
+  done
+  echo "[INFO] SDNC/R container ready after: $wait_time seconds"
+}
+
 function onap_dependent_components_launch() {
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-onap-addons.yaml pull
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-onap-addons.yaml up -d
@@ -92,8 +116,6 @@ function sdnr_launch() {
     else
         sdnr_launch_single_node $1
     fi
-    cd $WORKSPACE
-    ./getAllInfo.sh -c sdnr -kp
 }
 
 
@@ -103,38 +125,23 @@ function sdnr_launch_single_node() {
     #docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
     #                         -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
     #                         pull
-    docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
-                             pull
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
                              -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
                              up -d
-         for i in {1..50}; do
-             curl -sS -m 1 -D - ${HOST_IP}:8181/ready | grep 200 && break
-             echo sleep $i
-             sleep $i
-             if [ $i == 50 ]; then
-                echo "[ERROR] SDNC/R container not ready"
-                docker ps -a
-                docker logs sdnr
-                # exit 1
-             fi
-         done
+    check_for_ready_state ${SDNC_PORT}
 }
 
 function sdnr_web_launch() {
-    docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
-                             -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr-web.override.yaml \
-                             -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
-                             pull
+    # Use locally build sdnr, sdnc-web .. no need to pull
+    #docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
+    #                         -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr-web.override.yaml \
+    #                         -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
+    #                         pull
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr.yaml \
                              -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose-single-sdnr-web.override.yaml \
                              -f ${WORKSPACE}/scripts/sdnr/docker-compose/$sdnrdb_compose_file \
                              up -d
-         for i in {1..50}; do
-             curl -sS -m 1 -D - ${HOST_IP}:${SDNC_WEB_PORT}/ready | grep 200 && break
-             echo sleep $i
-             sleep $i
-    done
+    check_for_ready_state ${SDNC_WEB_PORT}
 }
 
 function sdnr_launch_cluster() {
@@ -142,13 +149,7 @@ function sdnr_launch_cluster() {
     SDNRDM="false"
     [[ -n "$1" ]]  && SDNRDM="true" && echo "SDNRDM arg detected - running in headless mode"
     echo "SDNR being launched in Cluster mode"
-    docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose/cluster-sdnr.yaml pull
+    #docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose/cluster-sdnr.yaml pull
     docker-compose $env_file -f ${WORKSPACE}/scripts/sdnr/docker-compose/docker-compose/cluster-sdnr.yaml up -d
-    # Wait for initialization of docker services. At the moment its the master SDNR node
-         HOST_IP=$(ip route get 8.8.8.8 | awk '/8.8.8.8/ {print $7}')
-         for i in {1..50}; do
-             curl -sS -m 1 -D - ${HOST_IP}:${ODLUXPORT}/ready | grep 200 && break
-             echo sleep $i
-             sleep $i
-         done
+    check_for_ready_state ${SDNC_WEB_PORT}
 }
diff --git a/csit/scripts/sdnr/sdnr-teardown.sh b/csit/scripts/sdnr/sdnr-teardown.sh
index eeb729e0..02e8beeb 100755
--- a/csit/scripts/sdnr/sdnr-teardown.sh
+++ b/csit/scripts/sdnr/sdnr-teardown.sh
@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 # Modifications copyright (c) 2018 AT&T Intellectual Property
+# Modifications copyright (C) 2022 highstreet technologies GmbH Intellectual Property
 #
 
 if [[ -z $WORKSPACE ]]; then
@@ -37,7 +38,7 @@ source ${SCRIPTS}/sdnr/sdnrEnv_Common.sh
 env_file="--env-file ${SCRIPTS}/sdnr/docker-compose/.env"
 
 function sdnr_teardown() {
-	running_containers=$(docker ps -aq)
+	running_containers=$(docker ps -a --format "{{.Names}}")
 	if [ -z "$running_containers" ]
 	then
 		echo "No containers to get logs from!"
@@ -50,6 +51,7 @@ function sdnr_teardown() {
 			echo "Getting logs from container $i"
 			docker logs $i >> ${WORKSPACE}/archives/getallinfo/$i.log 2>&1
 		done
+        docker cp sdnr:/opt/opendaylight/data/log/karaf.log ${WORKSPACE}/archives/getallinfo/sdnr_karaf.log
 	fi
 	echo "Starting teardown!"
 	# removes sdnrdb, sdnr AND all of the rest of the containers (--remove-orphans)
diff --git a/csit/tests/sdnr/functional/devicemanager/_templates/10_lifecycleNetconfSsh/devicemanagerlifecycle_netconf.robot b/csit/tests/sdnr/functional/devicemanager/_templates/10_lifecycleNetconfSsh/devicemanagerlifecycle_netconf.robot
index 084c1556..be94102a 100644
--- a/csit/tests/sdnr/functional/devicemanager/_templates/10_lifecycleNetconfSsh/devicemanagerlifecycle_netconf.robot
+++ b/csit/tests/sdnr/functional/devicemanager/_templates/10_lifecycleNetconfSsh/devicemanagerlifecycle_netconf.robot
@@ -21,6 +21,7 @@ Suite Setup  global suite setup    &{GLOBAL_SUITE_SETUP_CONFIG}
 Suite Teardown  global suite teardown
 
 
+
 *** Variables ***
 ${DEVICE_TYPE}  DEFINE_IN_INIT
 ${DEVICE_NAME}  robot-${DEVICE_TYPE}-sim-lifecycle
@@ -42,8 +43,8 @@ ${DEVICE_TO_DELETE}  devices
 *** Test Cases ***
 
 Add network element connection
-  [Documentation]  Add network-function to device manager 
-  ...              verify correct detection of specific device manager 
+  [Documentation]  Add network-function to device manager
+  ...              verify correct detection of specific device manager
   ...              verify correct entries in connection log
   [Tags]  smoke
 
@@ -52,7 +53,8 @@ Add network element connection
   ${start_time} =  Get Current Date  time_zone=UTC  result_format=%Y-%m-%dT%H:%M:%S.%f
   Sleep  1s  reason=insert time delay to account for time differences of container and host
   Log To Console  ${start_time}
-  ConnectApp.Add network element connection    ${DEVICE_NAME_TEST}    ${True}    ${HOST}    ${PORT}    ${USERNAME}    ${PASSWORD}
+  ConnectApp.Add network element connection    device_name=${DEVICE_NAME_TEST}    is_required=${True}
+  ...  host=${HOST}    port=${PORT}    username=${USERNAME}    password=${PASSWORD}
   Run Keyword And Continue On Failure  ConnectApp.Should Be Equal connection status until time  ${DEVICE_NAME_TEST}  Connected
   SDNCRestconfLibrary.should_be_equal_connection_status_until_time  ${DEVICE_NAME_TEST}  connected  time_in_sec=${10}
   Run Keyword And Continue On Failure  ConnectApp.should_be_equal_network_element_connection_details  ${DEVICE_NAME_TEST}
@@ -84,11 +86,11 @@ Retrieve yang capabilities from network element
   Should be True  ${is_yang_correct}  msg=Yang capabilities are different from expected list
 
 Remove network element connection
-  [Documentation]  remove network element connection from device manager 
-  ...              verify if all ressources are removed 
+  [Documentation]  remove network element connection from device manager
+  ...              verify if all ressources are removed
   ...              verify correct entries in connection log
   [Tags]  smoke
-  
+
   Sleep  1s  reason=insert time gap in log files
   ${start_time} =  Get Current Date  time_zone=UTC  result_format=%Y-%m-%dT%H:%M:%S.%f
   Sleep  1s  reason=insert time delay to account for time differences of container and host
@@ -117,7 +119,8 @@ Add network element connection wrong port
   ${start_time} =  Get Current Date  time_zone=UTC  result_format=%Y-%m-%dT%H:%M:%S.%f
   Sleep  1s  reason=insert time delay to account for time differences of container and host
   Set Test Variable  ${DEVICE_NAME_TEST}  ${DEVICE_NAME}-port-nok
-  ConnectApp.Add network element connection    ${DEVICE_NAME_TEST}    ${True}    ${HOST}    ${PORT_NOK}    ${USERNAME}    ${PASSWORD}
+  ConnectApp.Add network element connection    device_name=${DEVICE_NAME_TEST}    is_required=${True}    host=${HOST}
+  ...  port=${PORT_NOK}    username=${USERNAME}    password=${PASSWORD}
   Run Keyword And Continue On Failure  ConnectApp.Should Be Equal connection status until time  ${DEVICE_NAME_TEST}  Connecting
   SDNCRestconfLibrary.should_be_equal_connection_status_until_time  ${DEVICE_NAME_TEST}  connecting  time_in_sec=${10}
   Run Keyword And Continue On Failure  ConnectApp.should_be_equal_network_element_connection_details  ${DEVICE_NAME_TEST}
@@ -174,7 +177,8 @@ Add network element connection wrong ip
   ${start_time} =  Get Current Date  time_zone=UTC  result_format=%Y-%m-%dT%H:%M:%S.%f
   Sleep  1s  reason=insert time delay to account for time differences of container and host
   Set Test Variable  ${DEVICE_NAME_TEST}  ${DEVICE_NAME}-ip-nok
-  ConnectApp.Add network element connection    ${DEVICE_NAME_TEST}    ${True}    ${HOST_NOK}    ${PORT}    ${USERNAME}    ${PASSWORD}
+  ConnectApp.Add network element connection    device_name=${DEVICE_NAME_TEST}    is_required=${True}    host=${HOST_NOK}
+  ...  port=${PORT}    username=${USERNAME}    password=${PASSWORD}
   Run Keyword And Continue On Failure  ConnectApp.Should Be Equal connection status until time  ${DEVICE_NAME_TEST}  Connecting
   Run Keyword And Continue On Failure  SDNCRestconfLibrary.should_be_equal_connection_status_until_time  ${DEVICE_NAME_TEST}  connecting  time_in_sec=${10}
   Run Keyword And Continue On Failure  ConnectApp.should_be_equal_network_element_connection_details  ${DEVICE_NAME_TEST}
@@ -231,7 +235,8 @@ Add network element connection and change is required to false
   ${start_time} =  Get Current Date  time_zone=UTC  result_format=%Y-%m-%dT%H:%M:%S.%f
   Sleep  1s  reason=insert time delay to account for time differences of container and host
   Set Test Variable  ${DEVICE_NAME_TEST}  ${DEVICE_NAME}-required
-  ConnectApp.Add network element connection    ${DEVICE_NAME_TEST}    ${True}    ${HOST}    ${PORT}    ${USERNAME}    ${PASSWORD}
+  ConnectApp.Add network element connection    device_name=${DEVICE_NAME_TEST}    is_required=${True}    host=${HOST}
+  ...  port=${PORT}    username=${USERNAME}    password=${PASSWORD}
   Run Keyword And Continue On Failure  ConnectApp.Should Be Equal connection status until time  ${DEVICE_NAME_TEST}  Connected
   SDNCRestconfLibrary.should_be_equal_connection_status_until_time  ${DEVICE_NAME_TEST}  connected  time_in_sec=${10}
   Run Keyword And Continue On Failure  ConnectApp.should_be_equal_network_element_connection_details  ${DEVICE_NAME_TEST}
@@ -337,6 +342,19 @@ Mount network element
   Run Keyword And Continue On Failure  Dictionary Should Contain Item  ${conn_status_list_stats}  Connected  1  msg=wrong connection log entries for Connected state
   Run Keyword And Continue On Failure  Dictionary Should Contain Item  ${conn_status_list_stats}  Mounted  1    msg=wrong connection log entries for Mounted state
 
+Mount Nts Network Function with VALID TLS Key ID
+  IF    'DOCKER_TLS_PORT' in ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']}
+       Run Keyword And Continue On Failure  Add Network Element Connection     device_name=${DEVICE_NAME}_sim_key_0
+                                  ...  is_required=${True}
+                                  ...  host=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['IP']}
+                                  ...  port=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['DOCKER_TLS_PORT']}
+                                  ...  username=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['USER']}
+                                  ...  tls_key=ODL_private_key_0
+                                  ...  check_connection_status=Connected
+                                  ...  time_to_wait=60
+       ConnectApp.remove_network_element_connection    ${DEVICE_NAME}_sim_key_0
+  END
+
 Remove network element connection
   [Tags]  prio2
   Sleep  1s  reason=insert time gap in log files
@@ -348,7 +366,7 @@ Remove network element connection
   Run Keyword And Continue On Failure  SDNCRestconfLibrary.should_be_equal_connection_status_until_time  ${DEVICE_NAME_TEST}  not existing  time_in_sec=${10}
 
   # Check connection status log entries
-  Sleep  1s  reason=insert time gap to avoid time constrains
+  Sleep  5s  reason=insert time gap to avoid time constrains
   ${connection_status_list} =  FaultManagementApp.get_connection_log_list  node-id=${DEVICE_NAME_TEST}
                                                                      ...   timestamp=>=${start_time}
   Log  ${connection_status_list}
@@ -370,7 +388,8 @@ Remove unmounted network element connection
   ${start_time} =  Get Current Date  time_zone=UTC  result_format=%Y-%m-%dT%H:%M:%S.%f
   Sleep  1s  reason=insert time delay to account for time differences of container and host
   Set Test Variable  ${DEVICE_NAME_TEST}  ${DEVICE_NAME}-required-true
-  ConnectApp.Add network element connection    ${DEVICE_NAME_TEST}    ${True}    ${HOST}    ${PORT}    ${USERNAME}    ${PASSWORD}
+  ConnectApp.Add network element connection    device_name=${DEVICE_NAME_TEST}    is_required=${True}    host=${HOST}
+  ...  port=${PORT}    username=${USERNAME}    password=${PASSWORD}
   Run Keyword And Continue On Failure  ConnectApp.Should Be Equal connection status until time  ${DEVICE_NAME_TEST}  Connected
   Run Keyword And Continue On Failure  SDNCRestconfLibrary.should_be_equal_connection_status_until_time  ${DEVICE_NAME_TEST}  connected  time_in_sec=${10}
 
@@ -398,4 +417,3 @@ Remove unmounted network element connection
   Run Keyword And Continue On Failure  Dictionary Should Contain Key  ${conn_status_list_stats}  Unmounted  msg=no connection log entries for Unmounted state
   ConnectApp.Remove Network Element Connection    ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}
 
-
diff --git a/csit/tests/sdnr/functional/devicemanager/_templates/20_pnfRegistration/pnfRegistration.robot b/csit/tests/sdnr/functional/devicemanager/_templates/20_pnfRegistration/pnfRegistration.robot
index 3fadf223..669ceb01 100644
--- a/csit/tests/sdnr/functional/devicemanager/_templates/20_pnfRegistration/pnfRegistration.robot
+++ b/csit/tests/sdnr/functional/devicemanager/_templates/20_pnfRegistration/pnfRegistration.robot
@@ -29,10 +29,10 @@ ${PNF_REGISTRATION_TIMEOUT}  180
 Setup NTS function
   [Tags]  nts  bringup
   [Documentation]  configure NTS manager to support restconf registration
-  Add Network Element Connection   ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}    ${True}
-  ...  ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['IP']}     ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PORT']}
-  ...  ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['USER']}    ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PASSWORD']}
-  ...  Connected
+  Add Network Element Connection   device_name=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}    is_required=${True}
+  ...  host=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['IP']}     port=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PORT']}
+  ...  username=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['USER']}    password=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PASSWORD']}
+  ...  check_connection_status=Connected
   SDNCRestconfLibrary.Should Be Equal Connection Status Until Time    ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}    Connected
 
 Start pnf ves registration from NTS function
diff --git a/csit/tests/sdnr/functional/devicemanager/_templates/21_alarmNotificationVES/alarmNotificationVes.robot b/csit/tests/sdnr/functional/devicemanager/_templates/21_alarmNotificationVES/alarmNotificationVes.robot
index 068a6b89..842a3d82 100644
--- a/csit/tests/sdnr/functional/devicemanager/_templates/21_alarmNotificationVES/alarmNotificationVes.robot
+++ b/csit/tests/sdnr/functional/devicemanager/_templates/21_alarmNotificationVES/alarmNotificationVes.robot
@@ -34,10 +34,10 @@ ${PROCESS_TIME_NOTIF}  30s
 Setup NTS function
   [Tags]  nts  bringup
   [Documentation]  add network function to trigger alarm notification via VES in next tests
-  Add Network Element Connection   ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}    ${True}
-  ...  ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['IP']}     ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PORT']}
-  ...  ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['USER']}    ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PASSWORD']}
-  ...  Connected
+  Add Network Element Connection   device_name=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}  is_required=${True}
+  ...  host=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['IP']}     port=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PORT']}
+  ...  username=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['USER']}    password=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PASSWORD']}
+  ...  check_connection_status=Connected
   SDNCRestconfLibrary.Should Be Equal Connection Status Until Time    ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}    Connected
 
 Set alarm notification
diff --git a/csit/tests/sdnr/functional/devicemanager/_templates/22_alarmNotificationNETCONF/alarmNotificationNetconf.robot b/csit/tests/sdnr/functional/devicemanager/_templates/22_alarmNotificationNETCONF/alarmNotificationNetconf.robot
index df023c41..b0b8f0e1 100644
--- a/csit/tests/sdnr/functional/devicemanager/_templates/22_alarmNotificationNETCONF/alarmNotificationNetconf.robot
+++ b/csit/tests/sdnr/functional/devicemanager/_templates/22_alarmNotificationNETCONF/alarmNotificationNetconf.robot
@@ -43,10 +43,10 @@ ${TIME_PERIOD_SEND_NOTIF}  22s
 Setup NTS function
   [Tags]  nts  bringup
   [Documentation]  configure NTS manager to support restconf registration
-  Add Network Element Connection   ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}    ${True}
-  ...  ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['IP']}     ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PORT']}
-  ...  ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['USER']}    ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PASSWORD']}
-  ...  Connected
+  Add Network Element Connection   device_name=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}  is_required=${True}
+  ...  host=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['IP']}     port=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PORT']}
+  ...  username=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['USER']}    password=${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['PASSWORD']}
+  ...  check_connection_status=Connected
   SDNCRestconfLibrary.Should Be Equal Connection Status Until Time    ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}    Connected
 
 Set alarm notification
@@ -63,7 +63,7 @@ Set alarm notification
   ${alarm_status_start} =  FaultManagementApp.get_alarm_status
   Set Global Variable  ${alarm_status_start}
   NTSimManagerNG.set_fault_delay_list_nf  ${NETWORK_FUNCTIONS['${DEVICE_TYPE}']['NAME']}  delay-period=${fault_delay}
-  
+
   Log  Send notification every ${FAULT_DELAY} sec for ${TIME_PERIOD_SEND_NOTIF}  level=INFO  html=False  console=True  repr=False
   Sleep  ${TIME_PERIOD_SEND_NOTIF}
 
diff --git a/csit/tests/sdnr/healthcheck/20_healthcheckSUT.robot b/csit/tests/sdnr/healthcheck/20_healthcheckSUT.robot
index e11fbf03..2d29c9f0 100644
--- a/csit/tests/sdnr/healthcheck/20_healthcheckSUT.robot
+++ b/csit/tests/sdnr/healthcheck/20_healthcheckSUT.robot
@@ -1,44 +1,44 @@
-*** Settings ***

-Documentation  healthcheck of system under test: sdnc server, sdnrdb are available

-Library  ConnectLibrary

-Library  SDNCBaseLibrary

-Library  Collections

-Library  ElasticSearchLibrary

-Library  ConnectApp

-Library  RequestsLibrary

-

-Suite Setup  global suite setup    &{GLOBAL_SUITE_SETUP_CONFIG}

-Suite Teardown  global suite teardown

-

-*** Variables ***

-&{headers}  Content-Type=application/json  Authorization=Basic

-

-*** Test Cases ***

-Test Is SDNR Node Available

-    ${server_status}=    Server Is Ready

-    should be true    ${server_status}

-

-Test Is SDNRDB Available

-    ${es_version_info}=    Get Sdnrdb Version Info As Dict

-    ${length_of_response}=    Get Length    ${es_version_info}

-    should be true    ${length_of_response}>${0}

-

-Test Is SDNRDB Initialized

-    ${res}=  Check Aliases

-    Log  ${res}  level=INFO  html=False  console=False  repr=False

-

-Test Is VES Collector available

-    # curl -k -u sample1:sample1 https://172.40.0.1:8443

-    ${auth}=  Create List  ${VESCOLLECTOR}[USERNAME]  ${VESCOLLECTOR}[PASSWORD]

-    ${IPV6_ENABLED}=  Get Variable Value    ${ENABLE_IPV6}  ${False}

-    Log To Console    ${VESCOLLECTOR}[SCHEME]://[${VESCOLLECTOR}[IP]]:${VESCOLLECTOR}[PORT]

-    IF    ${IPV6_ENABLED} != ${True}

-        RequestsLibrary.Create Session  alias=ves  url=${VESCOLLECTOR}[SCHEME]://${VESCOLLECTOR}[IP]:${VESCOLLECTOR}[PORT]  headers=${headers}  auth=${auth}

-    ELSE

-        RequestsLibrary.Create Session  alias=ves  url=${VESCOLLECTOR}[SCHEME]://[${VESCOLLECTOR}[IP]]:${VESCOLLECTOR}[PORT]  headers=${headers}  auth=${auth}

-    END

-    ${resp}=  RequestsLibrary.GET On Session  ves  /

-    Should Be Equal As Strings  ${resp.text}  Welcome to VESCollector

-    Should Be Equal As Strings  ${resp.status_code}  200

-    RequestsLibrary.Delete All Sessions

-

+*** Settings ***
+Documentation  healthcheck of system under test: sdnc server, sdnrdb are available
+Library  ConnectLibrary
+Library  SDNCBaseLibrary
+Library  Collections
+Library  ElasticSearchLibrary
+Library  ConnectApp
+Library  RequestsLibrary
+
+Suite Setup  global suite setup    &{GLOBAL_SUITE_SETUP_CONFIG}
+Suite Teardown  global suite teardown
+
+*** Variables ***
+&{headers}  Content-Type=application/json  Authorization=Basic
+
+*** Test Cases ***
+Test Is SDNR Node Available
+    ${server_status}=    Server Is Ready
+    should be true    ${server_status}
+
+Test Is SDNRDB Available
+    ${es_version_info}=    Get Sdnrdb Version Info As Dict
+    ${length_of_response}=    Get Length    ${es_version_info}
+    should be true    ${length_of_response}>${0}
+
+Test Is SDNRDB Initialized
+    ${res}=  Check Aliases
+    Log  ${res}  level=INFO  html=False  console=False  repr=False
+
+Test Is VES Collector available
+    # curl -k -u sample1:sample1 https://172.40.0.1:8443
+    ${auth}=  Create List  ${VESCOLLECTOR}[USERNAME]  ${VESCOLLECTOR}[PASSWORD]
+    ${IPV6_ENABLED}=  Get Variable Value    ${ENABLE_IPV6}  ${False}
+    Log To Console    ${VESCOLLECTOR}[SCHEME]://[${VESCOLLECTOR}[IP]]:${VESCOLLECTOR}[PORT]
+    IF    ${IPV6_ENABLED} != ${True}
+        RequestsLibrary.Create Session  alias=ves  url=${VESCOLLECTOR}[SCHEME]://${VESCOLLECTOR}[IP]:${VESCOLLECTOR}[PORT]  headers=${headers}  auth=${auth}
+    ELSE
+        RequestsLibrary.Create Session  alias=ves  url=${VESCOLLECTOR}[SCHEME]://[${VESCOLLECTOR}[IP]]:${VESCOLLECTOR}[PORT]  headers=${headers}  auth=${auth}
+    END
+    ${resp}=  RequestsLibrary.GET On Session  ves  /
+    Should Be Equal As Strings  ${resp.text}  Welcome to VESCollector
+    Should Be Equal As Strings  ${resp.status_code}  200
+    RequestsLibrary.Delete All Sessions
+