Investigate csit failuremr/879/126934/9

Issue-ID: CPS-879
Signed-off-by: Bruno Sakoto <bruno.sakoto@bell.ca>
Change-Id: Ia9799f4ae3265cf47e46329d014055feb8b35e74

9 files changed, 120 insertions, 79 deletions

diff --git a/csit/README.md b/csit/README.md
index baa636bab4..cdf535f87e 100644
--- a/csit/README.md
+++ b/csit/README.md
@@ -68,4 +68,3 @@ Execute test from current cps folder:
 ```bash
 ./csit/run-project-csit.sh
 ```
-
diff --git a/csit/plans/cps/sdnc/sdnc_setup.sh b/csit/plans/cps/sdnc/sdnc_setup.sh
index 18d21b3889..c2b7a84f15 100644
--- a/csit/plans/cps/sdnc/sdnc_setup.sh
+++ b/csit/plans/cps/sdnc/sdnc_setup.sh
@@ -31,7 +31,7 @@ TIME_OUT=600
 INTERVAL=30
 TIME=0
 while [ "$TIME" -lt "$TIME_OUT" ]; do
-  response=$(curl --write-out '%{http_code}' --silent --output /dev/null -H "Authorization: Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==" -X POST -H "X-FromAppId: csit-sdnc" -H "X-TransactionId: csit-sdnc" -H "Accept: application/json" -H "Content-Type: application/json" http://localhost:8282/restconf/operations/SLI-API:healthcheck );
+  response=$(curl --write-out '%{http_code}' --silent --output /dev/null -H "Authorization: Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==" -X POST -H "X-FromAppId: csit-sdnc" -H "X-TransactionId: csit-sdnc" -H "Accept: application/json" -H "Content-Type: application/json" http://$SDNC_HOST:$SDNC_PORT/restconf/operations/SLI-API:healthcheck );
   echo $response
 
   if [ "$response" == "200" ]; then
diff --git a/csit/plans/cps/setup.sh b/csit/plans/cps/setup.sh
index 2fc0ec4877..e4bdd47a9e 100755
--- a/csit/plans/cps/setup.sh
+++ b/csit/plans/cps/setup.sh
@@ -23,107 +23,124 @@
 # Branched from ccsdk/distribution to this repository Feb 23, 2021
 #
 
-# Copy docker-compose.yml and application.yml to archives
-mkdir -p $WORKSPACE/archives/docker-compose
-cp $WORKSPACE/../docker-compose/*.yml $WORKSPACE/archives/docker-compose
-cd $WORKSPACE/archives/docker-compose
+check_health()
+{
+TIME_OUT=120
+INTERVAL=5
+TICKER=0
+
+while [ "$TICKER" -le "$TIME_OUT" ]; do
+
+  RESPONSE=$(curl --location --request GET 'http://'$1'/manage/health/readiness')
+
+  if [[ "$RESPONSE" == *"UP"* ]]; then
+    echo "$2 started in $TICKER"
+    break;
+  fi
+
+  sleep $INTERVAL
+  TICKER=$((TICKER + INTERVAL))
 
+done
+
+if [ "$TICKER" -ge "$TIME_OUT" ]; then
+  echo TIME OUT: $2 session not started in $TIME_OUT seconds... Could cause problems for testing activities...
+fi
+
+}
+
+###################### setup env ############################
 # Set env variables for docker compose
 export LOCAL_IP=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
-export DMI_SERVICE_URL=http://$LOCAL_IP:8783
-export DB_HOST=$LOCAL_IP
-export SDNC_HOST=$LOCAL_IP
-export CPS_CORE_HOST=$LOCAL_IP
-export DB_USERNAME=cps
-export DB_PASSWORD=cps
-# Use latest image version
-export VERSION=latest
+
+source $WORKSPACE/plans/cps/test.properties
+export $(cut -d= -f1 $WORKSPACE/plans/cps/test.properties)
+
+###################### setup cps-ncmp ############################
+mkdir -p $WORKSPACE/archives/dc-cps
+cp $WORKSPACE/../docker-compose/*.yml $WORKSPACE/archives/dc-cps
+cd $WORKSPACE/archives/dc-cps
 
 # download docker-compose of a required version (1.25.0 supports configuration of version 3.7)
 curl -L https://github.com/docker/compose/releases/download/1.25.0/docker-compose-`uname -s`-`uname -m` > docker-compose
 chmod +x docker-compose
 
 # start CPS and PostgreSQL containers with docker compose
-docker network create test_network
 ./docker-compose up -d
 
-###################### setup sdnc ############################
+###################### setup onap-dmi-plugin ############################
+
+cd $WORKSPACE/archives
+git clone "https://gerrit.onap.org/r/cps/ncmp-dmi-plugin"
+mkdir -p $WORKSPACE/archives/dc-dmi
+cp $WORKSPACE/archives/ncmp-dmi-plugin/docker-compose/*.yml $WORKSPACE/archives/dc-dmi
+cd $WORKSPACE/archives/dc-dmi
+# download docker-compose of a required version (1.25.0 supports configuration of version 3.7)
+curl -L https://github.com/docker/compose/releases/download/1.25.0/docker-compose-`uname -s`-`uname -m` > docker-compose
+chmod +x docker-compose
+./docker-compose up -d
+
+###################### setup sdnc #######################################
 source $WORKSPACE/plans/cps/sdnc/sdnc_setup.sh
 
-###################### setup pnfsim ##########################
+###################### setup pnfsim #####################################
 docker-compose -f $WORKSPACE/plans/cps/pnfsim/docker-compose.yml up -d
 
 # Allow time for netconf-pnp-simulator & SDNC to come up fully
 sleep 30s
 
+###################### mount pnf-sim as PNFDemo ##########################
 SDNC_TIME_OUT=250
 SDNC_INTERVAL=10
 SDNC_TIME=0
 
 while [ "$SDNC_TIME" -le "$SDNC_TIME_OUT" ]; do
 
-	# Mount netconf node
-
-	curl --location --request PUT 'http://'"$LOCAL_IP"':8282/restconf/config/network-topology:network-topology/topology/topology-netconf/node/PNFDemo' \
-	--header 'Authorization: Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==' \
-	--header 'Content-Type: application/json' \
-	--data-raw '{
-	  "node": [
-		{
-		  "node-id": "PNFDemo",
-		  "netconf-node-topology:protocol": {
-			"name": "TLS"
-		  },
-		  "netconf-node-topology:host": '"$LOCAL_IP"',
-		  "netconf-node-topology:key-based": {
-			"username": "netconf",
-			"key-id": "ODL_private_key_0"
-		  },
-		  "netconf-node-topology:port": 6512,
-		  "netconf-node-topology:tcp-only": false,
-		  "netconf-node-topology:max-connection-attempts": 5
-		}
-	  ]
-	 }'
-
-	# Verify node has been mounted
-
-	RESPONSE=$( curl --location --request GET 'http://'"$LOCAL_IP"':8282/restconf/config/network-topology:network-topology/topology/topology-netconf' --header 'Authorization: basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==')
-
-	  if [[ "$RESPONSE" == *"PNFDemo"* ]]; then
-	    echo "Node mounted in $SDNC_TIME"
-		  break;
-	  fi
-
-	 sleep $SDNC_INTERVAL
-	 SDNC_TIME=$((SDNC_TIME + SDNC_INTERVAL))
-
-done
+  # Mount netconf node
+  curl --location --request PUT 'http://'$SDNC_HOST:$SDNC_PORT'/restconf/config/network-topology:network-topology/topology/topology-netconf/node/PNFDemo' \
+  --header 'Authorization: Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==' \
+  --header 'Content-Type: application/json' \
+  --data-raw '{
+    "node": [
+    {
+      "node-id": "PNFDemo",
+      "netconf-node-topology:protocol": {
+      "name": "TLS"
+      },
+      "netconf-node-topology:host": "'$LOCAL_IP'",
+      "netconf-node-topology:key-based": {
+      "username": "netconf",
+      "key-id": "ODL_private_key_0"
+      },
+      "netconf-node-topology:port": 6512,
+      "netconf-node-topology:tcp-only": false,
+      "netconf-node-topology:max-connection-attempts": 5
+    }
+    ]
+  }'
+
+   # Verify node has been mounted
+
+  RESPONSE=$( curl --location --request GET 'http://'$SDNC_HOST:$SDNC_PORT'/restconf/config/network-topology:network-topology/topology/topology-netconf' --header 'Authorization: basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==')
+
+  if [[ "$RESPONSE" == *"PNFDemo"* ]]; then
+    echo "Node mounted in $SDNC_TIME"
+    break;
+  fi
 
-# Validate CPS service initialization completed via periodic log checking for line like below:
-# org.onap.cps.Application ... Started Application in X.XXX seconds
+  sleep $SDNC_INTERVAL
+  SDNC_TIME=$((SDNC_TIME + SDNC_INTERVAL))
 
-TIME_OUT=300
-INTERVAL=5
-TIME=0
+done
 
-while [ "$TIME" -le "$TIME_OUT" ]; do
-  LOG_FOUND=$( ./docker-compose logs --tail="all" | grep "org.onap.cps.Application" | egrep -c "Started Application in" )
+###################### verify ncmp-cps health ##########################
 
-  if [ "$LOG_FOUND" -gt 0 ]; then
-    echo "CPS Service started"
-    break;
-  fi
+check_health $CPS_CORE_HOST:$CPS_CORE_MANAGEMENT_PORT 'cps-ncmp'
 
-  echo "Sleep $INTERVAL seconds before next check for CPS initialization (waiting $TIME seconds; timeout is $TIME_OUT seconds)"
-  sleep $INTERVAL
-  TIME=$((TIME + INTERVAL))
-done
+###################### verify dmi health ##########################
 
-if [ "$TIME" -gt "$TIME_OUT" ]; then
-   echo "TIME OUT: CPS Service wasn't able to start in $TIME_OUT seconds, setup failed."
-   exit 1;
-fi
+check_health $DMI_HOST:$DMI_MANAGEMENT_PORT 'dmi-plugin'
 
+###################### ROBOT Configurations ##########################
 # Pass variables required for Robot test suites in ROBOT_VARIABLES
-ROBOT_VARIABLES="-v CPS_HOST:$LOCAL_IP -v CPS_PORT:8883 -v DMI_HOST:$LOCAL_IP -v DMI_PORT:8783 -v MANAGEMENT_PORT:8887 -v DATADIR:$WORKSPACE/data"
\ No newline at end of file
+ROBOT_VARIABLES="-v CPS_CORE_HOST:$CPS_CORE_HOST -v CPS_CORE_PORT:$CPS_CORE_PORT -v DMI_HOST:$LOCAL_IP -v DMI_PORT:$DMI_PORT -v CPS_CORE_MANAGEMENT_PORT:$CPS_CORE_MANAGEMENT_PORT -v DATADIR:$WORKSPACE/data"
\ No newline at end of file
diff --git a/csit/plans/cps/teardown.sh b/csit/plans/cps/teardown.sh
index 3ef50d4146..75f0af41fc 100755
--- a/csit/plans/cps/teardown.sh
+++ b/csit/plans/cps/teardown.sh
@@ -29,4 +29,3 @@ echo y | docker volume prune
 
 echo 'Removing Networks...'
 echo y | docker network prune
-
diff --git a/csit/plans/cps/test.properties b/csit/plans/cps/test.properties
new file mode 100644
index 0000000000..3e546fbec1
--- /dev/null
+++ b/csit/plans/cps/test.properties
@@ -0,0 +1,26 @@
+DB_HOST=$LOCAL_IP
+DB_USERNAME=cps
+DB_PASSWORD=cps
+
+SDNC_HOST=$LOCAL_IP
+SDNC_PORT=8282
+SDNC_USERNAME=admin
+SDNC_PASSWORD=Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U
+
+CPS_CORE_HOST=$LOCAL_IP
+CPS_CORE_PORT=8883
+CPS_CORE_MANAGEMENT_PORT=8887
+CPS_CORE_USERNAME=cpsuser
+CPS_CORE_PASSWORD=cpsr0cks!
+
+DMI_HOST=$LOCAL_IP
+DMI_PORT=8783
+DMI_USERNAME=cpsuser
+DMI_PASSWORD=cpsr0cks!
+DMI_MANAGEMENT_PORT=8787
+DMI_SERVICE_URL=http://$LOCAL_IP:$DMI_PORT
+
+DOCKER_REPO=nexus3.onap.org:10003
+
+CPS_VERSION=latest
+DMI_VERSION=latest
\ No newline at end of file
diff --git a/csit/tests/actuator/actuator.robot b/csit/tests/actuator/actuator.robot
index 55ff40e076..2ee575e406 100644
--- a/csit/tests/actuator/actuator.robot
+++ b/csit/tests/actuator/actuator.robot
@@ -23,7 +23,7 @@ Documentation         CPS - Actuator endpoints
 Library               Collections
 Library               RequestsLibrary
 
-Suite Setup           Create Session    MANAGEMENT_URL    http://${CPS_HOST}:${MANAGEMENT_PORT}/manage
+Suite Setup           Create Session    MANAGEMENT_URL    http://${CPS_CORE_HOST}:${CPS_CORE_MANAGEMENT_PORT}/manage
 
 *** Variables ***
 
diff --git a/csit/tests/cps-admin/cps-admin.robot b/csit/tests/cps-admin/cps-admin.robot
index 43a46d3274..a075788ff4 100644
--- a/csit/tests/cps-admin/cps-admin.robot
+++ b/csit/tests/cps-admin/cps-admin.robot
@@ -23,7 +23,7 @@ Library               Collections
 Library               OperatingSystem
 Library               RequestsLibrary
 
-Suite Setup           Create Session      CPS_URL    http://${CPS_HOST}:${CPS_PORT}
+Suite Setup           Create Session      CPS_URL    http://${CPS_CORE_HOST}:${CPS_CORE_PORT}
 
 *** Variables ***
 
diff --git a/csit/tests/cps-data/cps-data.robot b/csit/tests/cps-data/cps-data.robot
index e7334ae11e..1c48e00691 100644
--- a/csit/tests/cps-data/cps-data.robot
+++ b/csit/tests/cps-data/cps-data.robot
@@ -23,7 +23,7 @@ Library               Collections
 Library               OperatingSystem
 Library               RequestsLibrary
 
-Suite Setup           Create Session      CPS_URL    http://${CPS_HOST}:${CPS_PORT}
+Suite Setup           Create Session      CPS_URL    http://${CPS_CORE_HOST}:${CPS_CORE_PORT}
 
 *** Variables ***
 
diff --git a/csit/tests/ncmp-passthrough/ncmp-passthrough.robot b/csit/tests/ncmp-passthrough/ncmp-passthrough.robot
index 3b68c335f5..c23fa085b8 100644
--- a/csit/tests/ncmp-passthrough/ncmp-passthrough.robot
+++ b/csit/tests/ncmp-passthrough/ncmp-passthrough.robot
@@ -26,7 +26,7 @@ Library               OperatingSystem
 Library               RequestsLibrary
 Library               BuiltIn
 
-Suite Setup           Create Session      CPS_URL    http://${CPS_HOST}:${CPS_PORT}
+Suite Setup           Create Session      CPS_URL    http://${CPS_CORE_HOST}:${CPS_CORE_PORT}
 
 *** Variables ***