Adding time out handling in setup.sh script

Change-Id: I410d597da6a1fe15d4b02794f56b3507563978c8
Issue-ID: SO-1955
Signed-off-by: waqas.ikram <waqas.ikram@est.tech>

4 files changed, 117 insertions, 20 deletions

diff --git a/plans/so/integration-etsi-testing/config/apply-workarounds.sh b/plans/so/integration-etsi-testing/config/apply-workarounds.sh
index ff167bf7..4200a361 100755
--- a/plans/so/integration-etsi-testing/config/apply-workarounds.sh
+++ b/plans/so/integration-etsi-testing/config/apply-workarounds.sh
@@ -29,6 +29,7 @@ BUILDING_BLOCK_COUNT_QUERY="select count(*) from $BUIDLING_BLOCK_TABLE_NAME;"
 FLY_WAY_MIGRATION_QUERY="SELECT COUNT(*) FROM flyway_schema_history WHERE script LIKE '%R__MacroData%' AND installed_on IS NOT NULL;"
 SLEEP_TIME=5
 TIME_OUT_DEFAULT_VALUE_SEC=1200 #20 mins
+SCRIPT_NAME=$(basename $0)
 
 current_timestamp()
 {
@@ -37,53 +38,53 @@ current_timestamp()
 
 wait_for_database_availability()
 {
- echo "$(current_timestamp): Checking for database availability"
+ echo "$SCRIPT_NAME $(current_timestamp): Checking for database availability"
  until echo '\q' | mysql -h $DB_HOST -P $DB_PORT -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB; do
-     >&2 echo "$(current_timestamp): Database is unavailable - sleeping for ${SLEEP_TIME} seconds"
+     >&2 echo "$SCRIPT_NAME $(current_timestamp): Database is unavailable - sleeping for ${SLEEP_TIME} seconds"
      isTimeOut
      sleep ${SLEEP_TIME}
  done
 
- echo "$(current_timestamp): Database is available now"
+ echo "$SCRIPT_NAME $(current_timestamp): Database is available now"
 }
 
 wait_container_to_create_table()
 {
  while [ $(mysql -h $DB_HOST -P $DB_PORT -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB -sse "$TABLE_EXISTS_QUERY") -eq "0" ] ; do
-     echo "$(current_timestamp): Waiting for so-catalog container to create tables - sleeping for ${SLEEP_TIME} seconds"
+     echo "$SCRIPT_NAME $(current_timestamp): Waiting for so-catalog container to create tables - sleeping for ${SLEEP_TIME} seconds"
      isTimeOut
      sleep ${SLEEP_TIME}
  done
 
  while [ $(mysql -h $DB_HOST -P $DB_PORT -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB -sse "$BUILDING_BLOCK_COUNT_QUERY") -eq "0" ] ; do
-     echo "$(current_timestamp): Waiting for so-catalog container to insert records in $BUIDLING_BLOCK_TABLE_NAME - sleeping for ${SLEEP_TIME} seconds"
+     echo "$SCRIPT_NAME $(current_timestamp): Waiting for so-catalog container to insert records in $BUIDLING_BLOCK_TABLE_NAME - sleeping for ${SLEEP_TIME} seconds"
      isTimeOut
      sleep ${SLEEP_TIME}
  done
 
- echo "$(current_timestamp): $CATALOG_DB tables available now . . ."
+ echo "$SCRIPT_NAME $(current_timestamp): $CATALOG_DB tables available now . . ."
 }
 
 wait_for_flyway_migration_to_finish()
 {
  while [ $(mysql -h $DB_HOST -P $DB_PORT -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB -sse "$FLY_WAY_MIGRATION_QUERY") -ne "1" ] ; do
-     echo "$(current_timestamp): Waiting for flyway migration sql statement to finish with success - sleeping for ${SLEEP_TIME} seconds"
+     echo "$SCRIPT_NAME $(current_timestamp): Waiting for flyway migration sql statement to finish with success - sleeping for ${SLEEP_TIME} seconds"
      isTimeOut
      sleep ${SLEEP_TIME}
  done
- echo "$(current_timestamp): flyway migration finished . . . "
+ echo "$SCRIPT_NAME $(current_timestamp): flyway migration finished . . . "
 }
 
 
 apply_workaround()
 {
- echo "$(current_timestamp): Applying workaround . . ."
+ echo "$SCRIPT_NAME $(current_timestamp): Applying workaround . . ."
 
  wait_for_database_availability
  wait_container_to_create_table
  wait_for_flyway_migration_to_finish
 
- echo "$(current_timestamp): Will insert data into $CATALOG_DB"
+ echo "$SCRIPT_NAME $(current_timestamp): Will insert data into $CATALOG_DB"
 mysql -h $DB_HOST -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB << EOF
   BEGIN;
   INSERT INTO $BUIDLING_BLOCK_TABLE_NAME (BUILDING_BLOCK_NAME,RESOURCE_TYPE,TARGET_ACTION) values ("EtsiVnfInstantiateBB", "VNF", "ACTIVATE");
@@ -109,36 +110,36 @@ mysql -h $DB_HOST -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB << EOF
 EOF
 
  if [ $? -ne 0 ]; then
-    echo "$(current_timestamp): Failed to execute workaround . . ."
+    echo "$SCRIPT_NAME $(current_timestamp): Failed to execute workaround . . ."
     exit 1
  fi
 
- echo "$(current_timestamp): Finished applying workaround . . ."
+ echo "$SCRIPT_NAME $(current_timestamp): Finished applying workaround . . ."
 }
 
 isTimeOut()
 {
- if [ `date +%s` -gt $TIME_OUT_END_TIME_IN_MS ]; then
-    echo "$(current_timestamp): workaround script timed out . . ."
+ if [ `date +%s` -gt $TIME_OUT_END_TIME_IN_SECONDS ]; then
+    echo "$SCRIPT_NAME $(current_timestamp): workaround script timed out . . ."
     exit 1;
  fi
 }
 
 # main body
 if [ -z "$TIME_OUT_IN_SECONDS"]; then
-    echo "$(current_timestamp): TIME_OUT_IN_SECONDS attribute is empty will use default val: $TIME_OUT_DEFAULT_VALUE_SEC"
+    echo "$SCRIPT_NAME $(current_timestamp): TIME_OUT_IN_SECONDS attribute is empty will use default val: $TIME_OUT_DEFAULT_VALUE_SEC"
     TIME_OUT_IN_SECONDS=$TIME_OUT_DEFAULT_VALUE_SEC
 fi
 
 DIGITS_REGEX='^[0-9]+$'
 if ! [[ $TIME_OUT_IN_SECONDS =~ $DIGIT_REGEX ]] ; then
-    echo "$(current_timestamp): TIME_OUT_IN_SECONDS attribute Must be number: $TIME_OUT_IN_SECONDS, will use default val: $TIME_OUT_DEFAULT_VALUE_SEC"
+    echo "$SCRIPT_NAME $(current_timestamp): TIME_OUT_IN_SECONDS attribute Must be number: $TIME_OUT_IN_SECONDS, will use default val: $TIME_OUT_DEFAULT_VALUE_SEC"
     TIME_OUT_IN_SECONDS=$TIME_OUT_DEFAULT_VALUE_SEC
 fi
 
-START_TIME_IN_MS=`date +%s`
-TIME_OUT_END_TIME_IN_MS=$(($START_TIME_IN_MS+$TIME_OUT_IN_SECONDS));
-echo "$(current_timestamp): Workaround script will time out at `date -d @$TIME_OUT_END_TIME_IN_MS`"
+START_TIME_IN_SECONDS=`date +%s`
+TIME_OUT_END_TIME_IN_SECONDS=$(($START_TIME_IN_SECONDS+$TIME_OUT_IN_SECONDS));
+echo "$SCRIPT_NAME $(current_timestamp): Workaround script will time out at `date -d @$TIME_OUT_END_TIME_IN_SECONDS`"
 
 apply_workaround
 
diff --git a/plans/so/integration-etsi-testing/config/env b/plans/so/integration-etsi-testing/config/env
index f34f9c48..9b6dc557 100644
--- a/plans/so/integration-etsi-testing/config/env
+++ b/plans/so/integration-etsi-testing/config/env
@@ -1,2 +1,3 @@
 NEXUS_DOCKER_REPO_MSO=nexus3.onap.org:10001
 TAG=1.4.0-STAGING-latest
+TIME_OUT_DEFAULT_VALUE_SEC=1200
diff --git a/plans/so/integration-etsi-testing/config/wait-for-workaround-job.sh b/plans/so/integration-etsi-testing/config/wait-for-workaround-job.sh
new file mode 100755
index 00000000..ae6c2f57
--- /dev/null
+++ b/plans/so/integration-etsi-testing/config/wait-for-workaround-job.sh
@@ -0,0 +1,84 @@
+
+#
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2019 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+#
+
+# @author Waqas Ikram (waqas.ikram@est.tech)
+
+SLEEP_TIME=5
+WORKAROUND_SUCCESSFUL_TEXT="Finished applying workaround"
+WORKAROUND_FAILURE_TEXT="Failed to execute workaround"
+WORKAROUND_TIME_OUT_TEXT="workaround script timed out"
+WORKAROUND_CONTAINER_NAME=$(docker ps -aqf "name=workaround-config" --format "{{.Names}}")
+SCRIPT_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SCRIPT_NAME=$(basename $0)
+
+current_timestamp()
+{
+ date +"%Y-%m-%d %H:%M:%S"
+}
+
+# main body
+if [ -z $TIME_OUT_DEFAULT_VALUE_SEC ]; then
+    echo "$SCRIPT_NAME $(current_timestamp): ERROR: Undefined value for TIME_OUT_DEFAULT_VALUE_SEC attribute"
+    exit 1
+fi
+
+if [ -z $WORKAROUND_CONTAINER_NAME ]; then
+   echo "$SCRIPT_NAME $(current_timestamp): Unable to find docker container id "
+   exit 1
+fi
+
+START_TIME_IN_SECONDS=`date +%s`
+TIME_OUT_END_TIME_IN_SECONDS=$(($START_TIME_IN_SECONDS+$TIME_OUT_DEFAULT_VALUE_SEC));
+
+
+echo echo "$SCRIPT_NAME $(current_timestamp): $SCRIPT_NAME script Start Time `date -d @$START_TIME_IN_SECONDS`"
+echo echo "$SCRIPT_NAME $(current_timestamp): $SCRIPT_NAME will time out at `date -d @$TIME_OUT_END_TIME_IN_SECONDS`"
+
+while [ `date +%s` -lt "$TIME_OUT_END_TIME_IN_SECONDS" ]; do
+    echo "$(current_timestamp): Waiting for $WORKAROUND_CONTAINER_NAME to finish ..."
+
+    result=$(docker logs $WORKAROUND_CONTAINER_NAME 2>&1 | grep -E "$WORKAROUND_SUCCESSFUL_TEXT|$WORKAROUND_FAILURE_TEXT|$WORKAROUND_TIME_OUT_TEXT")
+    if [ ! -z "$result" ]; then
+        echo "$SCRIPT_NAME $(current_timestamp): Found result: $result"
+        break;
+    fi
+    echo "$(current_timestamp): Sleeping for ${SLEEP_TIME} seconds"
+    sleep ${SLEEP_TIME}
+done
+
+if [ -z "$result" ]; then
+   echo "$SCRIPT_NAME $(current_timestamp): ERROR: failed to apply workaround . . . "
+   echo "-------------- $WORKAROUND_CONTAINER_NAME logs -------------"
+   docker logs $WORKAROUND_CONTAINER_NAME
+   echo "------------------------------------------------------------"
+   exit 1
+fi
+
+if echo "$result" | grep -E "$WORKAROUND_FAILURE_TEXT|$WORKAROUND_TIME_OUT_TEXT"; then
+    echo "$SCRIPT_NAME $(current_timestamp): Work around script failed"
+    echo "-------------- $WORKAROUND_CONTAINER_NAME logs -------------"
+    docker logs $WORKAROUND_CONTAINER_NAME
+    echo "------------------------------------------------------------"
+    exit 1
+fi
+
+echo "$SCRIPT_NAME $(current_timestamp): Successfully applied workaround configuration . . ."
+exit 0
diff --git a/plans/so/integration-etsi-testing/setup.sh b/plans/so/integration-etsi-testing/setup.sh
index 75e75275..597bf2ea 100755
--- a/plans/so/integration-etsi-testing/setup.sh
+++ b/plans/so/integration-etsi-testing/setup.sh
@@ -39,6 +39,7 @@ MVN_VERSION="$MVN -v"
 MVN_SETTINGS_XML="$SCRIPT_HOME/settings.xml"
 MVN_CLEAN_INSTALL="$MVN clean install"
 SIMULATOR_MAVEN_PROJECT_POM="$SCRIPT_HOME/so-simulators/pom.xml"
+WAIT_FOR_WORKAROUND_SCRIPT=$CONFIG_DIR/"wait-for-workaround-job.sh"
 
 echo "Running $SCRIPT_HOME/$SCRIPT_NAME ..."
 
@@ -121,7 +122,6 @@ fi
 
 git clone http://gerrit.onap.org/r/so/docker-config.git $TEST_LAB_DIR_PATH
 
-
 export TEST_LAB_DIR=$TEST_LAB_DIR_PATH
 export CONFIG_DIR_PATH=$CONFIG_DIR
 
@@ -130,6 +130,17 @@ docker-compose -f $DOCKER_COMPOSE_FILE_PATH up -d
 echo "Sleeping for 3m"
 sleep 3m
 
+echo "Will execute $WAIT_FOR_WORKAROUND_SCRIPT script"
+$WAIT_FOR_WORKAROUND_SCRIPT
+
+if [ $? -ne 0 ]; then
+   echo "ERROR: $WAIT_FOR_WORKAROUND_SCRIPT failed"
+   echo "Will stop running docker containers . . ."
+   docker-compose -f $DOCKER_COMPOSE_FILE_PATH down
+   exit 1
+fi
+
+
 REPO_IP='127.0.0.1'
 ROBOT_VARIABLES="-v REPO_IP:${REPO_IP}"