DFC automated tests

Test cases and suites for DFC automated test

Issue-ID: DCAEGEN2-1434

Change-Id: Ibe2200f7dad358520d78217bad4ca6d3b514a3c3
Signed-off-by: BjornMagnussonXA <bjorn.magnusson@est.tech>

1 files changed, 865 insertions, 0 deletions

diff --git a/test/mocks/datafilecollector-testharness/common/testcase_common.sh b/test/mocks/datafilecollector-testharness/common/testcase_common.sh
new file mode 100755
index 000000000..1e0118e23
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/common/testcase_common.sh
@@ -0,0 +1,865 @@
+#!/bin/bash
+
+. ../common/test_env.sh
+
+echo "Test case started as: ${BASH_SOURCE[$i+1]} "$1 $2
+
+# Script containing all functions needed for auto testing of test cases
+# Arg: local [<image-tag>] ]| remote [<image-tag>] ]| remote-remove [<image-tag>]] | manual-container | manual-app
+
+START_ARG=$1
+IMAGE_TAG="latest"
+
+if [ $# -gt 1 ]; then
+	IMAGE_TAG=$2
+fi
+
+if [ $# -lt 1 ] || [ $# -gt 2 ]; then
+	echo "Expected arg: local [<image-tag>] ]| remote [<image-tag>] ]| remote-remove [<image-tag>]] | manual-container | manual-app"
+	exit 1
+elif [ $1 == "local" ]; then
+	if [ -z $DFC_LOCAL_IMAGE ]; then
+		echo "DFC_LOCAL_IMAGE not set in test_env"
+		exit 1
+	fi
+	DFC_IMAGE=$DFC_LOCAL_IMAGE":"$IMAGE_TAG
+elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
+	if [ -z $DFC_REMOTE_IMAGE ]; then
+		echo "DFC_REMOTE_IMAGE not set in test_env"
+		exit 1
+	fi
+	DFC_IMAGE=$DFC_REMOTE_IMAGE":"$IMAGE_TAG
+elif [ $1 == "manual-container" ] && [ $# -eq 1 ]; then
+	echo "DFC is expected to be started manually as a container with name 'dfc_app'"
+elif [ $1 == "manual-app" ] && [ $# -eq 1 ]; then
+	echo "DFC is expected to be started manually as a java application"
+else
+	echo "Expected arg: local [<image-tag>] ]| remote [<image-tag>] ]| remote-remove [<image-tag>]] | manual-container | manual-app"
+	exit 1
+fi
+
+# Set a description string for the test case
+if [ -z "$TC_ONELINE_DESCR" ]; then
+	TC_ONELINE_DESCR="<no-description>"
+	echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
+fi
+
+# Counter for test suites
+if [ -f .tmp_tcsuite_ctr ]; then
+	tmpval=$(< .tmp_tcsuite_ctr)
+	((tmpval++))
+	echo $tmpval > .tmp_tcsuite_ctr
+fi
+
+# Create a test case id, ATC (Auto Test Case), from the name of the test case script.
+# FTC1.sh -> ATC == FTC1
+ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
+
+# Create the logs dir if not already created in the current dir
+if [ ! -d "logs" ]; then
+    mkdir logs
+fi
+
+TESTLOGS=$PWD/logs
+
+# Create a log dir for the test case
+mkdir -p $TESTLOGS/$ATC 
+
+# Clear the log dir for the test case
+rm $TESTLOGS/$ATC/*.log &> /dev/null
+
+# Log all output from the test case to a TC log
+TCLOG=$TESTLOGS/$ATC/TC.log
+exec &>  >(tee ${TCLOG})
+
+#Variables for counting tests as well as passed and failed tests
+RES_TEST=0
+RES_PASS=0
+RES_FAIL=0
+TCTEST_START=$SECONDS
+
+echo "-------------------------------------------------------------------------------------------------"
+echo "-----------------------------------      Test case: "$ATC
+echo "-----------------------------------      Started:   "$(date)
+echo "-------------------------------------------------------------------------------------------------"
+echo "-- Description: "$TC_ONELINE_DESCR
+echo "-------------------------------------------------------------------------------------------------"
+echo "-----------------------------------      Test case setup      -----------------------------------"
+
+if [ -z "$SIM_GROUP" ]; then
+		SIM_GROUP=$PWD/../simulator-group
+		if [ ! -d  $SIM_GROUP ]; then
+			echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the integration repo, but failed."
+			echo "Please set the SIM_GROUP manually in the test_env.sh"
+			exit 1
+		else 
+			echo "SIM_GROUP auto set to: " $SIM_GROUP
+		fi
+elif [ $SIM_GROUP = *simulator_group ]; then
+			echo "Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the integration repo, check test_env.sh"
+			exit 1
+fi
+
+echo ""
+
+if [ $1 !=  "manual-container" ] && [ $1 !=  "manual-app" ]; then
+	echo -e "DFC image tag set to: \033[1m" $IMAGE_TAG"\033[0m"
+	echo "Configured image for DFC app (${1}): "$DFC_IMAGE 
+	tmp_im=$(docker images ${DFC_IMAGE} | grep -v REPOSITORY)
+
+	if [ $1 == "local" ]; then
+		if [ -z "$tmp_im" ]; then
+			echo "Local image (non nexus) "$DFC_IMAGE" does not exist in local registry, need to be built"
+			exit 1
+		else
+			echo -e "DFC local image: \033[1m"$tmp_im"\033[0m"
+			echo "If the DFC image seem outdated, rebuild the image and run the test again."
+		fi
+	elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
+
+		if [ $1 == "remote-remove" ]; then
+			echo "Attempt to stop dfc_app container if running"
+			docker stop $(docker ps -q --filter name=dfc_app) &> /dev/null
+			docker rm $(docker ps -q --filter name=dfc_app) &> /dev/null
+			docker rmi $DFC_IMAGE &> /dev/null
+			tmp_im=""
+		fi
+		if [ -z "$tmp_im" ]; then
+			echo "Pulling DFC image from nexus: "$DFC_IMAGE
+			docker pull $DFC_IMAGE	 > /dev/null
+			tmp_im=$(docker images ${DFC_IMAGE} | grep -v REPOSITORY)
+			if [ -z "$tmp_im" ]; then
+				echo "Image could not be pulled"
+				exit 1
+			fi
+			echo -e "DFC image: \033[1m"$tmp_im"\033[0m"
+		else 
+			echo -e "DFC image: \033[1m"$tmp_im"\033[0m"
+			echo "!! If the dfc image seem outdated, consider removing it from your docker registry and run the test again."
+		fi
+	fi
+fi
+
+echo ""
+
+echo "Building images for the simulators if needed, MR, DR and DR Redir simulators"
+curdir=$PWD
+cd $SIM_GROUP
+cd ../dr-sim
+docker build -t drsim_common:latest . &> /dev/null
+cd ../mr-sim
+docker build -t mrsim:latest . &> /dev/null
+cd ../simulator-group
+cp -r ../ftps-sftp-server/configuration .
+cp -r ../ftps-sftp-server/tls .
+cd $curdir
+
+echo ""
+
+echo "Local registry images for simulators:"
+echo "MR simulator        " $(docker images | grep mrsim)
+echo "DR simulator:       " $(docker images | grep drsim_common)
+echo "DR redir simulator: " $(docker images | grep drsim_common)
+echo "SFTP:               " $(docker images | grep atmoz/sftp)
+echo "FTPS:               " $(docker images | grep panubo/vsftpd)
+echo ""
+
+echo "-----------------------------------      Test case steps      -----------------------------------"
+
+# Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
+# Not to be called from test script.
+print_err() {
+    echo ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[$i+2]} " line" ${BASH_LINENO[$i+1]}
+}
+# Execute curl using the host and variable. Arg: <host> <variable-name>
+# Returns the variable value (if success) and return code 0 or an error message and return code 1
+do_curl() {
+	res=$(curl -sw "%{http_code}" $1)
+	http_code="${res:${#res}-3}"
+	if [ ${#res} -eq 3 ]; then
+  		echo "<no-response-from-server>"
+		return 1
+	else
+		if [ $http_code -lt 200 ] && [ $http_code -gt 299]; then
+			echo "<not found, resp:${http_code}>"
+			return 1
+		fi
+  		echo "${res:0:${#res}-3}"
+		return 0
+	fi
+}
+
+# Test a simulator variable value towards  target value using an condition operator with an optional timeout.
+# Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done 
+# immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
+# Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
+# before setting pass or fail depending on the result of comparing variable and target using the operator.
+# Not to be called from test script.
+
+var_test() {
+	if [ $# -eq 6 ]; then
+		echo -e "---- ${1} sim test criteria: \033[1m ${3} \033[0m ${4} ${5} within ${6} seconds ----"
+		((RES_TEST++))
+		start=$SECONDS
+		ctr=0
+		for (( ; ; ))
+		do
+			result="$(do_curl $2$3)"
+			retcode=$?
+			result=${result//[[:blank:]]/} #Strip blanks
+			duration=$((SECONDS-start))
+			if [ $((ctr%30)) -eq 0 ]; then
+				echo -ne "  Result=${result} after ${duration} seconds, DFC heartbeat="$(do_curl http://127.0.0.1:8100/heartbeat)
+				echo ""
+			else
+				echo -ne "  Result=${result} after ${duration} seconds\033[0K\r"
+			fi
+			let ctr=ctr+1
+			if [ $retcode -ne 0 ]; then
+				if [ $duration -gt $6 ]; then
+					((RES_FAIL++))
+					echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5}  not reached in ${6} seconds, result = ${result} ----"
+					return
+				fi
+			elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
+				((RES_PASS++))
+				echo -e "  Result=${result} after ${duration} seconds\033[0K\r"
+				echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds ----"
+				return
+			elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
+				((RES_PASS++))
+				echo -e "  Result=${result} after ${duration} seconds\033[0K\r"
+				echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds, result = ${result}  ----"
+				return
+			elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
+				((RES_PASS++))
+				echo -e "  Result=${result} after ${duration} seconds\033[0K\r"
+				echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds, result = ${result}  ----"
+				return
+			elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
+				((RES_PASS++))
+				echo -e "  Result=${result} after ${duration} seconds\033[0K\r"
+				echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds, result = ${result}  ----"
+				return
+			else
+				if [ $duration -gt $6 ]; then
+					((RES_FAIL++))
+					echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5}  not reached in ${6} seconds, result = ${result} ----"
+					return
+				fi
+			fi
+			sleep 1
+		done
+	elif [ $# -eq 5 ]; then
+		echo -e "---- ${1} sim test criteria: \033[1m ${3} \033[0m ${4} ${5} ----"
+		((RES_TEST++))
+		result="$(do_curl $2$3)"
+		retcode=$?
+		result=${result//[[:blank:]]/}  #Strip blanks
+		if [ $retcode -ne 0 ]; then
+			((RES_FAIL++))
+			echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5} not reached, result = ${result} ----"
+		elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
+			((RES_PASS++))
+			echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met"
+		elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
+			((RES_PASS++))
+			echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met, result = ${result} ----"
+		elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
+			((RES_PASS++))
+			echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met, result = ${result} ----"
+		elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
+			((RES_PASS++))
+			echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met, result = ${result} ----"
+		else
+			((RES_FAIL++))
+			echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5} not reached, result = ${result} ----"
+		fi
+	else 
+		echo "Wrong args to var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
+		exit 1
+	fi
+}
+# Stops a named container
+docker_stop() {
+	if [ $# -ne 1 ]; then
+		echo "docker_stop need 1 arg <container-name>"
+		exit 1
+	fi
+	tmp=$(docker stop $1  2>/dev/null)
+	if [ -z $tmp ] || [ $tmp != $1 ]; then
+		echo " ${1} container not stopped or not existing"
+	else
+		echo " ${1} container stopped"
+	fi
+}
+
+# Removes a named container
+docker_rm() {
+	if [ $# -ne 1 ]; then
+		echo "docker_rm need 1 arg <container-name>"
+		exit 1
+	fi
+	tmp=$(docker rm $1  2>/dev/null)
+	if [ -z $tmp ] || [ $tmp != $1 ]; then
+		echo " ${1} container not removed or not existing"
+	else
+		echo " ${1} container removed"
+	fi
+}
+
+start_dfc_image() {
+	echo "Starting DFC"
+	# Port mappning not needed since dfc is running in host mode
+	docker run -d --network="host" --name dfc_app $DFC_IMAGE > /dev/null
+	dfc_started=false
+	for i in {1..10}; do
+	if [ $(docker inspect --format '{{ .State.Running }}' dfc_app) ]
+ 	then
+	 	echo " Image: $(docker inspect --format '{{ .Config.Image }}' dfc_app)"
+   		echo "DFC app Running"
+		dfc_started=true
+   		break
+ 	else
+   		echo sleep $i
+ 	fi
+	done
+	if ! [ $dfc_started  ]; then
+		echo "DFC app could not be started"
+		exit 1
+	fi
+}
+
+#WFunction for waiting for named container to be started manually.
+wait_for_container() {
+	start=$SECONDS
+	if [ $# != 1 ]; then
+		echo "Need one arg: <container-name>"
+		exit 1
+	fi
+	echo "Waiting for container with name '${1}' to be started manually...."
+		
+	for (( ; ; ))
+	do
+		if [ $(docker inspect --format '{{ .State.Running }}' $1 2> /dev/null) ]; then
+   			echo "Container running: "$1
+   			break
+ 		else
+	 		duration=$((SECONDS-start))
+			echo -ne "  Waited ${duration} seconds\033[0K\r"
+   			sleep 1
+ 		fi 
+	done
+}
+
+#WFunction for waiting for named container to be stopped manually. 
+wait_for_container_gone() {
+	start=$SECONDS
+	if [ $# != 1 ]; then
+		echo "Need one arg: <container-name>"
+		exit 1
+	fi
+	echo "Waiting for container with name '${1}' to be stopped manually...."
+
+	for (( ; ; ))
+	do
+		if [ $(docker inspect --format '{{ .State.Running }}' $1 2> /dev/null) ]; then
+	 		duration=$((SECONDS-start))
+			echo -ne "  Waited ${duration} seconds\033[0K\r"
+   			sleep 1
+		else
+   			echo "Container stopped: "$1
+   			break
+ 		fi 
+	done
+}
+
+#Function for waiting to dfc to be started manually
+wait_for_dfc() {
+	read -p "Press enter to continue when dfc has been manually started"
+}
+
+#Function for waiting to dfc to be stopped manually
+wait_for_dfc_gone() {
+	read -p "Press enter to continue when dfc has been manually stopped"
+}
+
+#############################################################
+############## Functions for auto test scripts ##############
+#############################################################
+
+# Print the env variables needed for the simulators and their setup
+log_sim_settings() {
+	echo "Simulator settings"
+	echo "DR_TC=        "$DR_TC
+	echo "DR_REDIR_TC=  "$DR_REDIR_TC
+	echo "MR_TC=        "$MR_TC
+	echo "BC_TC=        "$BC_TC
+	echo "NUM_FTPFILES= "$NUM_FTPFILES
+	echo "NUM_PNFS=     "$NUM_PNFS
+	echo "FILE_SIZE=    "$FILE_SIZE
+	echo "FTP_TYPE=     "$FTP_TYPE
+	echo ""
+}
+
+# Stop and remove all containers including dfc app and simulators
+clean_containers() {
+	echo "Stopping all containers, dfc app and simulators with name prefix 'dfc_'"
+	docker stop $(docker ps -q --filter name=dfc_) &> /dev/null
+	echo "Removing all containers, dfc app and simulators with name prefix 'dfc_'"
+	docker rm $(docker ps -a -q --filter name=dfc_) &> /dev/null
+	echo ""
+}
+
+# Start all simulators in the simulator group
+start_simulators() {
+	echo "Starting all simulators"
+	curdir=$PWD
+	cd $SIM_GROUP
+	$SIM_GROUP/simulators-start.sh
+	cd $curdir
+	echo ""
+}
+
+# Start the dfc application
+start_dfc() {
+
+	if [ $START_ARG == "local" ] || [ $START_ARG == "remote" ] ||  [ $START_ARG == "remote-remove" ]; then
+		start_dfc_image
+	elif [ $START_ARG == "manual-container" ]; then
+		wait_for_container dfc_app
+	elif [ $START_ARG == "manual-app" ]; then
+		wait_for_dfc
+	fi
+}
+
+# Stop and remove the dfc app container
+kill_dfc() {
+	echo "Killing DFC"
+
+	if [ $START_ARG == "local" ] || [ $START_ARG == "remote" ] ||  [ $START_ARG == "remote-remove" ]; then
+		docker_stop dfc_app
+		docker_rm dfc_app
+	elif [ $START_ARG == "manual-container" ]; then
+		wait_for_container_gone dfc_app
+	elif [ $START_ARG == "manual-app" ]; then
+		wait_for_dfc_gone
+	fi
+}
+
+# Stop and remove the DR simulator container
+kill_dr() {
+	echo "Killing DR sim"
+	docker_stop dfc_dr-sim
+	docker_rm dfc_dr-sim
+}
+
+# Stop and remove the DR redir simulator container
+kill_drr() {
+	echo "Killing DR redir sim"
+	docker_stop dfc_dr-redir-sim
+	docker_rm dfc_dr-redir-sim
+}
+
+# Stop and remove the MR simulator container
+kill_mr() {
+	echo "Killing MR sim"
+	docker_stop dfc_mr-sim
+	docker_rm dfc_mr-sim
+}
+
+# Stop and remove the SFTP container
+kill_sftp() {
+	echo "Killing SFTP"
+	docker_stop dfc_sftp-server
+	docker_rm dfc_sftp-server
+}
+
+# Stop and remove the FTPS container
+kill_ftps() {
+	echo "Killing FTPS"
+	docker_stop dfc_ftpes-server-vsftpd
+	docker_rm dfc_ftpes-server-vsftpd
+}
+
+# Print a variable value from the MR simulator. Arg: <variable-name>
+mr_print() {
+	if [ $# != 1 ]; then
+    	print_err "need one arg, <sim-param>"
+		exit 1
+	fi
+	echo -e "---- MR sim, \033[1m $1 \033[0m: $(do_curl http://127.0.0.1:2222/$1)"
+}
+
+# Print a variable value from the DR simulator. Arg: <variable-name>
+dr_print() {
+	if [ $# != 1 ]; then
+    	print_err "need one arg, <sim-param>"
+		exit 1
+	fi
+	echo -e "---- DR sim, \033[1m $1 \033[0m: $(do_curl http://127.0.0.1:3906/$1)"
+}
+
+# Print a variable value from the DR redir simulator. Arg: <variable-name>
+drr_print() {
+	if [ $# != 1 ]; then
+    	print_err "need one arg, <sim-param>"
+		exit 1
+	fi
+	echo -e "---- DR redir sim, \033[1m $1 \033[0m: $(do_curl http://127.0.0.1:3908/$1)"
+}
+# Print a variable value from dfc. Arg: <variable-name>
+dfc_print() {
+	if [ $# != 1 ]; then
+    	print_err "need one arg, <dfc-param>"
+		exit 1
+	fi
+	echo -e "---- DFC, \033[1m $1 \033[0m: $(do_curl http://127.0.0.1:8100/$1)"
+}
+
+# Read a variable value from MR sim and send to stdout.
+mr_read() {
+	echo "$(do_curl http://127.0.0.1:2222/$1)"
+}
+
+# Read a variable value from DR sim and send to stdout.
+dr_read() {
+	echo "$(do_curl http://127.0.0.1:3906/$1)"
+}
+
+# Read a variable value from DR redir sim and send to stdout.
+drr_read() {
+	echo "$(do_curl http://127.0.0.1:3908/$1)"
+}
+
+
+# Sleep. Arg: <sleep-time-in-sec>
+sleep_wait() {
+	if [ $# != 1 ]; then
+		print_err "need one arg, <sleep-time-in-sec>"
+		exit 1
+	fi
+	echo "---- Sleep for " $1 " seconds ----"
+	start=$SECONDS
+	duration=$((SECONDS-start))
+	while [ $duration -lt $1 ]; do
+		echo -ne "  Slept for ${duration} seconds\033[0K\r"
+		sleep 1
+		duration=$((SECONDS-start))
+	done
+}
+
+# Sleep and print dfc heartbeat. Arg: <sleep-time-in-sec>
+sleep_heartbeat() {
+	if [ $# != 1 ]; then
+		print_err "need one arg, <sleep-time-in-sec>"
+		exit 1
+	fi
+	echo "---- Sleep for " $1 " seconds ----"
+	start=$SECONDS
+	duration=$((SECONDS-start))
+	ctr=0
+	while [ $duration -lt $1 ]; do
+		if [ $((ctr%30)) -eq 0 ]; then
+			echo -ne "  Slept for ${duration} seconds, \033[1m heartbeat \033[0m "$(do_curl http://127.0.0.1:8100/heartbeat)
+			echo ""
+		else
+			echo -ne "  Slept for ${duration} seconds, \033[1m heartbeat \033[0m "$(do_curl http://127.0.0.1:8100/heartbeat)" \033[0K\r"
+		fi
+		let ctr=ctr+1
+		sleep 1
+		duration=$((SECONDS-start))
+	done
+	echo ""
+}
+
+# Tests if a variable value in the MR simulator is equal to a target value and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
+# equal to the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value becomes equal to the target
+# value or not.
+mr_equal() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "MR" "http://127.0.0.1:2222/" $1 "=" $2 $3
+	else
+		print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the MR simulator is greater than a target value and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
+# greater than the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value greater than the target
+# value or not.
+mr_greater() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "MR" "http://127.0.0.1:2222/" $1 ">" $2 $3
+	else
+		print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the MR simulator is less than a target value and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
+# less than the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value less than the target
+# value or not.
+mr_less() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "MR" "http://127.0.0.1:2222/" $1 "<" $2 $3
+	else
+		print_err "Wrong args to mr_less, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the MR simulator contains the target string and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable contains
+# the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value contains the target
+# value or not.
+mr_contain_str() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "MR" "http://127.0.0.1:2222/" $1 "contain_str" $2 $3
+	else
+		print_err "Wrong args to mr_contain_str, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the DR simulator is equal to a target value and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
+# equal to the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value becomes equal to the target
+# value or not.
+dr_equal() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "DR" "http://127.0.0.1:3906/" $1 "=" $2 $3
+	else
+		print_err "Wrong args to dr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the DR simulator is greater than a target value and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
+# greater than the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value greater than the target
+# value or not.
+dr_greater() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "DR" "http://127.0.0.1:3906/" $1 ">" $2 $3
+	else
+		print_err "Wrong args to dr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the DR simulator is less than a target value and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
+# less than the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value less than the target
+# value or not.
+dr_less() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "DR" "http://127.0.0.1:3906/" $1 "<" $2 $3
+	else
+		print_err "Wrong args to dr_less, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the DR Redir simulator is equal to a target value and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
+# equal to the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value becomes equal to the target
+# value or not.
+drr_equal() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "DR REDIR" "http://127.0.0.1:3908/" $1 "=" $2 $3
+	else
+		print_err "Wrong args to drr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+
+# Tests if a variable value in the DR Redir simulator is greater a target value and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
+# greater the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value greater than the target
+# value or not.
+drr_greater() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "DR REDIR" "http://127.0.0.1:3908/" $1 ">" $2 $3
+	else
+		print_err "Wrong args to drr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the DR Redir simulator is less than a target value and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
+# less than the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value less than the target
+# value or not.
+drr_less() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		var_test "DR REDIR" "http://127.0.0.1:3908/" $1 "<" $2 $3
+	else
+		print_err "Wrong args to drr_less, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+#Test is a variable in the DFC contains a substring. Arg: <variable-name> <substring-in-quotes>
+dfc_contain_str() {
+	if [ $# -eq 2 ]; then
+		echo -e "---- DFC test criteria: \033[1m ${1} \033[0m contains: ${2} ----"
+		((RES_TEST++))
+		result="$(do_curl http://127.0.0.1:8100/${1})"
+		if [[ $result =~ $2 ]]; then
+			((RES_PASS++))
+			echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met"		
+		else
+			((RES_FAIL++))
+			echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${1} not reached, result = ${result} ----"
+		fi
+	else 
+		echo "Wrong args to dfc_contain_str, needs two arg: <dfc-variable> <str>"
+		exit 1
+	fi
+}
+
+# Store all dfc app and simulators log to the test case log dir. All logs gets a prefix to
+# separate logs stored at different steps in the test script. Arg: <tc-id> <log-prefix>
+store_logs() {
+	if [ $# != 1 ]; then
+    	print_err "need one arg, <file-prefix>"
+		exit 1
+	fi
+	echo "Storing all container logs and dfc app log using prefix: "$1
+	if ! [ $START_ARG == "manual-app" ]; then
+		docker cp dfc_app:/var/log/ONAP/application.log $TESTLOGS/$ATC/$1_application.log
+		docker logs dfc_app > $TESTLOGS/$ATC/$1_dfc_app-docker.log 2>&1
+	fi
+	docker logs dfc_mr-sim > $TESTLOGS/$ATC/$1_dfc_mr-sim-docker.log 2>&1
+	docker logs dfc_dr-sim > $TESTLOGS/$ATC/$1_dfc_dr-sim-docker.log 2>&1
+	docker logs dfc_dr-redir-sim > $TESTLOGS/$ATC/$1_dfc_dr-redir-sim-docker.log 2>&1
+	docker logs dfc_ftpes-server-vsftpd > $TESTLOGS/$ATC/$1_dfc_ftpes-server-vsftpd.log 2>&1
+	docker logs dfc_sftp-server > $TESTLOGS/$ATC/$1_dfc_sftp-server.log 2>&1
+}
+# Check the dfc application log for WARN and ERR messages and print the count.
+check_dfc_log() {
+	echo "Checking dfc log /var/log/ONAP/application.log for WARNINGs and ERRORs, excluding messages from CONSUL"
+	foundentries=$(docker exec -it dfc_app grep WARN /var/log/ONAP/application.log | grep -iv CONSUL | wc -l)
+	if [ $? -ne  0 ];then
+		echo "  Problem to search dfc log /var/log/ONAP/application.log"
+	else
+		if [ $foundentries -eq 0 ]; then
+			echo "  No WARN entries found in dfc log /var/log/ONAP/application.log"
+		else 
+			echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in dfc log /var/log/ONAP/application.log"
+		fi
+	fi
+		foundentries=$(docker exec -it dfc_app grep ERR /var/log/ONAP/application.log | grep -iv CONSUL | wc -l)
+	if [ $? -ne  0 ];then
+		echo "  Problem to search dfc log /var/log/ONAP/application.log"
+	else
+		if [ $foundentries -eq 0 ]; then
+			echo "  No ERR entries found in dfc log /var/log/ONAP/application.log"
+		else 
+			echo -e "  Found \033[1m"$foundentries"\033[0m ERR entries in dfc log /var/log/ONAP/application.log"
+		fi
+	fi
+}
+
+print_all() {
+
+	echo "---- DFC and all sim variables"
+
+	dfc_print heartbeat
+	
+	mr_print tc_info
+	mr_print execution_time
+	mr_print exe_time_first_poll
+	mr_print ctr_requests
+	mr_print ctr_responses
+	mr_print ctr_files
+	mr_print ctr_unique_files
+	mr_print ctr_events
+	mr_print ctr_unique_PNFs
+
+	dr_print tc_info
+	dr_print execution_time
+	dr_print ctr_publish_query
+	dr_print ctr_publish_query_published
+	dr_print ctr_publish_query_not_published
+	dr_print ctr_publish_req
+	dr_print ctr_publish_req_redirect
+	dr_print ctr_publish_req_published
+	dr_print ctr_published_files
+
+	drr_print tc_info
+	drr_print execution_time
+	drr_print ctr_publish_requests
+	drr_print ctr_publish_responses
+	drr_print dwl_volume
+	drr_print time_lastpublish
+}
+
+# Print the test result 
+print_result() {
+
+	TCTEST_END=$SECONDS
+	duration=$((TCTEST_END-TCTEST_START))
+
+	echo "-------------------------------------------------------------------------------------------------"
+	echo "-------------------------------------     Test case: "$ATC
+	echo "-------------------------------------     Ended:     "$(date)
+	echo "-------------------------------------------------------------------------------------------------"
+	echo "-- Description: "$TC_ONELINE_DESCR
+	echo "-- Execution time: " $duration " seconds"
+	echo "-------------------------------------------------------------------------------------------------"
+	echo "-------------------------------------     RESULTS"
+	echo ""
+
+
+	total=$((RES_PASS+RES_FAIL))
+	if [ $RES_TEST -eq 0 ]; then
+		echo -e "\033[1mNo tests seem to have executed. Check the script....\033[0m"
+	elif [ $total != $RES_TEST ]; then
+		echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
+	elif [ $RES_PASS = $RES_TEST ]; then
+		echo -e "All tests \033[32m\033[1mPASS\033[0m"
+		# Update test suite counter
+		if [ -f .tmp_tcsuite_pass_ctr ]; then
+			tmpval=$(< .tmp_tcsuite_pass_ctr)
+			((tmpval++))
+			echo $tmpval > .tmp_tcsuite_pass_ctr
+		fi
+		if [ -f .tmp_tcsuite_pass ]; then
+			echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
+		fi
+	else
+		echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
+		# Update test suite counter
+		if [ -f .tmp_tcsuite_fail_ctr ]; then
+			tmpval=$(< .tmp_tcsuite_fail_ctr)
+			((tmpval++))
+			echo $tmpval > .tmp_tcsuite_fail_ctr
+		fi
+		if [ -f .tmp_tcsuite_fail ]; then
+			echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
+		fi
+	fi
+
+	echo "++++ Number of tests:        "$RES_TEST
+	echo "++++ Number of passed tests: "$RES_PASS
+	echo "++++ Number of failed tests: "$RES_FAIL
+	echo "-------------------------------------     Test case complete    ---------------------------------"
+	echo "-------------------------------------------------------------------------------------------------"
+	echo ""
+}