Added support for Consul/CBS and multiple DFCs

Issue-ID: INT-1155
Change-Id: I3c1ed2f6072655c4396e406ddfd490d3786fe4d6
Signed-off-by: BjornMagnussonXA <bjorn.magnusson@est.tech>

3 files changed, 610 insertions, 155 deletions

diff --git a/test/mocks/datafilecollector-testharness/common/test_env.sh b/test/mocks/datafilecollector-testharness/common/test_env.sh
index 794b354b3..f4d443bff 100644
--- a/test/mocks/datafilecollector-testharness/common/test_env.sh
+++ b/test/mocks/datafilecollector-testharness/common/test_env.sh
@@ -16,3 +16,34 @@ export DFC_REMOTE_IMAGE=nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.
 export DFC_LOCAL_IMAGE=onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server
 
 
+# Common env var for auto-test.
+
+DFC_PORT=8100
+DFC_PORT_SECURE=8433
+DFC_LOGPATH="/var/log/ONAP/application.log"
+DOCKER_SIM_NWNAME="dfcnet"
+CONSUL_HOST="consul-server"
+CONSUL_PORT=8500
+CONFIG_BINDING_SERVICE="config-binding-service"
+MR_PORT=2222
+DR_PORT=3906
+DR_PORT_SECURE=3907
+DRR_PORT=3908
+DRR_PORT_SECURE=3909
+DFC_APP_BASE="dfc_app"
+DFC_MAX_NUM=5
+DFC_MAX_IDX=$(($DFC_MAX_NUM - 1))
+SFTP_BASE="dfc_sftp-server"
+FTPS_BASE="dfc_ftpes-server-vsftpd"
+FTP_MAX_NUM=5
+FTP_MAX_IDX=$(($FTP_MAX_NUM - 1))
+SFTP_SIMS_CONTAINER="sftp-server0:22,sftp-server1:22,sftp-server2:22,sftp-server3:22,sftp-server4:22"
+FTPS_SIMS_CONTAINER="ftpes-server-vsftpd0:21,ftpes-server-vsftpd1:21,ftpes-server-vsftpd2:21,ftpes-server-vsftpd3:21,ftpes-server-vsftpd4:21"
+SFTP_SIMS_LOCALHOST="localhost:1022,localhost:1023,localhost:1024,localhost:1025,localhost:1026"
+FTPS_SIMS_LOCALHOST="localhost:1032,localhost:1033,localhost:1034,localhost:1035,localhost:1036"
+
+export SFTP_SIMS=$SFTP_SIMS_CONTAINER   #This env will be set to SFTP_SIMS_LOCALHOST if auto test is executed with 'manual-app'
+export FTPS_SIMS=$FTPS_SIMS_CONTAINER   #This env will be set to FTPS_SIMS_LOCALHOST if auto test is executed with 'manual-app'
+
+export DR_REDIR_SIM="drsim_redir"       #This env will be set to 'localhost' if auto test is executed with 'manual-app'
+
diff --git a/test/mocks/datafilecollector-testharness/common/testcase_common.sh b/test/mocks/datafilecollector-testharness/common/testcase_common.sh
index 1e0118e23..b0a14aa35 100755
--- a/test/mocks/datafilecollector-testharness/common/testcase_common.sh
+++ b/test/mocks/datafilecollector-testharness/common/testcase_common.sh
@@ -2,15 +2,22 @@
 
 . ../common/test_env.sh
 
+
+
 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$1 $2
 
 # Script containing all functions needed for auto testing of test cases
 # Arg: local [<image-tag>] ]| remote [<image-tag>] ]| remote-remove [<image-tag>]] | manual-container | manual-app
 
+STARTED_DFCS="" #DFC app names added to this var to keep track of started container in the script
 START_ARG=$1
 IMAGE_TAG="latest"
 
 if [ $# -gt 1 ]; then
+	if [[ "$2" =~ ^1.1.* ]]; then
+		echo "This version of auto-test does not support DFC image version of 1.1.X"
+		exit 1
+	fi
 	IMAGE_TAG=$2
 fi
 
@@ -30,9 +37,9 @@ elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 	fi
 	DFC_IMAGE=$DFC_REMOTE_IMAGE":"$IMAGE_TAG
 elif [ $1 == "manual-container" ] && [ $# -eq 1 ]; then
-	echo "DFC is expected to be started manually as a container with name 'dfc_app'"
+	echo "DFC is expected to be started manually, when prompted,  as a container with name 'dfc_app'<index> with and index in the range from 0 to '${DFC_MAX_IDX}'"
 elif [ $1 == "manual-app" ] && [ $# -eq 1 ]; then
-	echo "DFC is expected to be started manually as a java application"
+	echo "DFC is expected to be started manually, when prompted, as a java application"
 else
 	echo "Expected arg: local [<image-tag>] ]| remote [<image-tag>] ]| remote-remove [<image-tag>]] | manual-container | manual-app"
 	exit 1
@@ -104,7 +111,7 @@ echo ""
 
 if [ $1 !=  "manual-container" ] && [ $1 !=  "manual-app" ]; then
 	echo -e "DFC image tag set to: \033[1m" $IMAGE_TAG"\033[0m"
-	echo "Configured image for DFC app (${1}): "$DFC_IMAGE 
+	echo "Configured image for DFC app(s) (${1}): "$DFC_IMAGE
 	tmp_im=$(docker images ${DFC_IMAGE} | grep -v REPOSITORY)
 
 	if [ $1 == "local" ]; then
@@ -118,9 +125,9 @@ if [ $1 !=  "manual-container" ] && [ $1 !=  "manual-app" ]; then
 	elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 
 		if [ $1 == "remote-remove" ]; then
-			echo "Attempt to stop dfc_app container if running"
-			docker stop $(docker ps -q --filter name=dfc_app) &> /dev/null
-			docker rm $(docker ps -q --filter name=dfc_app) &> /dev/null
+			echo "Attempt to stop dfc_app container(s) if running"
+			docker stop $(docker ps -q --filter name=${DFC_APP_BASE}]) &> /dev/null
+			docker rm $(docker ps -q --filter name=${DFC_APP_BASE}) &> /dev/null
 			docker rmi $DFC_IMAGE &> /dev/null
 			tmp_im=""
 		fi
@@ -133,25 +140,26 @@ if [ $1 !=  "manual-container" ] && [ $1 !=  "manual-app" ]; then
 				exit 1
 			fi
 			echo -e "DFC image: \033[1m"$tmp_im"\033[0m"
-		else 
+		else
 			echo -e "DFC image: \033[1m"$tmp_im"\033[0m"
-			echo "!! If the dfc image seem outdated, consider removing it from your docker registry and run the test again."
+			echo "!! If the dfc image seem outdated, consider removing it from your docker registry and run the test again. Or run the script with 'remote-remove'"
 		fi
 	fi
 fi
 
+
+
 echo ""
 
-echo "Building images for the simulators if needed, MR, DR and DR Redir simulators"
+echo "Building images for the simulators if needed, MR, DR, DR Redir and FTPS simulators"
 curdir=$PWD
 cd $SIM_GROUP
 cd ../dr-sim
 docker build -t drsim_common:latest . &> /dev/null
 cd ../mr-sim
 docker build -t mrsim:latest . &> /dev/null
-cd ../simulator-group
-cp -r ../ftps-sftp-server/configuration .
-cp -r ../ftps-sftp-server/tls .
+cd ../ftps-sftp-server
+docker build -t ftps_vsftpd:latest -f Dockerfile-ftps . &> /dev/null
 cd $curdir
 
 echo ""
@@ -161,19 +169,35 @@ echo "MR simulator        " $(docker images | grep mrsim)
 echo "DR simulator:       " $(docker images | grep drsim_common)
 echo "DR redir simulator: " $(docker images | grep drsim_common)
 echo "SFTP:               " $(docker images | grep atmoz/sftp)
-echo "FTPS:               " $(docker images | grep panubo/vsftpd)
+echo "FTPS:               " $(docker images | grep ftps_vsftpd)
+echo "Consul:             " $(docker images | grep consul)
+echo "CBS:                " $(docker images | grep platform.configbinding.app)
 echo ""
 
+#Configure MR sim to use correct host:port for running dfc as an app or as a container
+#Configure DR sim with correct address for DR redirect simulator
+if [ $START_ARG == "manual-app" ]; then
+	export SFTP_SIMS=$SFTP_SIMS_LOCALHOST
+	export FTPS_SIMS=$FTPS_SIMS_LOCALHOST
+	export DR_REDIR_SIM="localhost"
+fi
+#else
+#	export SFTP_SIMS=$SFTP_SIMS_CONTAINER
+#	export FTPS_SIMS=$FTPS_SIMS_CONTAINER
+#	export DR_REDIR_SIM="drsim_redir"
+#fi
+
 echo "-----------------------------------      Test case steps      -----------------------------------"
 
 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
 # Not to be called from test script.
-print_err() {
+__print_err() {
     echo ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[$i+2]} " line" ${BASH_LINENO[$i+1]}
 }
-# Execute curl using the host and variable. Arg: <host> <variable-name>
+# Execute curl using the host and variable. Arg: <host and variable-name>  [ <flag-to-strip-new-line> ]
+#<flag-to-strip-new-line> may contain any string, it is just a flag
 # Returns the variable value (if success) and return code 0 or an error message and return code 1
-do_curl() {
+__do_curl() {
 	res=$(curl -sw "%{http_code}" $1)
 	http_code="${res:${#res}-3}"
 	if [ ${#res} -eq 3 ]; then
@@ -184,19 +208,24 @@ do_curl() {
 			echo "<not found, resp:${http_code}>"
 			return 1
 		fi
-  		echo "${res:0:${#res}-3}"
+		if [ $# -eq 2 ]; then
+  			echo "${res:0:${#res}-3}" | xargs
+		else
+  			echo "${res:0:${#res}-3}"
+		fi
+
 		return 0
 	fi
 }
 
 # Test a simulator variable value towards  target value using an condition operator with an optional timeout.
-# Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done 
+# Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
 # before setting pass or fail depending on the result of comparing variable and target using the operator.
 # Not to be called from test script.
 
-var_test() {
+__var_test() {
 	if [ $# -eq 6 ]; then
 		echo -e "---- ${1} sim test criteria: \033[1m ${3} \033[0m ${4} ${5} within ${6} seconds ----"
 		((RES_TEST++))
@@ -204,13 +233,17 @@ var_test() {
 		ctr=0
 		for (( ; ; ))
 		do
-			result="$(do_curl $2$3)"
+			result="$(__do_curl $2$3)"
 			retcode=$?
 			result=${result//[[:blank:]]/} #Strip blanks
 			duration=$((SECONDS-start))
 			if [ $((ctr%30)) -eq 0 ]; then
-				echo -ne "  Result=${result} after ${duration} seconds, DFC heartbeat="$(do_curl http://127.0.0.1:8100/heartbeat)
-				echo ""
+				echo "  Result=${result} after ${duration} seconds"
+				for (( i=0; i<=$DFC_MAX_IDX; i++ )); do
+					if [[ $STARTED_DFCS =~ "_"$DFC_APP_BASE$i"_" ]]; then
+						echo "    HB ${DFC_APP_BASE}${i}: $(__do_curl http://127.0.0.1:$(($DFC_PORT+$i))/status strip)"
+					fi
+				done
 			else
 				echo -ne "  Result=${result} after ${duration} seconds\033[0K\r"
 			fi
@@ -253,7 +286,7 @@ var_test() {
 	elif [ $# -eq 5 ]; then
 		echo -e "---- ${1} sim test criteria: \033[1m ${3} \033[0m ${4} ${5} ----"
 		((RES_TEST++))
-		result="$(do_curl $2$3)"
+		result="$(__do_curl $2$3)"
 		retcode=$?
 		result=${result//[[:blank:]]/}  #Strip blanks
 		if [ $retcode -ne 0 ]; then
@@ -275,15 +308,15 @@ var_test() {
 			((RES_FAIL++))
 			echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5} not reached, result = ${result} ----"
 		fi
-	else 
-		echo "Wrong args to var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
+	else
+		echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
 		exit 1
 	fi
 }
 # Stops a named container
-docker_stop() {
+__docker_stop() {
 	if [ $# -ne 1 ]; then
-		echo "docker_stop need 1 arg <container-name>"
+		echo "__docker_stop need 1 arg <container-name>"
 		exit 1
 	fi
 	tmp=$(docker stop $1  2>/dev/null)
@@ -294,10 +327,24 @@ docker_stop() {
 	fi
 }
 
+# Starts a named container (that has previously been stopped)
+__docker_start() {
+	if [ $# -ne 1 ]; then
+		echo "__docker_start need 1 arg <container-name>"
+		exit 1
+	fi
+	tmp=$(docker start $1  2>/dev/null)
+	if [ -z $tmp ] || [ $tmp != $1 ]; then
+		echo " ${1} container not started or not existing"
+	else
+		echo " ${1} container started"
+	fi
+}
+
 # Removes a named container
-docker_rm() {
+__docker_rm() {
 	if [ $# -ne 1 ]; then
-		echo "docker_rm need 1 arg <container-name>"
+		echo "__docker_rm need 1 arg <container-name>"
 		exit 1
 	fi
 	tmp=$(docker rm $1  2>/dev/null)
@@ -308,37 +355,78 @@ docker_rm() {
 	fi
 }
 
-start_dfc_image() {
-	echo "Starting DFC"
-	# Port mappning not needed since dfc is running in host mode
-	docker run -d --network="host" --name dfc_app $DFC_IMAGE > /dev/null
+__start_dfc_image() {
+
+	if [ $# != 2 ]; then
+    	__print_err "need tow args, <dfc-instance-name> 0.."$$DFC_MAX_IDX
+		exit 1
+	fi
+
+	if [ $2 -lt 0 ] || [ $2 -gt $DFC_MAX_IDX ]; then
+		__print_err "need two args, <dfc-instance-name> 0.."$DFC_MAX_IDX
+		exit 1
+	fi
+	appname=$1
+	localport=$(($DFC_PORT + $2))
+	localport_secure=$(($DFC_PORT_SECURE + $2))
+
+	echo "Creating docker network $DOCKER_SIM_NWNAME, if needed"
+
+	docker network ls| grep $DOCKER_SIM_NWNAME > /dev/null || docker network create $DOCKER_SIM_NWNAME
+
+	echo "Starting DFC: " $appname " with ports mapped to " $localport " and " $localport_secure " in docker network "$DOCKER_SIM_NWNAME
+
+	docker run -d -p $localport":8100" -p $localport_secure":8433" --network=$DOCKER_SIM_NWNAME -e CONSUL_HOST=$CONSUL_HOST -e CONSUL_PORT=$CONSUL_PORT -e CONFIG_BINDING_SERVICE=$CONFIG_BINDING_SERVICE -e HOSTNAME=$appname --name $appname $DFC_IMAGE > /dev/null
+
 	dfc_started=false
 	for i in {1..10}; do
-	if [ $(docker inspect --format '{{ .State.Running }}' dfc_app) ]
+	if [ $(docker inspect --format '{{ .State.Running }}' $appname) ]
  	then
-	 	echo " Image: $(docker inspect --format '{{ .Config.Image }}' dfc_app)"
-   		echo "DFC app Running"
+	 	echo " Image: $(docker inspect --format '{{ .Config.Image }}' ${appname})"
+   		echo "DFC container ${appname} running"
 		dfc_started=true
    		break
  	else
-   		echo sleep $i
+   		sleep $i
  	fi
 	done
 	if ! [ $dfc_started  ]; then
-		echo "DFC app could not be started"
+		echo "DFC container ${appname} could not be started"
 		exit 1
 	fi
+
+	dfc_hb=false
+	echo "Waiting for DFC ${appname} heartbeat..."
+	for i in {1..10}; do
+		result="$(__do_curl http://127.0.0.1:${localport}/heartbeat)"
+		if [ $? -eq 0 ]; then
+	   		echo "DFC ${appname} responds to heartbeat: " $result
+	   		dfc_hb=true
+	   		result="$(__do_curl http://127.0.0.1:${localport}/actuator/info)"
+	   		echo "DFC ${appname} image build info: " $result
+	   		break
+	 	else
+	   		sleep $i
+	 	fi
+	done
+
+	if ! [ $dfc_hb  ]; then
+		echo "DFC ${appname} did not respond to heartbeat"
+	fi
 }
 
-#WFunction for waiting for named container to be started manually.
-wait_for_container() {
+# Function for waiting for named container to be started manually.
+__wait_for_container() {
 	start=$SECONDS
-	if [ $# != 1 ]; then
-		echo "Need one arg: <container-name>"
+	if [ $# != 2 ]; then
+		echo "Need one arg: <container-name> <instance-id>"
 		exit 1
 	fi
+	http=$(($DFC_PORT+$2))
+	https=$((DFC_PORT_SECURE+$2))
+	echo "The container is expected to map its ports (8100/8433) to the following port visibile on the host: http port ${http} and https port ${https}"
 	echo "Waiting for container with name '${1}' to be started manually...."
-		
+
 	for (( ; ; ))
 	do
 		if [ $(docker inspect --format '{{ .State.Running }}' $1 2> /dev/null) ]; then
@@ -348,17 +436,22 @@ wait_for_container() {
 	 		duration=$((SECONDS-start))
 			echo -ne "  Waited ${duration} seconds\033[0K\r"
    			sleep 1
- 		fi 
+ 		fi
 	done
+
+	echo "Connecting container "$1" to simulator network "$DOCKER_SIM_NWNAME
+	docker network connect $DOCKER_SIM_NWNAME $1
 }
 
-#WFunction for waiting for named container to be stopped manually. 
-wait_for_container_gone() {
+#WFunction for waiting for named container to be stopped manually.
+__wait_for_container_gone() {
 	start=$SECONDS
 	if [ $# != 1 ]; then
 		echo "Need one arg: <container-name>"
 		exit 1
 	fi
+	echo "Disconnecting container "$1" from simulator network "$DOCKER_SIM_NWNAME
+	docker network disconnect $DOCKER_SIM_NWNAME $1
 	echo "Waiting for container with name '${1}' to be stopped manually...."
 
 	for (( ; ; ))
@@ -370,18 +463,24 @@ wait_for_container_gone() {
 		else
    			echo "Container stopped: "$1
    			break
- 		fi 
+ 		fi
 	done
 }
 
 #Function for waiting to dfc to be started manually
-wait_for_dfc() {
-	read -p "Press enter to continue when dfc has been manually started"
+__wait_for_dfc() {
+	http=$(($DFC_PORT+$2))
+	https=$((DFC_PORT_SECURE+$2))
+	echo "The app is expected to listen to http port ${http} and https port ${https}"
+	echo "The app shall use 'localhost' and '8500' for CONSUL_HOST and CONSUL_PORT."
+	echo "The app shale use 'config-binding-service-localhost' for CONFIG_BINDING_SERVICE"
+	echo "The app shall use ${1} for HOSTNAME."
+	read -p "Press enter to continue when app mapping to ${1} has been manually started"
 }
 
 #Function for waiting to dfc to be stopped manually
-wait_for_dfc_gone() {
-	read -p "Press enter to continue when dfc has been manually stopped"
+__wait_for_dfc_gone() {
+	read -p "Press enter to continue when when app mapping to ${1} has been manually stopped"
 }
 
 #############################################################
@@ -391,28 +490,40 @@ wait_for_dfc_gone() {
 # Print the env variables needed for the simulators and their setup
 log_sim_settings() {
 	echo "Simulator settings"
-	echo "DR_TC=        "$DR_TC
-	echo "DR_REDIR_TC=  "$DR_REDIR_TC
-	echo "MR_TC=        "$MR_TC
-	echo "BC_TC=        "$BC_TC
-	echo "NUM_FTPFILES= "$NUM_FTPFILES
-	echo "NUM_PNFS=     "$NUM_PNFS
-	echo "FILE_SIZE=    "$FILE_SIZE
-	echo "FTP_TYPE=     "$FTP_TYPE
+	echo "MR_TC=                 "$MR_TC
+	echo "MR_GROUPS=             "$MR_GROUPS
+    echo "MR_FILE_PREFIX_MAPPING="$MR_FILE_PREFIX_MAPPING
+	echo "DR_TC=                 "$DR_TC
+	echo "DR_FEEDS=              "$DR_FEEDS
+	echo "DR_REDIR_SIM=          "$DR_REDIR_SIM
+	echo "DR_REDIR_TC=           "$DR_REDIR_TC
+	echo "DR_REDIR_FEEDS=        "$DR_REDIR_FEEDS
+
+	echo "NUM_FTPFILES=          "$NUM_FTPFILES
+	echo "NUM_PNFS=              "$NUM_PNFS
+	echo "FILE_SIZE=             "$FILE_SIZE
+	echo "FTP_TYPE=              "$FTP_TYPE
+	echo "FTP_FILE_PREFIXES=     "$FTP_FILE_PREFIXES
+	echo "NUM_FTP_SERVERS=       "$NUM_FTP_SERVERS
+	echo "SFTP_SIMS=             "$SFTP_SIMS
+	echo "FTPS_SIMS=             "$FTPS_SIMS
 	echo ""
 }
 
 # Stop and remove all containers including dfc app and simulators
 clean_containers() {
-	echo "Stopping all containers, dfc app and simulators with name prefix 'dfc_'"
+	echo "Stopping all containers, dfc app(s) and simulators with name prefix 'dfc_'"
 	docker stop $(docker ps -q --filter name=dfc_) &> /dev/null
 	echo "Removing all containers, dfc app and simulators with name prefix 'dfc_'"
 	docker rm $(docker ps -a -q --filter name=dfc_) &> /dev/null
+	echo "Removing unused docker networks with substring 'dfc' in network name"
+	docker network rm $(docker network ls -q --filter name=dfc)
 	echo ""
 }
 
 # Start all simulators in the simulator group
 start_simulators() {
+
 	echo "Starting all simulators"
 	curdir=$PWD
 	cd $SIM_GROUP
@@ -424,119 +535,304 @@ start_simulators() {
 # Start the dfc application
 start_dfc() {
 
+	if [ $# != 1 ]; then
+    	__print_err "need one arg, <dfc-instance-id>"
+		exit 1
+	fi
+
+	if [ $1 -lt 0 ] || [ $1 -gt $DFC_MAX_IDX ]; then
+		__print_err "arg should be 0.."$DFC_MAX_IDX
+		exit 1
+	fi 
+	appname=$DFC_APP_BASE$1
+	STARTED_DFCS=$STARTED_DFCS"_"$appname"_"
+
 	if [ $START_ARG == "local" ] || [ $START_ARG == "remote" ] ||  [ $START_ARG == "remote-remove" ]; then
-		start_dfc_image
+		__start_dfc_image $appname $1
 	elif [ $START_ARG == "manual-container" ]; then
-		wait_for_container dfc_app
+		__wait_for_container $appname $1
 	elif [ $START_ARG == "manual-app" ]; then
-		wait_for_dfc
+		__wait_for_dfc $appname $1
+	fi
+}
+
+# Configure consul with dfc config, args <app|dmaap> <dfc-instance-id> <json-file-path>
+# Not intended to be called directly by test scripts.
+__consul_config() {
+
+	if [ $# != 3 ]; then
+    	__print_err "need three args, <app|dmaap> <dfc-instance-id> <json-file-path>"
+		exit 1
+	fi
+
+	if [ $2 -lt 0 ] || [ $2 -gt $DFC_MAX_IDX ]; then
+		__print_err "dfc-instance-id should be 0.."$DFC_MAX_IDX
+		exit 1
+	fi
+	if ! [ -f $3 ]; then
+		__print_err "json file does not extis: "$3
+		exit 1
+	fi
+
+	if [ $1 == "app" ]; then
+		appname=$DFC_APP_BASE$2
+	elif [ $1 == "dmaap" ]; then
+		appname=$DFC_APP_BASE$2":dmaap"
+	else
+		__print_err "config type should be 'app' or 'dmaap'"
+		exit 1
+	fi
+
+	echo "Configuring consul for " $appname " from " $3
+	curl -s http://127.0.0.1:${CONSUL_PORT}/v1/kv/${appname}?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary "@"$3 >/dev/null
+}
+
+# Configure consul with dfc app config, args <dfc-instance-id> <json-file-path>
+consul_config_app() {
+	if [ $START_ARG == "manual-app" ]; then
+		echo "Replacing 'mrsim' with 'localhost' in json app config for consul"
+		sed 's/mrsim/localhost/g' $2 > .tmp_app.json
+		__consul_config app $1 .tmp_app.json
+	else
+		__consul_config app $1 $2
+	fi
+}
+
+# Configure consul with dfc dmaap config, args <dfc-instance-id> <json-file-path>
+consul_config_dmaap() {
+	if [ $START_ARG == "manual-app" ]; then
+		echo "Replacing 'drsim' with 'localhost' in json dmaap config for consul"
+		sed 's/drsim/localhost/g' $2 > .tmp_dmaap.json
+		__consul_config dmaap $1 .tmp_dmaap.json
+	else
+		__consul_config dmaap $1 $2
 	fi
 }
 
 # Stop and remove the dfc app container
 kill_dfc() {
-	echo "Killing DFC"
+
+	if [ $# != 1 ]; then
+    	__print_err "need one arg, <dfc-instance-id>"
+		exit 1
+	fi
+
+	if [ $1 -lt 0 ] || [ $1 -gt $DFC_MAX_IDX ]; then
+		__print_err "arg should be 0.."$DFC_MAX_IDX
+		exit 1
+	fi
+	appname=$DFC_APP_BASE$1
+
+	echo "Killing DFC, instance id: "$1
 
 	if [ $START_ARG == "local" ] || [ $START_ARG == "remote" ] ||  [ $START_ARG == "remote-remove" ]; then
-		docker_stop dfc_app
-		docker_rm dfc_app
+		__docker_stop $appname
+		__docker_rm $appname
 	elif [ $START_ARG == "manual-container" ]; then
-		wait_for_container_gone dfc_app
+		__wait_for_container_gone $appname
 	elif [ $START_ARG == "manual-app" ]; then
-		wait_for_dfc_gone
+		__wait_for_dfc_gone $appname
 	fi
 }
 
 # Stop and remove the DR simulator container
 kill_dr() {
 	echo "Killing DR sim"
-	docker_stop dfc_dr-sim
-	docker_rm dfc_dr-sim
+	__docker_stop dfc_dr-sim
+	__docker_rm dfc_dr-sim
 }
 
 # Stop and remove the DR redir simulator container
 kill_drr() {
 	echo "Killing DR redir sim"
-	docker_stop dfc_dr-redir-sim
-	docker_rm dfc_dr-redir-sim
+	__docker_stop dfc_dr-redir-sim
+	__docker_rm dfc_dr-redir-sim
 }
 
 # Stop and remove the MR simulator container
 kill_mr() {
 	echo "Killing MR sim"
-	docker_stop dfc_mr-sim
-	docker_rm dfc_mr-sim
+	__docker_stop dfc_mr-sim
+	__docker_rm dfc_mr-sim
 }
 
-# Stop and remove the SFTP container
+# Stop and remove the SFTP container, arg: <sftp-instance-id>
 kill_sftp() {
-	echo "Killing SFTP"
-	docker_stop dfc_sftp-server
-	docker_rm dfc_sftp-server
+
+	if [ $# != 1 ]; then
+    	__print_err "need one arg, <sftp-instance-id>"
+		exit 1
+	fi
+
+	if [ $1 -lt 0 ] || [ $1 -gt $FTP_MAX_IDX ]; then
+		__print_err "arg should be 0.."$FTP_MAX_IDX
+		exit 1
+	fi
+	appname=$SFTP_BASE$1
+
+	echo "Killing SFTP, instance id: "$1
+
+	__docker_stop $appname
+	__docker_rm $appname
 }
 
-# Stop and remove the FTPS container
+# Stop SFTP container, arg: <sftp-instance-id>
+stop_sftp() {
+
+	if [ $# != 1 ]; then
+    	__print_err "need one arg, <sftp-instance-id>"
+		exit 1
+	fi
+
+	if [ $1 -lt 0 ] || [ $1 -gt $FTP_MAX_IDX ]; then
+		__print_err "arg should be 0.."$FTP_MAX_IDX
+		exit 1
+	fi
+	appname=$SFTP_BASE$1
+
+	echo "Stopping SFTP, instance id: "$1
+
+	__docker_stop $appname
+}
+
+# Starts a stopped SFTP container, arg: <sftp-instance-id>
+start_sftp() {
+
+	if [ $# != 1 ]; then
+    	__print_err "need one arg, <sftp-instance-id>"
+		exit 1
+	fi
+
+	if [ $1 -lt 0 ] || [ $1 -gt $FTP_MAX_IDX ]; then
+		__print_err "arg should be 0.."$FTP_MAX_IDX
+		exit 1
+	fi
+	appname=$SFTP_BASE$1
+
+	echo "Starting SFTP, instance id: "$1
+
+	__docker_start $appname
+}
+
+# Stop and remove the FTPS container, arg: <ftps-instance-id>
 kill_ftps() {
-	echo "Killing FTPS"
-	docker_stop dfc_ftpes-server-vsftpd
-	docker_rm dfc_ftpes-server-vsftpd
+
+	if [ $# != 1 ]; then
+    	__print_err "need one arg, <ftpS-instance-id>"
+		exit 1
+	fi
+
+	if [ $1 -lt 0 ] || [ $1 -gt $FTP_MAX_IDX ]; then
+		__print_err "arg should be 0.."$FTP_MAX_IDX
+		exit 1
+	fi
+	appname=$FTPS_BASE$1
+
+	echo "Killing FTPS, instance id: "$1
+
+	__docker_stop $appname
+	__docker_rm $appname
+}
+
+# Stop FTPS container, arg: <ftps-instance-id>
+stop_ftps() {
+
+	if [ $# != 1 ]; then
+    	__print_err "need one arg, <ftps-instance-id>"
+		exit 1
+	fi
+
+	if [ $1 -lt 0 ] || [ $1 -gt $FTP_MAX_IDX ]; then
+		__print_err "arg should be 0.."$FTP_MAX_IDX
+		exit 1
+	fi
+	appname=$FTPS_BASE$1
+
+	echo "Stopping FTPS, instance id: "$1
+
+	__docker_stop $appname
+}
+
+# Starts a stopped FTPS container, arg: <ftps-instance-id>
+start_ftps() {
+
+	if [ $# != 1 ]; then
+    	__print_err "need one arg, <ftps-instance-id>"
+		exit 1
+	fi
+
+	if [ $1 -lt 0 ] || [ $1 -gt $FTP_MAX_IDX ]; then
+		__print_err "arg should be 0.."$FTP_MAX_IDX
+		exit 1
+	fi
+	appname=$FTPS_BASE$1
+
+	echo "Starting FTPS, instance id: "$1
+
+	__docker_start $appname
 }
 
 # Print a variable value from the MR simulator. Arg: <variable-name>
 mr_print() {
 	if [ $# != 1 ]; then
-    	print_err "need one arg, <sim-param>"
+    	__print_err "need one arg, <sim-param>"
 		exit 1
 	fi
-	echo -e "---- MR sim, \033[1m $1 \033[0m: $(do_curl http://127.0.0.1:2222/$1)"
+	echo -e "---- MR sim, \033[1m $1 \033[0m: $(__do_curl http://127.0.0.1:$MR_PORT/$1)"
 }
 
 # Print a variable value from the DR simulator. Arg: <variable-name>
 dr_print() {
 	if [ $# != 1 ]; then
-    	print_err "need one arg, <sim-param>"
+    	__print_err "need one arg, <sim-param>"
 		exit 1
 	fi
-	echo -e "---- DR sim, \033[1m $1 \033[0m: $(do_curl http://127.0.0.1:3906/$1)"
+	echo -e "---- DR sim, \033[1m $1 \033[0m: $(__do_curl http://127.0.0.1:$DR_PORT/$1)"
 }
 
 # Print a variable value from the DR redir simulator. Arg: <variable-name>
 drr_print() {
 	if [ $# != 1 ]; then
-    	print_err "need one arg, <sim-param>"
+    	__print_err "need one arg, <sim-param>"
 		exit 1
 	fi
-	echo -e "---- DR redir sim, \033[1m $1 \033[0m: $(do_curl http://127.0.0.1:3908/$1)"
+	echo -e "---- DR redir sim, \033[1m $1 \033[0m: $(__do_curl http://127.0.0.1:$DRR_PORT/$1)"
 }
-# Print a variable value from dfc. Arg: <variable-name>
+# Print a variable value from dfc. Arg: <dfc-instance-id> <variable-name>
 dfc_print() {
-	if [ $# != 1 ]; then
-    	print_err "need one arg, <dfc-param>"
+	if [ $# != 2 ]; then
+    	__print_err "need two args, <dfc-instance-id> <dfc-param>"
+		exit 1
+	fi
+	if [ $1 -lt 0 ] || [ $1 -gt $DFC_MAX_IDX ]; then
+		__print_err "dfc instance id should be in range 0.."DFC_MAX_IDX
 		exit 1
 	fi
-	echo -e "---- DFC, \033[1m $1 \033[0m: $(do_curl http://127.0.0.1:8100/$1)"
+	localport=$(($DFC_PORT + $1))
+	appname=$DFC_APP_BASE$1
+	echo -e "---- DFC $appname, \033[1m $2 \033[0m: $(__do_curl http://127.0.0.1:$localport/$2)"
 }
 
-# Read a variable value from MR sim and send to stdout.
+# Read a variable value from MR sim and send to stdout. Arg: <variable-name>
 mr_read() {
-	echo "$(do_curl http://127.0.0.1:2222/$1)"
+	echo "$(__do_curl http://127.0.0.1:$MR_PORT/$1)"
 }
 
-# Read a variable value from DR sim and send to stdout.
+# Read a variable value from DR sim and send to stdout. Arg: <variable-name>
 dr_read() {
-	echo "$(do_curl http://127.0.0.1:3906/$1)"
+	echo "$(__do_curl http://127.0.0.1:$DR_PORT/$1)"
 }
 
-# Read a variable value from DR redir sim and send to stdout.
+# Read a variable value from DR redir sim and send to stdout. Arg: <variable-name>
 drr_read() {
-	echo "$(do_curl http://127.0.0.1:3908/$1)"
+	echo "$(__do_curl http://127.0.0.1:$DRR_PORT/$1)"
 }
 
 
 # Sleep. Arg: <sleep-time-in-sec>
 sleep_wait() {
 	if [ $# != 1 ]; then
-		print_err "need one arg, <sleep-time-in-sec>"
+		__print_err "need one arg, <sleep-time-in-sec>"
 		exit 1
 	fi
 	echo "---- Sleep for " $1 " seconds ----"
@@ -547,25 +843,36 @@ sleep_wait() {
 		sleep 1
 		duration=$((SECONDS-start))
 	done
+	echo ""
 }
 
 # Sleep and print dfc heartbeat. Arg: <sleep-time-in-sec>
 sleep_heartbeat() {
 	if [ $# != 1 ]; then
-		print_err "need one arg, <sleep-time-in-sec>"
+		__print_err "need one arg, <sleep-time-in-sec>"
 		exit 1
 	fi
 	echo "---- Sleep for " $1 " seconds ----"
+	echo ""
 	start=$SECONDS
 	duration=$((SECONDS-start))
 	ctr=0
+	rows=0
 	while [ $duration -lt $1 ]; do
+		if [ $rows -eq 0 ]; then
+			tput cuu1
+		fi
+		rows=0
+		echo "  Slept for ${duration} seconds"
 		if [ $((ctr%30)) -eq 0 ]; then
-			echo -ne "  Slept for ${duration} seconds, \033[1m heartbeat \033[0m "$(do_curl http://127.0.0.1:8100/heartbeat)
-			echo ""
-		else
-			echo -ne "  Slept for ${duration} seconds, \033[1m heartbeat \033[0m "$(do_curl http://127.0.0.1:8100/heartbeat)" \033[0K\r"
+			for (( i=0; i<=$DFC_MAX_IDX; i++ )); do
+				if [[ $STARTED_DFCS =~ "_"$DFC_APP_BASE$i"_" ]]; then
+					let rows=rows+1
+					echo "    HB ${DFC_APP_BASE}${i}: $(__do_curl http://127.0.0.1:$(($DFC_PORT+$i))/heartbeat)"
+				fi
+			done
 		fi
+
 		let ctr=ctr+1
 		sleep 1
 		duration=$((SECONDS-start))
@@ -581,9 +888,9 @@ sleep_heartbeat() {
 # value or not.
 mr_equal() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "MR" "http://127.0.0.1:2222/" $1 "=" $2 $3
+		__var_test "MR" "http://127.0.0.1:$MR_PORT/" $1 "=" $2 $3
 	else
-		print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
@@ -595,9 +902,9 @@ mr_equal() {
 # value or not.
 mr_greater() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "MR" "http://127.0.0.1:2222/" $1 ">" $2 $3
+		__var_test "MR" "http://127.0.0.1:$MR_PORT/" $1 ">" $2 $3
 	else
-		print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
@@ -609,9 +916,9 @@ mr_greater() {
 # value or not.
 mr_less() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "MR" "http://127.0.0.1:2222/" $1 "<" $2 $3
+		__var_test "MR" "http://127.0.0.1:$MR_PORT/" $1 "<" $2 $3
 	else
-		print_err "Wrong args to mr_less, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to mr_less, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
@@ -623,9 +930,9 @@ mr_less() {
 # value or not.
 mr_contain_str() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "MR" "http://127.0.0.1:2222/" $1 "contain_str" $2 $3
+		__var_test "MR" "http://127.0.0.1:$MR_PORT/" $1 "contain_str" $2 $3
 	else
-		print_err "Wrong args to mr_contain_str, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to mr_contain_str, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
@@ -637,9 +944,9 @@ mr_contain_str() {
 # value or not.
 dr_equal() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "DR" "http://127.0.0.1:3906/" $1 "=" $2 $3
+		__var_test "DR" "http://127.0.0.1:$DR_PORT/" $1 "=" $2 $3
 	else
-		print_err "Wrong args to dr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to dr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
@@ -651,9 +958,9 @@ dr_equal() {
 # value or not.
 dr_greater() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "DR" "http://127.0.0.1:3906/" $1 ">" $2 $3
+		__var_test "DR" "http://127.0.0.1:$DR_PORT/" $1 ">" $2 $3
 	else
-		print_err "Wrong args to dr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to dr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
@@ -665,9 +972,23 @@ dr_greater() {
 # value or not.
 dr_less() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "DR" "http://127.0.0.1:3906/" $1 "<" $2 $3
+		__var_test "DR" "http://127.0.0.1:$DR_PORT/" $1 "<" $2 $3
+	else
+		__print_err "Wrong args to dr_less, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the DR simulator contains the target string and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable contains
+# the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value contains the target
+# value or not.
+dr_contain_str() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		__var_test "DR" "http://127.0.0.1:$DR_PORT/" $1 "contain_str" $2 $3
 	else
-		print_err "Wrong args to dr_less, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to dr_contain_str, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
@@ -679,9 +1000,9 @@ dr_less() {
 # value or not.
 drr_equal() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "DR REDIR" "http://127.0.0.1:3908/" $1 "=" $2 $3
+		__var_test "DR REDIR" "http://127.0.0.1:$DRR_PORT/" $1 "=" $2 $3
 	else
-		print_err "Wrong args to drr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to drr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
@@ -694,9 +1015,9 @@ drr_equal() {
 # value or not.
 drr_greater() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "DR REDIR" "http://127.0.0.1:3908/" $1 ">" $2 $3
+		__var_test "DR REDIR" "http://127.0.0.1:$DRR_PORT/" $1 ">" $2 $3
 	else
-		print_err "Wrong args to drr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to drr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
@@ -708,27 +1029,47 @@ drr_greater() {
 # value or not.
 drr_less() {
 	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
-		var_test "DR REDIR" "http://127.0.0.1:3908/" $1 "<" $2 $3
+		__var_test "DR REDIR" "http://127.0.0.1:$DRR_PORT/" $1 "<" $2 $3
+	else
+		__print_err "Wrong args to drr_less, needs two or three args: <sim-param> <target-value> [ timeout ]"
+	fi
+}
+
+# Tests if a variable value in the DR redir simulator contains the target string and and optional timeout.
+# Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable contains
+# the target or not.
+# Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
+# before setting pass or fail depending on if the variable value contains the target
+# value or not.
+drr_contain_str() {
+	if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+		__var_test "DR REDIR" "http://127.0.0.1:$DRR_PORT/" $1 "contain_str" $2 $3
 	else
-		print_err "Wrong args to drr_less, needs two or three args: <sim-param> <target-value> [ timeout ]"
+		__print_err "Wrong args to drr_contain_str, needs two or three args: <sim-param> <target-value> [ timeout ]"
 	fi
 }
 
-#Test is a variable in the DFC contains a substring. Arg: <variable-name> <substring-in-quotes>
+#Test if a variable in the DFC contains a substring. Arg: <dfc-index> <variable-name> <substring-in-quotes>
 dfc_contain_str() {
-	if [ $# -eq 2 ]; then
-		echo -e "---- DFC test criteria: \033[1m ${1} \033[0m contains: ${2} ----"
+	if [ $# -eq 3 ]; then
+		if [ $1 -lt 0 ] || [ $1 -gt $DFC_MAX_IDX ]; then
+			__print_err "arg should be 0.."$DFC_MAX_IDX
+			exit 1
+		fi
+		appname=$DFC_APP_BASE$1
+		localport=$(($DFC_PORT + $1))
+		echo -e "---- DFC test criteria: $appname \033[1m ${2} \033[0m contains: ${3} ----"
 		((RES_TEST++))
-		result="$(do_curl http://127.0.0.1:8100/${1})"
-		if [[ $result =~ $2 ]]; then
+		result="$(__do_curl http://127.0.0.1:$localport/${2})"
+		if [[ $result =~ $3 ]]; then
 			((RES_PASS++))
-			echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met"		
+			echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met"
 		else
 			((RES_FAIL++))
-			echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${1} not reached, result = ${result} ----"
+			echo -e "----  \033[31m\033[1mFAIL\033[0m - Target '${3}' not reached, result = ${result} ----"
 		fi
-	else 
-		echo "Wrong args to dfc_contain_str, needs two arg: <dfc-variable> <str>"
+	else
+		echo "Wrong args to dfc_contain_str, needs three arg: <dfc-index> <dfc-variable> <str>"
 		exit 1
 	fi
 }
@@ -737,41 +1078,66 @@ dfc_contain_str() {
 # separate logs stored at different steps in the test script. Arg: <tc-id> <log-prefix>
 store_logs() {
 	if [ $# != 1 ]; then
-    	print_err "need one arg, <file-prefix>"
+    	__print_err "need one arg, <file-prefix>"
 		exit 1
 	fi
 	echo "Storing all container logs and dfc app log using prefix: "$1
 	if ! [ $START_ARG == "manual-app" ]; then
-		docker cp dfc_app:/var/log/ONAP/application.log $TESTLOGS/$ATC/$1_application.log
-		docker logs dfc_app > $TESTLOGS/$ATC/$1_dfc_app-docker.log 2>&1
+		for (( i=0; i<=$DFC_MAX_IDX; i++ )); do
+			appname=$DFC_APP_BASE$i
+			tmp=$(docker ps | grep $appname)
+			if ! [ -z "$tmp" ]; then   #Only stored logs from running DFC apps
+				docker cp $appname:/var/log/ONAP/application.log $TESTLOGS/$ATC/${1}_${appname}_application.log
+				docker logs $appname > $TESTLOGS/$ATC/$1_$appname-docker.log 2>&1
+			fi
+		done
 	fi
 	docker logs dfc_mr-sim > $TESTLOGS/$ATC/$1_dfc_mr-sim-docker.log 2>&1
 	docker logs dfc_dr-sim > $TESTLOGS/$ATC/$1_dfc_dr-sim-docker.log 2>&1
 	docker logs dfc_dr-redir-sim > $TESTLOGS/$ATC/$1_dfc_dr-redir-sim-docker.log 2>&1
-	docker logs dfc_ftpes-server-vsftpd > $TESTLOGS/$ATC/$1_dfc_ftpes-server-vsftpd.log 2>&1
-	docker logs dfc_sftp-server > $TESTLOGS/$ATC/$1_dfc_sftp-server.log 2>&1
+
+	for (( i=0; i<=$FTP_MAX_IDX; i++ )); do
+		appname=$SFTP_BASE$i
+		docker logs $appname > $TESTLOGS/$ATC/${1}_${appname}.log 2>&1
+		appname=$FTPS_BASE$i
+		docker logs $appname > $TESTLOGS/$ATC/${1}_${appname}.log 2>&1
+	done
+
+	docker logs dfc_consul > $TESTLOGS/$ATC/$1_consul.log 2>&1
+	docker logs dfc_cbs > $TESTLOGS/$ATC/$1_cbs.log 2>&1
+}
+# Check the dfc application log, for all dfc instances, for WARN and ERR messages and print the count.
+check_dfc_logs() {
+	for (( i=0; i<=$DFC_MAX_IDX; i++ )); do
+		appname=$DFC_APP_BASE$i
+		tmp=$(docker ps | grep $appname)
+		if ! [ -z "$tmp" ]; then  #Only check logs for running dfc_apps
+			_check_dfc_log $appname
+		fi
+	done
 }
-# Check the dfc application log for WARN and ERR messages and print the count.
-check_dfc_log() {
-	echo "Checking dfc log /var/log/ONAP/application.log for WARNINGs and ERRORs, excluding messages from CONSUL"
-	foundentries=$(docker exec -it dfc_app grep WARN /var/log/ONAP/application.log | grep -iv CONSUL | wc -l)
+
+# Check dfc app log for one dfc instance, arg <dfc-app-name>
+_check_dfc_log() {
+	echo "Checking $1 log $DFC_LOGPATH for WARNINGs and ERRORs"
+	foundentries=$(docker exec -it $1 grep WARN /var/log/ONAP/application.log | wc -l)
 	if [ $? -ne  0 ];then
-		echo "  Problem to search dfc log /var/log/ONAP/application.log"
+		echo "  Problem to search $1 log $DFC_LOGPATH"
 	else
 		if [ $foundentries -eq 0 ]; then
-			echo "  No WARN entries found in dfc log /var/log/ONAP/application.log"
-		else 
-			echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in dfc log /var/log/ONAP/application.log"
+			echo "  No WARN entries found in $1 log $DFC_LOGPATH"
+		else
+			echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $1 log $DFC_LOGPATH"
 		fi
 	fi
-		foundentries=$(docker exec -it dfc_app grep ERR /var/log/ONAP/application.log | grep -iv CONSUL | wc -l)
+	foundentries=$(docker exec -it $1 grep ERR $DFC_LOGPATH | wc -l)
 	if [ $? -ne  0 ];then
-		echo "  Problem to search dfc log /var/log/ONAP/application.log"
+		echo "  Problem to search $1 log $DFC_LOGPATH"
 	else
 		if [ $foundentries -eq 0 ]; then
-			echo "  No ERR entries found in dfc log /var/log/ONAP/application.log"
-		else 
-			echo -e "  Found \033[1m"$foundentries"\033[0m ERR entries in dfc log /var/log/ONAP/application.log"
+			echo "  No ERR entries found in $1 log $DFC_LOGPATH"
+		else
+			echo -e "  Found \033[1m"$foundentries"\033[0m ERR entries in $1 log $DFC_LOGPATH"
 		fi
 	fi
 }
@@ -780,37 +1146,76 @@ print_all() {
 
 	echo "---- DFC and all sim variables"
 
-	dfc_print heartbeat
-	
+	for (( i=0; i<=$DFC_MAX_IDX; i++ )); do
+		appname=$DFC_APP_BASE$i
+		tmp=$(docker ps | grep $appname)
+		if ! [ -z "$tmp" ]; then  #Only check running dfc_apps
+			dfc_print $i status
+		fi
+	done
+
+
 	mr_print tc_info
+	mr_print status
 	mr_print execution_time
+	mr_print groups
+	mr_print changeids
+	mr_print fileprefixes
 	mr_print exe_time_first_poll
+	mr_print groups/exe_time_first_poll
 	mr_print ctr_requests
+	mr_print groups/ctr_requests
 	mr_print ctr_responses
+	mr_print groups/ctr_responses
 	mr_print ctr_files
+	mr_print groups/ctr_files
 	mr_print ctr_unique_files
+	mr_print groups/ctr_unique_files
+	mr_print groups/ctr_events
 	mr_print ctr_events
 	mr_print ctr_unique_PNFs
+	mr_print groups/ctr_unique_PNFs
 
 	dr_print tc_info
 	dr_print execution_time
+	dr_print feeds
 	dr_print ctr_publish_query
+	dr_print feeds/ctr_publish_query
+	dr_print ctr_publish_query_bad_file_prefix
+	dr_print feeds/ctr_publish_query_bad_file_prefix
 	dr_print ctr_publish_query_published
+	dr_print feeds/ctr_publish_query_published
 	dr_print ctr_publish_query_not_published
+	dr_print feeds/ctr_publish_query_not_published
 	dr_print ctr_publish_req
+	dr_print feeds/ctr_publish_req
+	dr_print ctr_publish_req_bad_file_prefix
+	dr_print feeds/ctr_publish_req_bad_file_prefix
 	dr_print ctr_publish_req_redirect
+	dr_print feeds/ctr_publish_req_redirect
 	dr_print ctr_publish_req_published
+	dr_print feeds/ctr_publish_req_published
 	dr_print ctr_published_files
+	dr_print feeds/ctr_published_files
+	dr_print ctr_double_publish
+	dr_print feeds/ctr_double_publish
 
 	drr_print tc_info
 	drr_print execution_time
+	drr_print feeds
 	drr_print ctr_publish_requests
+	drr_print feeds/ctr_publish_requests
+	drr_print ctr_publish_requests_bad_file_prefix
+	drr_print feeds/ctr_publish_requests_bad_file_prefix
 	drr_print ctr_publish_responses
+	drr_print feeds/ctr_publish_responses
 	drr_print dwl_volume
+	drr_print feeds/dwl_volume
 	drr_print time_lastpublish
+	drr_print feeds/time_lastpublish
 }
 
-# Print the test result 
+# Print the test result
 print_result() {
 
 	TCTEST_END=$SECONDS
@@ -830,10 +1235,24 @@ print_result() {
 	total=$((RES_PASS+RES_FAIL))
 	if [ $RES_TEST -eq 0 ]; then
 		echo -e "\033[1mNo tests seem to have executed. Check the script....\033[0m"
+ 		echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
+ 		echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
+		echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
+ 		echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 	elif [ $total != $RES_TEST ]; then
 		echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
+		echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
+		echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
+		echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
+ 		echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 	elif [ $RES_PASS = $RES_TEST ]; then
 		echo -e "All tests \033[32m\033[1mPASS\033[0m"
+		echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
+		echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
+		echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
+		echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
+		echo ""
+
 		# Update test suite counter
 		if [ -f .tmp_tcsuite_pass_ctr ]; then
 			tmpval=$(< .tmp_tcsuite_pass_ctr)
@@ -845,6 +1264,11 @@ print_result() {
 		fi
 	else
 		echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
+		echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
+		echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
+		echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
+		echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
+		echo ""
 		# Update test suite counter
 		if [ -f .tmp_tcsuite_fail_ctr ]; then
 			tmpval=$(< .tmp_tcsuite_fail_ctr)
diff --git a/test/mocks/datafilecollector-testharness/common/testsuite_common.sh b/test/mocks/datafilecollector-testharness/common/testsuite_common.sh
index 2476f6ccf..e35d670de 100755
--- a/test/mocks/datafilecollector-testharness/common/testsuite_common.sh
+++ b/test/mocks/datafilecollector-testharness/common/testsuite_common.sh
@@ -50,7 +50,7 @@ suite_setup() {
     touch .tmp_tcsuite_fail
 }
 
-print_err() {
+__print_err() {
     echo ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[$i+2]} " line" ${BASH_LINENO[$i+1]}
 }