CSIT uplift for CBS, generalized DFC, bugfixes, stability, traceability

Issue-ID: DCAEGEN2-1719
Change-Id: If5c96a7f205a34f78e50585a7187c5b9a3820023
Signed-off-by: TamasBakai <tamas.bakai@est.tech>

11 files changed, 340 insertions, 168 deletions

diff --git a/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh b/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh
index e2a0c126..2d276e51 100644
--- a/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh
+++ b/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh
@@ -3,14 +3,26 @@
 #Stop all running containers
 docker kill "$(docker ps -q -a)"
 docker rm "$(docker ps -q -a)"
+docker system prune -f
 
 # Clone Simulators for DFC from integration repo.
 mkdir -p $WORKSPACE/archives/dfc
 cd $WORKSPACE/archives/dfc
-git clone --depth 1 https://gerrit.onap.org/r/integration -b master
 
-#Location of all individual simulators for DFC
-SIM_ROOT=$WORKSPACE/archives/dfc/integration/test/mocks/datafilecollector-testharness
+
+if [ -z "$SIM_ROOT" ]
+then
+	#git clone --depth 1 https://gerrit.onap.org/r/integration -b master
+	git clone https://github.com/yanhuanwang/integration.git
+	#Location of all individual simulators for DFC
+	echo "Determine SIM_ROOT based on the WORKSPACE"
+	SIM_ROOT=$WORKSPACE/archives/dfc/integration/test/mocks/datafilecollector-testharness
+
+	rm $SIM_ROOT/simulator-group/consul/consul/cbs_localhost_config.hcl || true
+else
+	echo "Using SIM_ROOT from environmental variable: " $SIM_ROOT
+fi
+
 #Location of the above simulators when run as a group. For start+config and stop.
 SIMGROUP_ROOT=$SIM_ROOT/simulator-group
 
@@ -20,7 +32,11 @@ SIM_IP="127.0.0.1"
 DFC_ROOT=$WORKSPACE/scripts/dcaegen2-collectors-datafile/dfc-management
 
 #Make the env vars availble to the robot scripts
-ROBOT_VARIABLES="-v SIMGROUP_ROOT:${SIMGROUP_ROOT} -v SIM_IP:${SIM_IP} -v DFC_ROOT:${DFC_ROOT}"
+ROBOT_VARIABLES="-b debug.log -v SIMGROUP_ROOT:${SIMGROUP_ROOT} -v SIM_IP:${SIM_IP} -v DFC_ROOT:${DFC_ROOT}"
+
+
+
+
 
 #Build needed simulator images. DR and MR simulators
 
@@ -40,5 +56,9 @@ cd $SIMGROUP_ROOT
 cp -r ../ftps-sftp-server/configuration .
 cp -r ../ftps-sftp-server/tls .
 
+cd ../ftps-sftp-server
+docker build -t ftps_vsftpd:latest -f Dockerfile-ftps .
+
+
 #All containers will be started and stopped via the robot tests.
 
diff --git a/plans/dcaegen2-collectors-datafile/Functional-suite/teardown.sh b/plans/dcaegen2-collectors-datafile/Functional-suite/teardown.sh
index 1a0ec6e1..0dc2af18 100644
--- a/plans/dcaegen2-collectors-datafile/Functional-suite/teardown.sh
+++ b/plans/dcaegen2-collectors-datafile/Functional-suite/teardown.sh
@@ -1,13 +1,14 @@
 #!/usr/bin/env bash
 
-#Stop dfc
+running_containers=$(docker ps --filter name=dfc_ -q)
 
-kill-instance.sh dfc_app
 
-#Stop all simulators 
+if [ -z "$running_containers" ]
+then
+    echo "No container requires termination"
+else
+    echo "Stopping and removing containers"
+    docker stop $running_containers
+    docker rm $running_containers
+fi
 
-kill-instance.sh dfc_dr-sim
-kill-instance.sh dfc_dr-redir-sim
-kill-instance.sh dfc_mr-sim
-kill-instance.sh dfc_sftp-server
-kill-instance.sh dfc_ftpes-server-vsftpd
\ No newline at end of file
diff --git a/plans/dcaegen2-collectors-datafile/ManagementInterface-suite/teardown.sh b/plans/dcaegen2-collectors-datafile/ManagementInterface-suite/teardown.sh
index 97ef4037..d8877fbc 100644
--- a/plans/dcaegen2-collectors-datafile/ManagementInterface-suite/teardown.sh
+++ b/plans/dcaegen2-collectors-datafile/ManagementInterface-suite/teardown.sh
@@ -1,3 +1,6 @@
 #!/usr/bin/env bash
-
-kill-instance.sh dfc_app
\ No newline at end of file
+set +e
+docker exec -it dfc_app0 cat /var/log/ONAP/application.log > $WORKSPACE/archives/dfc_app0_application.log
+kill-instance.sh dfc_app0
+set -e
+exit 0
diff --git a/plans/so/integration-etsi-testing/config/distribution-test-zip/distributionTestZip.zip b/plans/so/integration-etsi-testing/config/distribution-test-zip/distributionTestZip.zip
index e3152841..ac96bd94 100644
--- a/plans/so/integration-etsi-testing/config/distribution-test-zip/distributionTestZip.zip
+++ b/plans/so/integration-etsi-testing/config/distribution-test-zip/distributionTestZip.zip
diff --git a/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh b/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh
index 03919343..0e7c34ea 100755
--- a/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh
+++ b/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh
@@ -1,11 +1,15 @@
 #!/bin/bash
-
 # Kill dfc and all simulator
 
-docker kill dfc_app
-docker kill dfc_dr-sim
-docker kill dfc_dr-redir-sim
-docker kill dfc_mr-sim
-docker kill dfc_sftp-server
-docker kill dfc_ftpes-server-vsftpd
+running_containers=$(docker ps --filter name=dfc_ -q)
+
+
+if [ -z "$running_containers" ]
+then
+    echo "No container requires termination"
+else
+    echo "Stopping and removing containers"
+    docker stop $running_containers
+    docker rm $running_containers
+fi
 
diff --git a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-kill.sh b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-kill.sh
index 39f2d0d7..491b6405 100755
--- a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-kill.sh
+++ b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-kill.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 
-docker kill dfc_app
\ No newline at end of file
+docker kill dfc_app0
\ No newline at end of file
diff --git a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh
index 8478bcb0..c79467a4 100755
--- a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh
+++ b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh
@@ -1,10 +1,10 @@
 #!/bin/bash
 
-#Restart DFC app 
+#Restart DFC app
 
-docker restart dfc_app
+docker restart dfc_app0
 
-DFC_APP="$(docker ps -q --filter='name=dfc_app')"
+DFC_APP="$(docker ps -q --filter='name=dfc_app0')"
 
 
 #Wait for initialization of docker container for dfc app
diff --git a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-start.sh b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-start.sh
index 4bec4ff8..68e0745d 100755
--- a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-start.sh
+++ b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-start.sh
@@ -1,20 +1,35 @@
 #!/bin/bash
 
-#Start DFC app 
+set -x
 
-docker-compose up -d 
+#Start DFC app
 
-DFC_APP="$(docker ps -q --filter='name=dfc_app')"
+DOCKER_SIM_NWNAME="dfcnet"
+echo "Creating docker network $DOCKER_SIM_NWNAME, if needed"
+docker network ls| grep $DOCKER_SIM_NWNAME > /dev/null || docker network create $DOCKER_SIM_NWNAME
+
+docker-compose up -d
+
+DFC_APP="$(docker ps -q --filter='name=dfc_app0')"
 
 #Wait for initialization of docker containers for dfc app and all simulators
 for i in {1..10}; do
-if [ $(docker inspect --format '{{ .State.Running }}' $DFC_APP) ]
- then
-   echo "DFC app Running"
-   break
- else
-   echo sleep $i
-   sleep $i
- fi 
-done
+  if [ $(docker inspect --format '{{ .State.Running }}' $DFC_APP) ]
+    then
+      echo "DFC app Running"
+
+      # enable TRACE logging of DFC
+      docker exec $DFC_APP /bin/sh -c " sed -i 's/org.onap.dcaegen2.collectors.datafile: WARN/org.onap.dcaegen2.collectors.datafile: TRACE/g' /opt/app/datafile/config/application.yaml"
 
+      #enable TRACE logging of spring-framework
+      docker exec $DFC_APP /bin/sh -c " sed -i 's/org.springframework.data: ERROR/org.springframework.data: TRACE/g' /opt/app/datafile/config/application.yaml"
+
+      docker restart $DFC_APP
+      sleep 10
+
+      break
+    else
+      echo sleep $i
+      sleep $i
+  fi
+done
diff --git a/scripts/dcaegen2-collectors-datafile/dfc-management/docker-compose.yml b/scripts/dcaegen2-collectors-datafile/dfc-management/docker-compose.yml
index 28dfbbf8..9738bcde 100644
--- a/scripts/dcaegen2-collectors-datafile/dfc-management/docker-compose.yml
+++ b/scripts/dcaegen2-collectors-datafile/dfc-management/docker-compose.yml
@@ -1,9 +1,23 @@
 version: '2'
+networks:
+  dfcnet:
+    external:
+      name: dfcnet
 services:
   dfc:
-    network_mode: "host"
-    image: nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server:latest
-    container_name: dfc_app
+    #image fron Nexus
+    #image: nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server:latest
+    image: yanhuanwang/dfc:latest
+    #local image for test purposes
+    #image: org.onap.dcaegen2.collectors.datafile.datafile-app-server:latest
+    container_name: dfc_app0
     ports:
       - "8100:8100"
       - "8433:8433"
+    networks:
+      dfcnet: null
+    environment:
+      CONSUL_HOST: "consul-server"
+      CONSUL_PORT: 8500
+      CONFIG_BINDING_SERVICE: "config-binding-service"
+      HOSTNAME: "dfc_app0"
diff --git a/scripts/kill-instance.sh b/scripts/kill-instance.sh
index 525f96e4..59970986 100755
--- a/scripts/kill-instance.sh
+++ b/scripts/kill-instance.sh
@@ -18,7 +18,14 @@
 
 mkdir -p $WORKSPACE/archives
 
-docker logs $1 >> $WORKSPACE/archives/$1.log
-docker kill $1
-docker rm $1
+running_containers=$(docker ps --filter name=$1 -q)
+if [ -z "$running_containers" ]
+then
+    echo "$1 already terminated"
+else
+    echo "Stopping and removing containers"
+    docker logs $running_containers >> $WORKSPACE/archives/$1.log
+    docker stop $running_containers
+    docker rm $running_containers
+fi
 
diff --git a/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot b/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot
index 5e971da6..823a9b27 100755
--- a/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot
+++ b/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot
@@ -1,151 +1,259 @@
 *** Settings ***
-Library		OperatingSystem
-Library		RequestsLibrary
-Library		Process
+Library        OperatingSystem
+Library        RequestsLibrary
+Library        Process
 
-Resource	../../resources/common-keywords.robot
+Resource    ../../resources/common-keywords.robot
 
 *** Variables ***
-
+${CONSUL_UPL_APP}                   /usr/bin/curl -v http://127.0.0.1:8500/v1/kv/dfc_app0?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary @${SIMGROUP_ROOT}/consul/c12_feed2_PM_MEAS.json
+${CONSUL_GET_APP}                   /usr/bin/curl -v http://127.0.0.1:8500/v1/kv/dfc_app0?raw
+${CBS_GET_MERGED_CONFIG}            /usr/bin/curl -v http://127.0.0.1:10000/service_component_all/dfc_app0
 
 *** Test Cases ***
 
 ######### Single file, SFTP
 
 Verify single event with single 1MB SFTP file. From event poll to published file
-	[TAGS]							DFC_FUNCTIONAL_1
-	[Documentation]					Verify single event with single SFTP 1MB file from event poll to published file.
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
-    Set Environment Variable  		MR_TC           --tc100
-    Set Environment Variable  		DR_TC           --tc normal
-    Set Environment Variable  		DR_REDIR_TC     --tc normal
-    Set Environment Variable  		NUM_FTPFILES    1
-    Set Environment Variable  		NUM_PNFS        1
-    Set Environment Variable  		FILE_SIZE       1MB
-    Set Environment Variable  		FTP_TYPE        SFTP
-    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
-	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
-	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
-	DR Redir Sim Downloaded Volume Equal  	1 000 000													                #Verify 1 000 000 bytes published file data in DR redir sim
-	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+    [TAGS]                          DFC_FUNCTIONAL_1
+    [Documentation]                 Verify single event with single SFTP 1MB file from event poll to published file.
+    ${cli_cmd_output}=              Run Process             ${DFC_ROOT}/../dfc-containers-clean.sh           stderr=STDOUT
+    Log To Console                  Dfc containter clean: ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+    Set Environment Variable        MR_TC                   --tc100
+    Set Environment Variable        DR_TC                   --tc normal
+    Set Environment Variable        DR_REDIR_TC             --tc normal
+    Set Environment Variable        MR_GROUPS               OpenDcae-c12:PM_MEAS_FILES
+    Set Environment Variable        MR_FILE_PREFIX_MAPPING  PM_MEAS_FILES:A
+    Set Environment Variable        DR_REDIR_FEEDS          2:A
+    Set Environment Variable        FTP_FILE_PREFIXES       A
+    Set Environment Variable        NUM_FTPFILES            1
+    Set Environment Variable        NUM_PNFS                1
+    Set Environment Variable        FILE_SIZE               1MB
+    Set Environment Variable        FTP_TYPE                SFTP
+    Set Environment Variable        NUM_FTP_SERVERS         1
+    Set Environment Variable        DR_FEEDS                2:A
+    Set Environment Variable        DRR_SIM_IP              drsim_redir
+    Set Environment Variable        SFTP_SIMS               sftp-server0:22
+    Set Environment Variable        FTPS_SIMS               ftpes-server-vsftpd0:21
+
+    ${cli_cmd_output}=              Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+    Log To Console                  Simulator-start: 
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+    MR Sim Emitted Files Equal      0                                                                                   #Verify 0 file emitted from MR sim
+    DR Sim Published Files Equal    0                                                                                   #Verify 0 file published to DR sim
+
+    ${cli_cmd_output}=              Run Process                     ${CONSUL_UPL_APP}           shell=yes
+    Log To Console                  Consul APP write:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    ${cli_cmd_output}=              Run Process                     ${CONSUL_GET_APP}           shell=yes
+    Log To Console                  Consul APP read:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    ${cli_cmd_output}=              Run Process                     ${CBS_GET_MERGED_CONFIG}    shell=yes
+    Log To Console                  CBS merged configuration:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    Sleep                           10
+
+    ${cli_cmd_output}=              Run Process                    ${DFC_ROOT}/dfc-start.sh   cwd=${DFC_ROOT}
+    Log To Console                  Dfc-start:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    Wait Until Keyword Succeeds     1 minute      10 sec    MR Sim Emitted Files Equal          1                       #Verify 1 file emitted from MR sim
+    Wait Until Keyword Succeeds     1 minute      10 sec    DR Sim Query Not Published Equal    1                       #Verify 1 query response for not published files
+    Wait Until Keyword Succeeds     1 minute      10 sec    DR Sim Published Files Equal        1                       #Verify 1 file published to DR sim
+    DR Redir Sim Downloaded Volume Equal          1 000 000                                                             #Verify 1 000 000 bytes published file data in DR redir sim
+
+    ${cli_cmd_output}=              Run Process             ${SIMGROUP_ROOT}/simulators-kill.sh
+    Log                             ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+    ${cli_cmd_output}=              Run Process             ${DFC_ROOT}/dfc-kill.sh
+    Log                             ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
 
 Verify single event with single 5MB SFTP file. From event poll to published file
-	[TAGS]							DFC_FUNCTIONAL_2
-	[Documentation]					Verify single event with single SFTP 5MB file from event poll to published file.
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
-    Set Environment Variable  		MR_TC           --tc101
-    Set Environment Variable  		DR_TC           --tc normal
-    Set Environment Variable  		DR_REDIR_TC     --tc normal
-    Set Environment Variable  		NUM_FTPFILES    1
-    Set Environment Variable  		NUM_PNFS        1
-    Set Environment Variable  		FILE_SIZE       5MB
-    Set Environment Variable  		FTP_TYPE        SFTP
-    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
-	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
-	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
-	DR Redir Sim Downloaded Volume Equal  	5 000 000													                #Verify 1 000 000 bytes published file data in DR redir sim
-	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+    [TAGS]                          DFC_FUNCTIONAL_2
+    [Documentation]                 Verify single event with single SFTP 5MB file from event poll to published file.
+    ${cli_cmd_output}=              Run Process             ${DFC_ROOT}/../dfc-containers-clean.sh           stderr=STDOUT
+    Log To Console                  Dfc containter clean: ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+    Set Environment Variable        MR_TC                   --tc101
+    Set Environment Variable        DR_TC                   --tc normal
+    Set Environment Variable        DR_REDIR_TC             --tc normal
+    Set Environment Variable        MR_GROUPS               OpenDcae-c12:PM_MEAS_FILES
+    Set Environment Variable        MR_FILE_PREFIX_MAPPING  PM_MEAS_FILES:A
+    Set Environment Variable        DR_REDIR_FEEDS          2:A
+    Set Environment Variable        FTP_FILE_PREFIXES       A
+    Set Environment Variable        NUM_FTPFILES            1
+    Set Environment Variable        NUM_PNFS                1
+    Set Environment Variable        FILE_SIZE               5MB
+    Set Environment Variable        FTP_TYPE                SFTP
+    Set Environment Variable        NUM_FTP_SERVERS         1
+    Set Environment Variable        DR_FEEDS                2:A
+    Set Environment Variable        DRR_SIM_IP              drsim_redir
+    Set Environment Variable        SFTP_SIMS               sftp-server0:22
+    Set Environment Variable        FTPS_SIMS               ftpes-server-vsftpd0:21
+
+    ${cli_cmd_output}=              Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+    Log To Console                  Simulator-start: ${cli_cmd_output.stdout}
+    MR Sim Emitted Files Equal      0                                                                                   #Verify 0 file emitted from MR sim
+    DR Sim Published Files Equal    0                                                                                   #Verify 0 file published to DR sim
+
+    ${cli_cmd_output}=              Run Process             ${CONSUL_UPL_APP}           shell=yes
+    Log To Console                  Consul APP write:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    ${cli_cmd_output}=              Run Process             ${CONSUL_GET_APP}           shell=yes
+    Log To Console                  Consul APP read:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    ${cli_cmd_output}=              Run Process             ${CBS_GET_MERGED_CONFIG}    shell=yes
+    Log To Console                  CBS merged configuration:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    Sleep                           10
+
+    ${cli_cmd_output}=              Run Process                    ${DFC_ROOT}/dfc-start.sh   cwd=${DFC_ROOT}
+    Log To Console                  Dfc-start:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    Wait Until Keyword Succeeds     1 minute      10 sec    MR Sim Emitted Files Equal          1                       #Verify 1 file emitted from MR sim
+    Wait Until Keyword Succeeds     1 minute      10 sec    DR Sim Query Not Published Equal    1                       #Verify 1 query response for not published files
+    Wait Until Keyword Succeeds     1 minute      10 sec    DR Sim Published Files Equal        1                       #Verify 1 file published to DR sim
+    DR Redir Sim Downloaded Volume Equal          5 000 000                                                             #Verify 5 000 000 bytes published file data in DR redir sim
+
+    ${cli_cmd_output}=              Run Process             ${SIMGROUP_ROOT}/simulators-kill.sh
+    Log                             ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+    ${cli_cmd_output}=              Run Process             ${DFC_ROOT}/dfc-kill.sh
+    Log                             ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    ${cli_cmd_output}=              Run Process             docker  stop  $(docker ps -aq)      shell=yes
+    Log                             ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    ${cli_cmd_output}=              Run Process             docker  rm $(docker ps -aq)          shell=yes
+    Log                             ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    Sleep                           10
 
 Verify single event with single 50MB SFTP file. From event poll to published file
-	[TAGS]							DFC_FUNCTIONAL_3
-	[Documentation]					Verify single event with single SFTP 50MB file from event poll to published file.
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
-    Set Environment Variable  		MR_TC           --tc102
-    Set Environment Variable  		DR_TC           --tc normal
-    Set Environment Variable  		DR_REDIR_TC     --tc normal
-    Set Environment Variable  		NUM_FTPFILES    1
-    Set Environment Variable  		NUM_PNFS        1
-    Set Environment Variable  		FILE_SIZE       50MB
-    Set Environment Variable  		FTP_TYPE        SFTP
-    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
-	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
-	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
-	DR Redir Sim Downloaded Volume Equal  	50 000 000													                #Verify 50 000 000 bytes published file data in DR redir sim
-	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+    [TAGS]                          DFC_FUNCTIONAL_3
+    [Documentation]                 Verify single event with single SFTP 50MB file from event poll to published file.
+    ${cli_cmd_output}=              Run Process             ${DFC_ROOT}/../dfc-containers-clean.sh
+    Set Environment Variable        MR_TC                   --tc102
+    Set Environment Variable        DR_TC                   --tc normal
+    Set Environment Variable        DR_REDIR_TC             --tc normal
+    Set Environment Variable        MR_GROUPS               OpenDcae-c12:PM_MEAS_FILES
+    Set Environment Variable        MR_FILE_PREFIX_MAPPING  PM_MEAS_FILES:A
+    Set Environment Variable        DR_REDIR_FEEDS          2:A
+    Set Environment Variable        FTP_FILE_PREFIXES       A
+    Set Environment Variable        NUM_FTPFILES            1
+    Set Environment Variable        NUM_PNFS                1
+    Set Environment Variable        FILE_SIZE               50MB
+    Set Environment Variable        FTP_TYPE                SFTP
+    Set Environment Variable        NUM_FTP_SERVERS         1
+    Set Environment Variable        DR_FEEDS                2:A
+    Set Environment Variable        DRR_SIM_IP              drsim_redir
+    Set Environment Variable        SFTP_SIMS               sftp-server0:22
+    Set Environment Variable        FTPS_SIMS               ftpes-server-vsftpd0:21
+
+    ${cli_cmd_output}=              Run Process             ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+    Log To Console                  Simulator-start: ${cli_cmd_output.stdout}
+    MR Sim Emitted Files Equal      0                                                                                   #Verify 0 file emitted from MR sim
+    DR Sim Published Files Equal    0                                                                                   #Verify 0 file published to DR sim
+
+    ${cli_cmd_output}=              Run Process             ${CONSUL_UPL_APP}           shell=yes
+    Log To Console                  Consul APP write:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    ${cli_cmd_output}=              Run Process             ${CONSUL_GET_APP}           shell=yes
+    Log To Console                  Consul APP read:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    ${cli_cmd_output}=              Run Process             ${CBS_GET_MERGED_CONFIG}    shell=yes
+    Log To Console                  CBS merged configuration:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    Sleep                           10
+
+    ${cli_cmd_output}=              Run Process             ${DFC_ROOT}/dfc-start.sh   cwd=${DFC_ROOT}
+    Log To Console                  Dfc-start:
+    Log To Console                  ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+
+    Wait Until Keyword Succeeds     1 minute      10 sec    MR Sim Emitted Files Equal          1                       #Verify 1 file emitted from MR sim
+    Wait Until Keyword Succeeds     1 minute      10 sec    DR Sim Query Not Published Equal    1                       #Verify 1 query response for not published files
+    Wait Until Keyword Succeeds     1 minute      10 sec    DR Sim Published Files Equal        1                       #Verify 1 file published to DR sim
+    DR Redir Sim Downloaded Volume Equal          50 000 000                                                            #Verify 50 000 000 bytes published file data in DR redir sim
 
+    ${cli_cmd_output}=              Run Process             ${SIMGROUP_ROOT}/simulators-kill.sh
+    Log                             ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
+    ${cli_cmd_output}=              Run Process             ${DFC_ROOT}/dfc-kill.sh
+    Log                             ${cli_cmd_output.stdout} ${cli_cmd_output.stderr}
 
 ######### Single file, FTPS
 # Temporarily removed due to issues with the certificates for the ftsp server simulator cert.
 #Verify single event with single 1MB FTPS file. From event poll to published file
-#	[TAGS]							DFC_FUNCTIONAL_10
-#	[Documentation]					Verify single event with single FTPS 1MB file from event poll to published file.
-#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
-#    Set Environment Variable  		MR_TC           --tc200
-#    Set Environment Variable  		DR_TC           --tc normal
-#    Set Environment Variable  		DR_REDIR_TC     --tc normal
-#    Set Environment Variable  		NUM_FTPFILES    1
-#    Set Environment Variable  		NUM_PNFS        1
-#    Set Environment Variable  		FILE_SIZE       1MB
-#    Set Environment Variable  		FTP_TYPE        FTPS
-#    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-#	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-#	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
-#	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-#	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
-#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
-#	DR Redir Sim Downloaded Volume Equal  	1 000 000													                #Verify 1 000 000 bytes published file data in DR redir sim
-#	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+#    [TAGS]                         DFC_FUNCTIONAL_10
+#    [Documentation]                Verify single event with single FTPS 1MB file from event poll to published file.
+#    ${cli_cmd_output}=             Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
+#    Set Environment Variable       MR_TC           --tc200
+#    Set Environment Variable       DR_TC           --tc normal
+#    Set Environment Variable       DR_REDIR_TC     --tc normal
+#    Set Environment Variable       NUM_FTPFILES    1
+#    Set Environment Variable       NUM_PNFS        1
+#    Set Environment Variable       FILE_SIZE       1MB
+#    Set Environment Variable       FTP_TYPE        FTPS
+#    ${cli_cmd_output}=             Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+#    MR Sim Emitted Files Equal     0                                                                                   #Verify 0 file emitted from MR sim
+#    DR Sim Published Files Equal   0                                                                                   #Verify 0 file published to DR sim
+#    ${cli_cmd_output}=             Run Process        ${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
+#    Wait Until Keyword Succeeds    1 minute    10 sec    MR Sim Emitted Files Equal            1                       #Verify 1 file emitted from MR sim
+#    Wait Until Keyword Succeeds    1 minute    10 sec    DR Sim Query Not Published Equal      1                       #Verify 1 query response for not published files
+#    Wait Until Keyword Succeeds    1 minute    10 sec    DR Sim Published Files Equal          1                       #Verify 1 file published to DR sim
+#    DR Redir Sim Downloaded Volume Equal       1 000 000                                                               #Verify 1 000 000 bytes published file data in DR redir sim
+#    ${cli_cmd_output}=             Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
+#    ${cli_cmd_output}=             Run Process     ${DFC_ROOT}/dfc-kill.sh
 #
 #Verify single event with single 5MB FTPS file. From event poll to published file
-#	[TAGS]							DFC_FUNCTIONAL_11
-#	[Documentation]					Verify single event with single FTPS 5MB file from event poll to published file.
-#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
-#    Set Environment Variable  		MR_TC           --tc201
-#    Set Environment Variable  		DR_TC           --tc normal
-#    Set Environment Variable  		DR_REDIR_TC     --tc normal
-#    Set Environment Variable  		NUM_FTPFILES    1
-#    Set Environment Variable  		NUM_PNFS        1
-#    Set Environment Variable  		FILE_SIZE       5MB
-#    Set Environment Variable  		FTP_TYPE        FTPS
-#    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-#	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-#	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
-#	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-#	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
-#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
-#	DR Redir Sim Downloaded Volume Equal  	5 000 000													                #Verify 5 000 000 bytes published file data in DR redir sim
-#	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+#    [TAGS]                         DFC_FUNCTIONAL_11
+#    [Documentation]                Verify single event with single FTPS 5MB file from event poll to published file.
+#    ${cli_cmd_output}=             Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
+#    Set Environment Variable       MR_TC           --tc201
+#    Set Environment Variable       DR_TC           --tc normal
+#    Set Environment Variable       DR_REDIR_TC     --tc normal
+#    Set Environment Variable       NUM_FTPFILES    1
+#    Set Environment Variable       NUM_PNFS        1
+#    Set Environment Variable       FILE_SIZE       5MB
+#    Set Environment Variable       FTP_TYPE        FTPS
+#    ${cli_cmd_output}=             Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+#    MR Sim Emitted Files Equal     0                                                                                   #Verify 0 file emitted from MR sim
+#    DR Sim Published Files Equal   0                                                                                   #Verify 0 file published to DR sim
+#    ${cli_cmd_output}=             Run Process        ${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
+#    Wait Until Keyword Succeeds    1 minute    10 sec    MR Sim Emitted Files Equal            1                       #Verify 1 file emitted from MR sim
+#    Wait Until Keyword Succeeds    1 minute    10 sec    DR Sim Query Not Published Equal      1                       #Verify 1 query response for not published files
+#    Wait Until Keyword Succeeds    1 minute    10 sec    DR Sim Published Files Equal          1                       #Verify 1 file published to DR sim
+#    DR Redir Sim Downloaded Volume Equal       5 000 000                                                               #Verify 5 000 000 bytes published file data in DR redir sim
+#    ${cli_cmd_output}=             Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
+#    ${cli_cmd_output}=             Run Process     ${DFC_ROOT}/dfc-kill.sh
 #
 #Verify single event with single 50MB FTPS file. From event poll to published file
-#	[TAGS]							DFC_FUNCTIONAL_12
-#	[Documentation]					Verify single event with single FTPS 50MB file from event poll to published file.
-#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
-#    Set Environment Variable  		MR_TC           --tc202
-#    Set Environment Variable  		DR_TC           --tc normal
-#    Set Environment Variable  		DR_REDIR_TC     --tc normal
-#    Set Environment Variable  		NUM_FTPFILES    1
-#    Set Environment Variable  		NUM_PNFS        1
-#    Set Environment Variable  		FILE_SIZE       50MB
-#    Set Environment Variable  		FTP_TYPE        FTPS
-#    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-#	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-#	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
-#	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-#	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
-#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
-#	DR Redir Sim Downloaded Volume Equal  	50 000 000													                #Verify 50 000 000 bytes published file data in DR redir sim
-#	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+#    [TAGS]                         DFC_FUNCTIONAL_12
+#    [Documentation]                Verify single event with single FTPS 50MB file from event poll to published file.
+#    ${cli_cmd_output}=             Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
+#    Set Environment Variable       MR_TC           --tc202
+#    Set Environment Variable       DR_TC           --tc normal
+#    Set Environment Variable       DR_REDIR_TC     --tc normal
+#    Set Environment Variable       NUM_FTPFILES    1
+#    Set Environment Variable       NUM_PNFS        1
+#    Set Environment Variable       FILE_SIZE       50MB
+#    Set Environment Variable       FTP_TYPE        FTPS
+#    ${cli_cmd_output}=             Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+#    MR Sim Emitted Files Equal     0                                                                                   #Verify 0 file emitted from MR sim
+#    DR Sim Published Files Equal   0                                                                                   #Verify 0 file published to DR sim
+#    ${cli_cmd_output}=             Run Process        ${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
+#    Wait Until Keyword Succeeds    1 minute    10 sec    MR Sim Emitted Files Equal            1                       #Verify 1 file emitted from MR sim
+#    Wait Until Keyword Succeeds    1 minute    10 sec    DR Sim Query Not Published Equal      1                       #Verify 1 query response for not published files
+#    Wait Until Keyword Succeeds    1 minute    10 sec    DR Sim Published Files Equal          1                       #Verify 1 file published to DR sim
+#    DR Redir Sim Downloaded Volume Equal       50 000 000                                                              #Verify 50 000 000 bytes published file data in DR redir sim
+#    ${cli_cmd_output}=             Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
+#    ${cli_cmd_output}=             Run Process     ${DFC_ROOT}/dfc-kill.sh
 #
 
 *** Keywords ***