Updates to DFC CSIT test

Adaptation of Functional tests due to updated dfc simulators
Adaptation of Management tests due to changes in DFC app
Removed test for FTPS due to issue with ftps sim cert

Issue-ID: DCAEGEN2-1434

Change-Id: Ic92ac4901586cb55164177a8640ba23b60c02f0d
Signed-off-by: BjornMagnussonXA <bjorn.magnusson@est.tech>

6 files changed, 143 insertions, 51 deletions

diff --git a/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh b/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh
index e4940280..e2a0c126 100644
--- a/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh
+++ b/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh
@@ -4,10 +4,10 @@
 docker kill "$(docker ps -q -a)"
 docker rm "$(docker ps -q -a)"
 
-# Clone Simulators for DFC from integration repo. 
+# Clone Simulators for DFC from integration repo.
 mkdir -p $WORKSPACE/archives/dfc
 cd $WORKSPACE/archives/dfc
-git clone --depth 1 https://gerrit.onap.org/r/integration -b master   #How to make this version controlled for Dublin, El Alto etc etc
+git clone --depth 1 https://gerrit.onap.org/r/integration -b master
 
 #Location of all individual simulators for DFC
 SIM_ROOT=$WORKSPACE/archives/dfc/integration/test/mocks/datafilecollector-testharness
diff --git a/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh b/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh
new file mode 100755
index 00000000..03919343
--- /dev/null
+++ b/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Kill dfc and all simulator
+
+docker kill dfc_app
+docker kill dfc_dr-sim
+docker kill dfc_dr-redir-sim
+docker kill dfc_mr-sim
+docker kill dfc_sftp-server
+docker kill dfc_ftpes-server-vsftpd
+
diff --git a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh
index ebc28f49..8478bcb0 100755
--- a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh
+++ b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh
@@ -7,7 +7,7 @@ docker restart dfc_app
 DFC_APP="$(docker ps -q --filter='name=dfc_app')"
 
 
-#Wait for initialization of docker containers for dfc app and all simulators
+#Wait for initialization of docker container for dfc app
 for i in {1..10}; do
 if [ $(docker inspect --format '{{ .State.Running }}' $DFC_APP) ]
  then
diff --git a/tests/dcaegen2-collectors-datafile/resources/common-keywords.robot b/tests/dcaegen2-collectors-datafile/resources/common-keywords.robot
index a1135375..76dc0338 100644
--- a/tests/dcaegen2-collectors-datafile/resources/common-keywords.robot
+++ b/tests/dcaegen2-collectors-datafile/resources/common-keywords.robot
@@ -5,14 +5,36 @@ Library		Process
 
 *** Variables ***
 
-${CLI_MRSIM_CTR_REQUESTS}                  curl http://${SIM_IP}:2222/ctr_requests
-${CLI_MRSIM_CTR_RESPONSES}                 curl http://${SIM_IP}:2222/ctr_responses
-${CLI_MRSIM_CTR_FILES}                     curl http://${SIM_IP}:2222/ctr_unique_files
+${CLI_MRSIM_CTR_REQUESTS}                   curl --connect-timeout 10 -X GET http://${SIM_IP}:2222/ctr_requests
+${CLI_MRSIM_CTR_RESPONSES}                  curl --connect-timeout 10 -X GET http://${SIM_IP}:2222/ctr_responses
+${CLI_MRSIM_CTR_FILES}                      curl --connect-timeout 10 -X GET http://${SIM_IP}:2222/ctr_unique_files
+
+${CLI_DRSIM_CTR_QUERY_NOT_PUBLISHED}        curl --connect-timeout 10 -X GET http://${SIM_IP}:3906/ctr_publish_query_not_published
+${CLI_DRSIM_CTR_PUBLISHED_FILES}            curl --connect-timeout 10 -X GET http://${SIM_IP}:3906/ctr_published_files
+${CLI_DR_REDIR_SIM_DOWNLOADED_VOLUME}       curl --connect-timeout 10 -X GET http://${SIM_IP}:3908/dwl_volume
 
 *** Keywords ***
 
 MR Sim Emitted Files Equal
-	[Documentation]				Verify that the number of emitted unique files are equal to a target value    
+	[Documentation]				Verify that the number of emitted unique files are equal to a target value
 	[Arguments]            		${target_ctr_value}
     ${resp}=					Run Process     ${CLI_MRSIM_CTR_FILES}  shell=yes
+    Should Be Equal As Strings  ${resp.stdout}  ${target_ctr_value}
+
+DR Sim Query Not Published Equal
+	[Documentation]				Verify that the number responsed of queries of not published files are equal to a target value
+	[Arguments]            		${target_ctr_value}
+    ${resp}=					Run Process     ${CLI_DRSIM_CTR_QUERY_NOT_PUBLISHED}  shell=yes
+    Should Be Equal As Strings  ${resp.stdout}  ${target_ctr_value}
+
+DR Sim Published Files Equal
+	[Documentation]				Verify that the number published files are equal to a target value
+	[Arguments]            		${target_ctr_value}
+    ${resp}=					Run Process     ${CLI_DRSIM_CTR_PUBLISHED_FILES}  shell=yes
+    Should Be Equal As Strings  ${resp.stdout}  ${target_ctr_value}
+    
+DR Redir Sim Downloaded Volume Equal
+	[Documentation]				Verify that the size of the downloaded data volume is equal to a target value
+	[Arguments]            		${target_ctr_value}
+    ${resp}=					Run Process     ${CLI_DR_REDIR_SIM_DOWNLOADED_VOLUME}  shell=yes
     Should Be Equal As Strings  ${resp.stdout}  ${target_ctr_value}
\ No newline at end of file
diff --git a/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot b/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot
index a59fec6b..5e971da6 100755
--- a/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot
+++ b/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot
@@ -15,75 +15,138 @@ Resource	../../resources/common-keywords.robot
 Verify single event with single 1MB SFTP file. From event poll to published file
 	[TAGS]							DFC_FUNCTIONAL_1
 	[Documentation]					Verify single event with single SFTP 1MB file from event poll to published file.
+	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
     Set Environment Variable  		MR_TC           --tc100
+    Set Environment Variable  		DR_TC           --tc normal
+    Set Environment Variable  		DR_REDIR_TC     --tc normal
+    Set Environment Variable  		NUM_FTPFILES    1
+    Set Environment Variable  		NUM_PNFS        1
+    Set Environment Variable  		FILE_SIZE       1MB
+    Set Environment Variable  		FTP_TYPE        SFTP
     ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
 	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
 	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
 	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+	DR Redir Sim Downloaded Volume Equal  	1 000 000													                #Verify 1 000 000 bytes published file data in DR redir sim
 	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
 	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
 
 Verify single event with single 5MB SFTP file. From event poll to published file
 	[TAGS]							DFC_FUNCTIONAL_2
 	[Documentation]					Verify single event with single SFTP 5MB file from event poll to published file.
+	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
     Set Environment Variable  		MR_TC           --tc101
+    Set Environment Variable  		DR_TC           --tc normal
+    Set Environment Variable  		DR_REDIR_TC     --tc normal
+    Set Environment Variable  		NUM_FTPFILES    1
+    Set Environment Variable  		NUM_PNFS        1
+    Set Environment Variable  		FILE_SIZE       5MB
+    Set Environment Variable  		FTP_TYPE        SFTP
     ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
 	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
 	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
 	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+	DR Redir Sim Downloaded Volume Equal  	5 000 000													                #Verify 1 000 000 bytes published file data in DR redir sim
 	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
 	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
 
 Verify single event with single 50MB SFTP file. From event poll to published file
 	[TAGS]							DFC_FUNCTIONAL_3
 	[Documentation]					Verify single event with single SFTP 50MB file from event poll to published file.
+	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
     Set Environment Variable  		MR_TC           --tc102
+    Set Environment Variable  		DR_TC           --tc normal
+    Set Environment Variable  		DR_REDIR_TC     --tc normal
+    Set Environment Variable  		NUM_FTPFILES    1
+    Set Environment Variable  		NUM_PNFS        1
+    Set Environment Variable  		FILE_SIZE       50MB
+    Set Environment Variable  		FTP_TYPE        SFTP
     ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
 	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
 	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
 	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+	DR Redir Sim Downloaded Volume Equal  	50 000 000													                #Verify 50 000 000 bytes published file data in DR redir sim
 	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
 	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
 
 
 ######### Single file, FTPS
-
-Verify single event with single 1MB FTPS file. From event poll to published file
-	[TAGS]							DFC_FUNCTIONAL_10
-	[Documentation]					Verify single event with single FTPS 1MB file from event poll to published file.
-    Set Environment Variable  		MR_TC           --tc200
-    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
-
-Verify single event with single 5MB FTPS file. From event poll to published file
-	[TAGS]							DFC_FUNCTIONAL_11
-	[Documentation]					Verify single event with single FTPS 5MB file from event poll to published file.
-    Set Environment Variable  		MR_TC           --tc201
-    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
-
-Verify single event with single 50MB FTPS file. From event poll to published file
-	[TAGS]							DFC_FUNCTIONAL_12
-	[Documentation]					Verify single event with single FTPS 50MB file from event poll to published file.
-    Set Environment Variable  		MR_TC           --tc202
-    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
-
+# Temporarily removed due to issues with the certificates for the ftsp server simulator cert.
+#Verify single event with single 1MB FTPS file. From event poll to published file
+#	[TAGS]							DFC_FUNCTIONAL_10
+#	[Documentation]					Verify single event with single FTPS 1MB file from event poll to published file.
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
+#    Set Environment Variable  		MR_TC           --tc200
+#    Set Environment Variable  		DR_TC           --tc normal
+#    Set Environment Variable  		DR_REDIR_TC     --tc normal
+#    Set Environment Variable  		NUM_FTPFILES    1
+#    Set Environment Variable  		NUM_PNFS        1
+#    Set Environment Variable  		FILE_SIZE       1MB
+#    Set Environment Variable  		FTP_TYPE        FTPS
+#    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+#	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+#	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
+#	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
+#	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+#	DR Redir Sim Downloaded Volume Equal  	1 000 000													                #Verify 1 000 000 bytes published file data in DR redir sim
+#	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+#
+#Verify single event with single 5MB FTPS file. From event poll to published file
+#	[TAGS]							DFC_FUNCTIONAL_11
+#	[Documentation]					Verify single event with single FTPS 5MB file from event poll to published file.
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
+#    Set Environment Variable  		MR_TC           --tc201
+#    Set Environment Variable  		DR_TC           --tc normal
+#    Set Environment Variable  		DR_REDIR_TC     --tc normal
+#    Set Environment Variable  		NUM_FTPFILES    1
+#    Set Environment Variable  		NUM_PNFS        1
+#    Set Environment Variable  		FILE_SIZE       5MB
+#    Set Environment Variable  		FTP_TYPE        FTPS
+#    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+#	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+#	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
+#	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
+#	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+#	DR Redir Sim Downloaded Volume Equal  	5 000 000													                #Verify 5 000 000 bytes published file data in DR redir sim
+#	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+#
+#Verify single event with single 50MB FTPS file. From event poll to published file
+#	[TAGS]							DFC_FUNCTIONAL_12
+#	[Documentation]					Verify single event with single FTPS 50MB file from event poll to published file.
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
+#    Set Environment Variable  		MR_TC           --tc202
+#    Set Environment Variable  		DR_TC           --tc normal
+#    Set Environment Variable  		DR_REDIR_TC     --tc normal
+#    Set Environment Variable  		NUM_FTPFILES    1
+#    Set Environment Variable  		NUM_PNFS        1
+#    Set Environment Variable  		FILE_SIZE       50MB
+#    Set Environment Variable  		FTP_TYPE        FTPS
+#    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+#	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+#	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
+#	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
+#	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+#	DR Redir Sim Downloaded Volume Equal  	50 000 000													                #Verify 50 000 000 bytes published file data in DR redir sim
+#	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+#
 
 *** Keywords ***
 
-
-    
-    
diff --git a/tests/dcaegen2-collectors-datafile/testsuites/ManagementInterface-suite/MI.robot b/tests/dcaegen2-collectors-datafile/testsuites/ManagementInterface-suite/MI.robot
index 875228bd..9d9f26d6 100755
--- a/tests/dcaegen2-collectors-datafile/testsuites/ManagementInterface-suite/MI.robot
+++ b/tests/dcaegen2-collectors-datafile/testsuites/ManagementInterface-suite/MI.robot
@@ -1,14 +1,13 @@
 *** Settings ***
 Library       OperatingSystem
 Library       RequestsLibrary
-Library  Process
+Library       Process
 
 *** Variables ***
 
 
 *** Test Cases ***
 
-
 Heartbeat test
     [Documentation]    Check DFC heartbeat
     Heartbeat    	   I'm living
@@ -29,27 +28,24 @@ Stop test - secure
     [Documentation]    Check DFC stop, secure
     Stop-secure   	   Datafile Service has already been stopped!
 
-
 Start test - secure
     [Documentation]    Check DFC start, secure
     Start-secure       Datafile Service has been started!
 
 
-#PRobably move definitions of common Keywords to a common file
-
 *** Keywords ***
 #Probably simplyfy the test cases by using variables for port numbers/urls etc
 Heartbeat
     [Arguments]                  ${respbody}
     Create Session               session              http://localhost:8100/heartbeat
     ${resp}=                     Get Request          session                  /
-    Should Be Equal				 ${resp.text}	      ${respbody}
+    Should Match Regexp			 ${resp.text}	      ${respbody}*
 
 Heartbeat-secure
     [Arguments]                  ${respbody}
     Create Session               session              https://localhost:8433/heartbeat
     ${resp}=                     Get Request          session                  /
-    Should Be Equal				 ${resp.text}	      ${respbody}
+    Should Match Regexp			 ${resp.text}	      ${respbody}*
 
 Stop
     [Arguments]                  ${respbody}