Merge "Updates to DFC CSIT test"

6 files changed, 143 insertions, 51 deletions

diff --git a/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh b/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh
index e4940280..e2a0c126 100644
--- a/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh
+++ b/plans/dcaegen2-collectors-datafile/Functional-suite/setup.sh
@@ -4,10 +4,10 @@
 docker kill "$(docker ps -q -a)"
 docker rm "$(docker ps -q -a)"
 
-# Clone Simulators for DFC from integration repo. 
+# Clone Simulators for DFC from integration repo.
 mkdir -p $WORKSPACE/archives/dfc
 cd $WORKSPACE/archives/dfc
-git clone --depth 1 https://gerrit.onap.org/r/integration -b master   #How to make this version controlled for Dublin, El Alto etc etc
+git clone --depth 1 https://gerrit.onap.org/r/integration -b master
 
 #Location of all individual simulators for DFC
 SIM_ROOT=$WORKSPACE/archives/dfc/integration/test/mocks/datafilecollector-testharness
diff --git a/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh b/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh
new file mode 100755
index 00000000..03919343
--- /dev/null
+++ b/scripts/dcaegen2-collectors-datafile/dfc-containers-clean.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Kill dfc and all simulator
+
+docker kill dfc_app
+docker kill dfc_dr-sim
+docker kill dfc_dr-redir-sim
+docker kill dfc_mr-sim
+docker kill dfc_sftp-server
+docker kill dfc_ftpes-server-vsftpd
+
diff --git a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh
index ebc28f49..8478bcb0 100755
--- a/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh
+++ b/scripts/dcaegen2-collectors-datafile/dfc-management/dfc-restart.sh
@@ -7,7 +7,7 @@ docker restart dfc_app
 DFC_APP="$(docker ps -q --filter='name=dfc_app')"
 
 
-#Wait for initialization of docker containers for dfc app and all simulators
+#Wait for initialization of docker container for dfc app
 for i in {1..10}; do
 if [ $(docker inspect --format '{{ .State.Running }}' $DFC_APP) ]
  then
diff --git a/tests/dcaegen2-collectors-datafile/resources/common-keywords.robot b/tests/dcaegen2-collectors-datafile/resources/common-keywords.robot
index a1135375..76dc0338 100644
--- a/tests/dcaegen2-collectors-datafile/resources/common-keywords.robot
+++ b/tests/dcaegen2-collectors-datafile/resources/common-keywords.robot
@@ -5,14 +5,36 @@ Library		Process
 
 *** Variables ***
 
-${CLI_MRSIM_CTR_REQUESTS}                  curl http://${SIM_IP}:2222/ctr_requests
-${CLI_MRSIM_CTR_RESPONSES}                 curl http://${SIM_IP}:2222/ctr_responses
-${CLI_MRSIM_CTR_FILES}                     curl http://${SIM_IP}:2222/ctr_unique_files
+${CLI_MRSIM_CTR_REQUESTS}                   curl --connect-timeout 10 -X GET http://${SIM_IP}:2222/ctr_requests
+${CLI_MRSIM_CTR_RESPONSES}                  curl --connect-timeout 10 -X GET http://${SIM_IP}:2222/ctr_responses
+${CLI_MRSIM_CTR_FILES}                      curl --connect-timeout 10 -X GET http://${SIM_IP}:2222/ctr_unique_files
+
+${CLI_DRSIM_CTR_QUERY_NOT_PUBLISHED}        curl --connect-timeout 10 -X GET http://${SIM_IP}:3906/ctr_publish_query_not_published
+${CLI_DRSIM_CTR_PUBLISHED_FILES}            curl --connect-timeout 10 -X GET http://${SIM_IP}:3906/ctr_published_files
+${CLI_DR_REDIR_SIM_DOWNLOADED_VOLUME}       curl --connect-timeout 10 -X GET http://${SIM_IP}:3908/dwl_volume
 
 *** Keywords ***
 
 MR Sim Emitted Files Equal
-	[Documentation]				Verify that the number of emitted unique files are equal to a target value    
+	[Documentation]				Verify that the number of emitted unique files are equal to a target value
 	[Arguments]            		${target_ctr_value}
     ${resp}=					Run Process     ${CLI_MRSIM_CTR_FILES}  shell=yes
+    Should Be Equal As Strings  ${resp.stdout}  ${target_ctr_value}
+
+DR Sim Query Not Published Equal
+	[Documentation]				Verify that the number responsed of queries of not published files are equal to a target value
+	[Arguments]            		${target_ctr_value}
+    ${resp}=					Run Process     ${CLI_DRSIM_CTR_QUERY_NOT_PUBLISHED}  shell=yes
+    Should Be Equal As Strings  ${resp.stdout}  ${target_ctr_value}
+
+DR Sim Published Files Equal
+	[Documentation]				Verify that the number published files are equal to a target value
+	[Arguments]            		${target_ctr_value}
+    ${resp}=					Run Process     ${CLI_DRSIM_CTR_PUBLISHED_FILES}  shell=yes
+    Should Be Equal As Strings  ${resp.stdout}  ${target_ctr_value}
+    
+DR Redir Sim Downloaded Volume Equal
+	[Documentation]				Verify that the size of the downloaded data volume is equal to a target value
+	[Arguments]            		${target_ctr_value}
+    ${resp}=					Run Process     ${CLI_DR_REDIR_SIM_DOWNLOADED_VOLUME}  shell=yes
     Should Be Equal As Strings  ${resp.stdout}  ${target_ctr_value}
\ No newline at end of file
diff --git a/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot b/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot
index a59fec6b..5e971da6 100755
--- a/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot
+++ b/tests/dcaegen2-collectors-datafile/testsuites/Functional-Single-File-suite/FuncSingleFile.robot
@@ -15,75 +15,138 @@ Resource	../../resources/common-keywords.robot
 Verify single event with single 1MB SFTP file. From event poll to published file
 	[TAGS]							DFC_FUNCTIONAL_1
 	[Documentation]					Verify single event with single SFTP 1MB file from event poll to published file.
+	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
     Set Environment Variable  		MR_TC           --tc100
+    Set Environment Variable  		DR_TC           --tc normal
+    Set Environment Variable  		DR_REDIR_TC     --tc normal
+    Set Environment Variable  		NUM_FTPFILES    1
+    Set Environment Variable  		NUM_PNFS        1
+    Set Environment Variable  		FILE_SIZE       1MB
+    Set Environment Variable  		FTP_TYPE        SFTP
     ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
 	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
 	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
 	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+	DR Redir Sim Downloaded Volume Equal  	1 000 000													                #Verify 1 000 000 bytes published file data in DR redir sim
 	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
 	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
 
 Verify single event with single 5MB SFTP file. From event poll to published file
 	[TAGS]							DFC_FUNCTIONAL_2
 	[Documentation]					Verify single event with single SFTP 5MB file from event poll to published file.
+	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
     Set Environment Variable  		MR_TC           --tc101
+    Set Environment Variable  		DR_TC           --tc normal
+    Set Environment Variable  		DR_REDIR_TC     --tc normal
+    Set Environment Variable  		NUM_FTPFILES    1
+    Set Environment Variable  		NUM_PNFS        1
+    Set Environment Variable  		FILE_SIZE       5MB
+    Set Environment Variable  		FTP_TYPE        SFTP
     ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
 	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
 	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
 	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+	DR Redir Sim Downloaded Volume Equal  	5 000 000													                #Verify 1 000 000 bytes published file data in DR redir sim
 	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
 	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
 
 Verify single event with single 50MB SFTP file. From event poll to published file
 	[TAGS]							DFC_FUNCTIONAL_3
 	[Documentation]					Verify single event with single SFTP 50MB file from event poll to published file.
+	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
     Set Environment Variable  		MR_TC           --tc102
+    Set Environment Variable  		DR_TC           --tc normal
+    Set Environment Variable  		DR_REDIR_TC     --tc normal
+    Set Environment Variable  		NUM_FTPFILES    1
+    Set Environment Variable  		NUM_PNFS        1
+    Set Environment Variable  		FILE_SIZE       50MB
+    Set Environment Variable  		FTP_TYPE        SFTP
     ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
 	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
 	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
 	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+	DR Redir Sim Downloaded Volume Equal  	50 000 000													                #Verify 50 000 000 bytes published file data in DR redir sim
 	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
 	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
 
 
 ######### Single file, FTPS
-
-Verify single event with single 1MB FTPS file. From event poll to published file
-	[TAGS]							DFC_FUNCTIONAL_10
-	[Documentation]					Verify single event with single FTPS 1MB file from event poll to published file.
-    Set Environment Variable  		MR_TC           --tc200
-    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
-
-Verify single event with single 5MB FTPS file. From event poll to published file
-	[TAGS]							DFC_FUNCTIONAL_11
-	[Documentation]					Verify single event with single FTPS 5MB file from event poll to published file.
-    Set Environment Variable  		MR_TC           --tc201
-    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
-
-Verify single event with single 50MB FTPS file. From event poll to published file
-	[TAGS]							DFC_FUNCTIONAL_12
-	[Documentation]					Verify single event with single FTPS 50MB file from event poll to published file.
-    Set Environment Variable  		MR_TC           --tc202
-    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
-	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
-	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
-	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
-	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
-	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
-
+# Temporarily removed due to issues with the certificates for the ftsp server simulator cert.
+#Verify single event with single 1MB FTPS file. From event poll to published file
+#	[TAGS]							DFC_FUNCTIONAL_10
+#	[Documentation]					Verify single event with single FTPS 1MB file from event poll to published file.
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
+#    Set Environment Variable  		MR_TC           --tc200
+#    Set Environment Variable  		DR_TC           --tc normal
+#    Set Environment Variable  		DR_REDIR_TC     --tc normal
+#    Set Environment Variable  		NUM_FTPFILES    1
+#    Set Environment Variable  		NUM_PNFS        1
+#    Set Environment Variable  		FILE_SIZE       1MB
+#    Set Environment Variable  		FTP_TYPE        FTPS
+#    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+#	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+#	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
+#	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
+#	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+#	DR Redir Sim Downloaded Volume Equal  	1 000 000													                #Verify 1 000 000 bytes published file data in DR redir sim
+#	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+#
+#Verify single event with single 5MB FTPS file. From event poll to published file
+#	[TAGS]							DFC_FUNCTIONAL_11
+#	[Documentation]					Verify single event with single FTPS 5MB file from event poll to published file.
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
+#    Set Environment Variable  		MR_TC           --tc201
+#    Set Environment Variable  		DR_TC           --tc normal
+#    Set Environment Variable  		DR_REDIR_TC     --tc normal
+#    Set Environment Variable  		NUM_FTPFILES    1
+#    Set Environment Variable  		NUM_PNFS        1
+#    Set Environment Variable  		FILE_SIZE       5MB
+#    Set Environment Variable  		FTP_TYPE        FTPS
+#    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+#	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+#	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
+#	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
+#	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+#	DR Redir Sim Downloaded Volume Equal  	5 000 000													                #Verify 5 000 000 bytes published file data in DR redir sim
+#	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+#
+#Verify single event with single 50MB FTPS file. From event poll to published file
+#	[TAGS]							DFC_FUNCTIONAL_12
+#	[Documentation]					Verify single event with single FTPS 50MB file from event poll to published file.
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/../dfc-containers-clean.sh
+#    Set Environment Variable  		MR_TC           --tc202
+#    Set Environment Variable  		DR_TC           --tc normal
+#    Set Environment Variable  		DR_REDIR_TC     --tc normal
+#    Set Environment Variable  		NUM_FTPFILES    1
+#    Set Environment Variable  		NUM_PNFS        1
+#    Set Environment Variable  		FILE_SIZE       50MB
+#    Set Environment Variable  		FTP_TYPE        FTPS
+#    ${cli_cmd_output}=          	Run Process     ./simulators-start.sh    cwd=${SIMGROUP_ROOT}
+#	MR Sim Emitted Files Equal    	0													                                #Verify 0 file emitted from MR sim
+#	DR Sim Published Files Equal  	0													                                #Verify 0 file published to DR sim
+#	${cli_cmd_output}=          	Run Process		${DFC_ROOT}/dfc-start.sh  cwd=${DFC_ROOT}
+#	Wait Until Keyword Succeeds	1 minute	10 sec	MR Sim Emitted Files Equal    	1	                                #Verify 1 file emitted from MR sim
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Query Not Published Equal    	1	                        #Verify 1 query response for not published files
+#	Wait Until Keyword Succeeds	1 minute	10 sec	DR Sim Published Files Equal    	1	                            #Verify 1 file published to DR sim
+#	DR Redir Sim Downloaded Volume Equal  	50 000 000													                #Verify 50 000 000 bytes published file data in DR redir sim
+#	${cli_cmd_output}=				Run Process     ${SIMGROUP_ROOT}/simulators-kill.sh
+#	${cli_cmd_output}=				Run Process     ${DFC_ROOT}/dfc-kill.sh
+#
 
 *** Keywords ***
 
-
-    
-    
diff --git a/tests/dcaegen2-collectors-datafile/testsuites/ManagementInterface-suite/MI.robot b/tests/dcaegen2-collectors-datafile/testsuites/ManagementInterface-suite/MI.robot
index 875228bd..9d9f26d6 100755
--- a/tests/dcaegen2-collectors-datafile/testsuites/ManagementInterface-suite/MI.robot
+++ b/tests/dcaegen2-collectors-datafile/testsuites/ManagementInterface-suite/MI.robot
@@ -1,14 +1,13 @@
 *** Settings ***
 Library       OperatingSystem
 Library       RequestsLibrary
-Library  Process
+Library       Process
 
 *** Variables ***
 
 
 *** Test Cases ***
 
-
 Heartbeat test
     [Documentation]    Check DFC heartbeat
     Heartbeat    	   I'm living
@@ -29,27 +28,24 @@ Stop test - secure
     [Documentation]    Check DFC stop, secure
     Stop-secure   	   Datafile Service has already been stopped!
 
-
 Start test - secure
     [Documentation]    Check DFC start, secure
     Start-secure       Datafile Service has been started!
 
 
-#PRobably move definitions of common Keywords to a common file
-
 *** Keywords ***
 #Probably simplyfy the test cases by using variables for port numbers/urls etc
 Heartbeat
     [Arguments]                  ${respbody}
     Create Session               session              http://localhost:8100/heartbeat
     ${resp}=                     Get Request          session                  /
-    Should Be Equal				 ${resp.text}	      ${respbody}
+    Should Match Regexp			 ${resp.text}	      ${respbody}*
 
 Heartbeat-secure
     [Arguments]                  ${respbody}
     Create Session               session              https://localhost:8433/heartbeat
     ${resp}=                     Get Request          session                  /
-    Should Be Equal				 ${resp.text}	      ${respbody}
+    Should Match Regexp			 ${resp.text}	      ${respbody}*
 
 Stop
     [Arguments]                  ${respbody}