Remove race condition

- moved containers logs to separate folder in archives to allow
easier debugging. Also saved xNF simulators logs there as they are
not specified in docker-compose file
- renamed message-routing test file as it's name was accidentaly set
during one of rebases
- fixed bug where variables passed from Robot testcase were evaluated
as strings instead of booleans

Change-Id: If21c3983df0869bbc7b8378f3bf5344cad05350b
Issue-ID: DCAEGEN2-664
Signed-off-by: Filip Krzywka <filip.krzywka@nokia.com>

3 files changed, 52 insertions, 75 deletions

diff --git a/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/libraries/XnfSimulatorLibrary.py b/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/libraries/XnfSimulatorLibrary.py
index d85eb4dee..b2466d7ca 100644
--- a/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/libraries/XnfSimulatorLibrary.py
+++ b/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/libraries/XnfSimulatorLibrary.py
@@ -8,9 +8,12 @@ from time import sleep
 XNF_SIMULATOR_NAME = "xNF Simulator"
 SIMULATOR_IMAGE_NAME = "onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-xnf-simulator"
 SIMULATOR_IMAGE_FULL_NAME = os.getenv("DOCKER_REGISTRY") + "/" + SIMULATOR_IMAGE_NAME + ":latest"
-certificates_dir_path = os.getenv("WORKSPACE") + "/test/csit/plans/dcaegen2-collectors-hv-ves/testsuites/ssl/"
+WORKSPACE_ENV = os.getenv("WORKSPACE")
+certificates_dir_path = WORKSPACE_ENV + "/test/csit/plans/dcaegen2-collectors-hv-ves/testsuites/ssl/"
+collector_certs_lookup_dir = "/etc/ves-hv/"
 ONE_SECOND_IN_NANOS = 10 ** 9
 
+
 class XnfSimulatorLibrary:
 
     def start_xnf_simulators(self, list_of_ports, valid_certs=True):
@@ -36,48 +39,43 @@ class XnfSimulatorLibrary:
         simulators_addresses = []
         for port in list_of_ports:
             container = self.run_simulator(dockerClient, port,
-                                           "/etc/ves-hv/" + cert_name_prefix + "client.crt",
-                                           "/etc/ves-hv/" + cert_name_prefix + "client.key",
-                                           "/etc/ves-hv/" + cert_name_prefix + "trust.crt"
+                                           collector_certs_lookup_dir + cert_name_prefix + "client.crt",
+                                           collector_certs_lookup_dir + cert_name_prefix + "client.key",
+                                           collector_certs_lookup_dir + cert_name_prefix + "trust.crt"
                                            )
 
-            self.copy_required_certificates_into_simulator(container)
             logger.info("Started container: " + container.name + "  " + container.id)
             simulators_addresses.append(container.name + ":" + port)
         return simulators_addresses
 
     def run_simulator(self, dockerClient, port, client_crt_path, client_key_path, client_trust_store):
+        xNF_startup_command = ["--listen-port", port,
+                               "--ves-host", "ves-hv-collector",
+                               "--ves-port", "6061",
+                               "--cert-file", client_crt_path,
+                               "--private-key-file", client_key_path,
+                               "--trust-cert-file", client_trust_store]
+        xNF_healthcheck_command = {
+            "interval": 5 * ONE_SECOND_IN_NANOS,
+            "timeout": 3 * ONE_SECOND_IN_NANOS,
+            "retries": 1,
+            "test": ["CMD", "curl", "--request", "GET",
+                     "--fail", "--silent", "--show-error",
+                     "localhost:" + port + "/healthcheck"]
+        }
+        logger.info("Startup command: " + str(xNF_startup_command))
+        logger.info("Healthcheck command: " + str(xNF_healthcheck_command))
         return dockerClient.containers.run(SIMULATOR_IMAGE_FULL_NAME,
-                                           command=["--listen-port", port,
-                                                    "--ves-host", "ves-hv-collector",
-                                                    "--ves-port", "6061",
-                                                    "--cert-file", client_crt_path,
-                                                    "--private-key-file", client_key_path,
-                                                    "--trust-cert-file", client_trust_store
-                                                    ],
-                                           healthcheck={
-                                               "interval": 5 * ONE_SECOND_IN_NANOS,
-                                               "timeout": 3 * ONE_SECOND_IN_NANOS,
-                                               "retries": 1,
-                                               "test": ["CMD", "curl", "--request", "GET",
-                                                        "--fail", "--silent", "--show-error",
-                                                        "localhost:" + port + "/healthcheck"]
-                                           },
+                                           command=xNF_startup_command,
+                                           healthcheck=xNF_healthcheck_command,
                                            detach=True,
                                            network="ves-hv-default",
                                            ports={port + "/tcp": port},
+                                           volumes=self.container_volumes(),
                                            name="ves-hv-collector-xnf-simulator" + port)
 
-    def copy_required_certificates_into_simulator(self, container):
-        container.exec_run("mkdir -p /etc/ves-hv")
-        copy_to_container(container.id, [
-            certificates_dir_path + "client.crt",
-            certificates_dir_path + "client.key",
-            certificates_dir_path + "trust.crt",
-            certificates_dir_path + "invalid_client.crt",
-            certificates_dir_path + "invalid_client.key",
-            certificates_dir_path + "invalid_trust.crt",
-        ])
+    def container_volumes(self):
+        return {certificates_dir_path: {"bind": collector_certs_lookup_dir, "mode": 'rw'}}
 
     def assert_containers_startup_was_successful(self, dockerClient):
         checks_amount = 6
@@ -95,11 +93,15 @@ class XnfSimulatorLibrary:
         container_health = container.attrs['State']['Health']['Status']
         return container_health == 'healthy' and container.status == 'running'
 
-    def stop_and_remove_all_xnf_simulators(self):
+    def stop_and_remove_all_xnf_simulators(self, suite_name):
         dockerClient = docker.from_env()
         for container in self.get_simulators_list(dockerClient):
             logger.info("Stopping and removing container: " + container.id)
-            logger.debug(container.logs())
+            log_filename = WORKSPACE_ENV + "/archives/containers_logs/" + \
+                           suite_name.split(".")[-1] + "_" + container.name + ".log"
+            file = open(log_filename, "w+")
+            file.write(container.logs())
+            file.close()
             container.stop()
             container.remove()
         dockerClient.close()
diff --git a/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/hv-ves.robot b/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/message-routing.robot
index 482b698fe..6153afa0a 100644
--- a/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/hv-ves.robot
+++ b/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/message-routing.robot
@@ -1,19 +1,28 @@
 *** Settings ***
-Library    DcaeAppSimulatorLibrary
+Library       DcaeAppSimulatorLibrary
 Library       XnfSimulatorLibrary
 Library       VesHvContainersUtilsLibrary
 Library       Collections
 
+Resource      resources/common-keywords.robot
+
 Suite Setup       Message Routing Suite Setup
 Suite Teardown    VES-HV Collector Suite Teardown
 Test Teardown     VES-HV Collector Test Shutdown
 
+*** Keywords ***
+Message Routing Suite Setup
+    Log   Started Suite: VES-HV Message Routing
+    ${XNF_PORTS_LIST}=    Create List    7000
+    Configure Valid xNF Simulators On Ports    ${XNF_PORTS_LIST}
+    Log   Suite setup finished
+
 *** Test Cases ***
 Correct Messages Routing
     [Documentation]   VES-HV Collector should route all valid messages to topics specified in configuration
     ...               and do not change message payload generated in XNF simulator
 
-    ${SIMULATORS_LIST}=   Get xNF Simulators   1
+    ${SIMULATORS_LIST}=   Get Valid xNF Simulators   1
     Send Messages From xNF Simulators   ${SIMULATORS_LIST}   ${XNF_FIXED_PAYLOAD_REQUEST}
 
     Wait until keyword succeeds   60 sec   5 sec
@@ -24,7 +33,7 @@ Correct Messages Routing
 Too big payload message handling
     [Documentation]   VES-HV Collector should interrupt the stream when encountered message with too big payload
 
-    ${SIMULATORS_LIST}=   Get xNF Simulators   1
+    ${SIMULATORS_LIST}=   Get Valid xNF Simulators   1
     Send Messages From xNF Simulators   ${SIMULATORS_LIST}   ${XNF_TOO_BIG_PAYLOAD_REQUEST}
 
     Wait until keyword succeeds   60 sec   5 sec
@@ -34,7 +43,7 @@ Too big payload message handling
 Invalid wire frame message handling
     [Documentation]  VES-HV Collector should skip messages with invalid wire frame
 
-    ${SIMULATORS_LIST}=   Get xNF Simulators   1
+    ${SIMULATORS_LIST}=   Get Valid xNF Simulators   1
     Send Messages From xNF Simulators   ${SIMULATORS_LIST}   ${XNF_INVALID_WIRE_FRAME_REQUEST}
 
     Wait until keyword succeeds   60 sec   5 sec
@@ -45,7 +54,7 @@ Invalid wire frame message handling
 Invalid GPB data message handling
     [Documentation]   VES-HV Collector should skip messages with invalid GPB data
 
-    ${SIMULATORS_LIST}=   Get xNF Simulators   1
+    ${SIMULATORS_LIST}=   Get Valid xNF Simulators   1
     Send Messages From xNF Simulators   ${SIMULATORS_LIST}   ${XNF_INVALID_GPB_DATA_REQUEST}
 
     Wait until keyword succeeds   60 sec   5 sec
@@ -56,47 +65,13 @@ Invalid GPB data message handling
 Unsupported domain message handling
     [Documentation]   VES-HV Collector should skip messages with unsupported domain
 
-    ${SIMULATORS_LIST}=   Get xNF Simulators   1
+    ${SIMULATORS_LIST}=   Get Valid xNF Simulators   1
     Send Messages From xNF Simulators   ${SIMULATORS_LIST}   ${XNF_UNSUPPORTED_DOMAIN_REQUEST}
 
     Wait until keyword succeeds   60 sec   5 sec
     ...    Assert Dcae App Consumed  ${DCAE_APP_API_MESSAGES_COUNT_URL}   ${AMOUNT_50000}
     Assert Dcae App Consumed Proper Messages   ${DCAE_APP_API_MESSAGES_VALIDATION_URL}   ${DCAE_UNSUPPORTED_DOMAIN_REQUEST}
 
-*** Keywords ***
-Message Routing Suite Setup
-    Log   Started Suite: VES-HV Message Routing
-    ${XNF_PORTS_LIST}=    Create List    7000
-    Configure xNF Simulators On Ports    ${XNF_PORTS_LIST}
-    Log   Suite setup finished
-
-Configure xNF Simulators On Ports
-    [Arguments]    ${XNF_PORTS_LIST}
-    ${XNF_SIMULATORS_ADDRESSES}=   Start Xnf Simulators    ${XNF_PORTS_LIST}    True
-    Set Suite Variable    ${XNF_SIMULATORS_ADDRESSES}
-
-
-Get xNF Simulators
-    [Arguments]  ${AMOUNT}
-    ${SIMULATORS}=   Get Slice From List   ${XNF_SIMULATORS_ADDRESSES}   0   ${AMOUNT}
-    [Return]   ${SIMULATORS}
-
-
-Send Messages From xNF Simulators
-    [Arguments]    ${XNF_HOSTS_LIST}   ${MESSAGE_FILEPATH}
-    :FOR   ${HOST}   IN    @{XNF_HOSTS_LIST}
-    \    ${XNF_SIM_API_ACCESS}=   Get xNF Sim Api Access Url   ${HTTP_METHOD_URL}   ${HOST}
-    \    ${XNF_SIM_API_URL}=  Catenate   SEPARATOR=   ${XNF_SIM_API_ACCESS}   ${XNF_SIM_API_PATH}
-    \    Send messages   ${XNF_SIM_API_URL}   ${MESSAGE_FILEPATH}
-
-
-VES-HV Collector Test Shutdown
-    Reset DCAE App Simulator  ${DCAE_APP_API_MESSAGE_RESET_URL}
-
-
-VES-HV Collector Suite Teardown
-    Stop And Remove All Xnf Simulators
-
 *** Variables ***
 ${HTTP_METHOD_URL}                             http://
 
diff --git a/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/resources/common-keywords.robot b/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/resources/common-keywords.robot
index 345118657..bc03de232 100644
--- a/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/resources/common-keywords.robot
+++ b/test/csit/tests/dcaegen2-collectors-hv-ves/testcases/resources/common-keywords.robot
@@ -6,13 +6,13 @@ Library       Collections
 *** Keywords ***
 Configure Valid xNF Simulators On Ports
     [Arguments]    ${XNF_PORTS_LIST}
-    ${VALID_XNF_SIMULATORS_ADDRESSES}=   Start Xnf Simulators    ${XNF_PORTS_LIST}    True
+    ${VALID_XNF_SIMULATORS_ADDRESSES}=   Start Xnf Simulators    ${XNF_PORTS_LIST}    ${true}
     Set Suite Variable    ${VALID_XNF_SIMULATORS_ADDRESSES}
 
 
 Configure Invalid xNF Simulators On Ports
     [Arguments]    ${XNF_PORTS_LIST}
-    ${INVALID_XNF_SIMULATORS_ADDRESSES}=   Start Xnf Simulators    ${XNF_PORTS_LIST}    False
+    ${INVALID_XNF_SIMULATORS_ADDRESSES}=   Start Xnf Simulators    ${XNF_PORTS_LIST}    ${false}
     Set Suite Variable    ${INVALID_XNF_SIMULATORS_ADDRESSES}
 
 
@@ -41,7 +41,7 @@ VES-HV Collector Test Shutdown
 
 
 VES-HV Collector Suite Teardown
-    Stop And Remove All Xnf Simulators
+    Stop And Remove All Xnf Simulators   ${SUITE NAME}
 
 *** Variables ***
 ${HTTP_METHOD_URL}                             http://