diff options
| author |   Krzysztof Gajewski <krzysztof.gajewski@nokia.com> | 2020-12-29 13:58:35 +0100 |
|---|---|---|
| committer |   Morgan Richomme <morgan.richomme@orange.com> | 2021-01-13 14:43:19 +0000 |
| commit | b837e334c875d2c2392fe51d711515a97aeb6c78 (patch) | |
| tree | 85f6fbd45a8f9cc8f7c8c0a94757f4505b289eaf | |
| parent | a00402a48e87b132ca8cf90e474970c3aebdb125 (diff) | |
Add HTTP to DFC tests
- testing environment preparation for DFC with HTTP client
- add suitable TCs for CSIT tests
Issue-ID: DCAEGEN2-2527
Signed-off-by: Krzysztof Gajewski <krzysztof.gajewski@nokia.com>
Change-Id: Ic419c38125201ed6578a308ba258709f20ae65bd
23 files changed, 1334 insertions, 102 deletions
diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC7.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC7.sh new file mode 100755 index 000000000..0da52f102 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC7.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +TC_ONELINE_DESCR="One 1MB file from one PNF in one event using HTTP, from poll to publish" + +. ../common/testcase_common.sh $1 $2 + +#### TEST BEGIN #### + +clean_containers + +export MR_TC="--tc300" +export MR_GROUPS="OpenDcae-c12:PM_MEAS_FILES" +export MR_FILE_PREFIX_MAPPING="PM_MEAS_FILES:A" + +export DR_TC="--tc normal" +export DR_FEEDS="2:A" + +export DR_REDIR_TC="--tc normal" +export DR_REDIR_FEEDS="2:A" + +export NUM_HTTPFILES="10" +export NUM_PNFS="1" +export FILE_SIZE="1MB" +export HTTP_TYPE="HTTP" +export HTTP_FILE_PREFIXES="A" +export NUM_HTTP_SERVERS=1 + +log_sim_settings + +start_simulators + +consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" + +mr_equal ctr_requests 0 60 +dr_equal ctr_published_files 0 60 + +mr_print tc_info +dr_print tc_info +drr_print tc_info + +start_dfc 0 + +dr_equal ctr_published_files 1 60 + +sleep_wait 30 + +dr_equal ctr_published_files 1 + +mr_greater ctr_requests 1 + +mr_equal ctr_events 1 +mr_equal ctr_unique_files 1 +mr_equal ctr_unique_PNFs 1 + +dr_equal ctr_publish_query 1 +dr_equal ctr_publish_query_bad_file_prefix 0 +dr_equal ctr_publish_query_published 0 +dr_equal ctr_publish_query_not_published 1 +dr_equal ctr_publish_req 1 +dr_equal ctr_publish_req_bad_file_prefix 0 +dr_equal ctr_publish_req_redirect 1 +dr_equal ctr_publish_req_published 0 +dr_equal ctr_published_files 1 +dr_equal ctr_double_publish 0 + +drr_equal ctr_publish_requests 1 +drr_equal ctr_publish_requests_bad_file_prefix 0 +drr_equal ctr_publish_responses 1 + +drr_equal dwl_volume 1000000 + +check_dfc_logs + +#### TEST COMPLETE #### + +store_logs END + +print_result diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC8.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC8.sh new file mode 100755 index 000000000..e0d30a5d6 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC8.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +TC_ONELINE_DESCR="One 5MB file from one PNF in one event using HTTP, from poll to publish" + +. ../common/testcase_common.sh $1 $2 + +#### TEST BEGIN #### + +clean_containers + +export MR_TC="--tc301" +export MR_GROUPS="OpenDcae-c12:PM_MEAS_FILES" +export MR_FILE_PREFIX_MAPPING="PM_MEAS_FILES:A" + +export DR_TC="--tc normal" +export DR_FEEDS="2:A" + +export DR_REDIR_TC="--tc normal" +export DR_REDIR_FEEDS="2:A" + +export NUM_HTTPFILES="10" +export NUM_PNFS="1" +export FILE_SIZE="5MB" +export HTTP_TYPE="HTTP" +export HTTP_FILE_PREFIXES="A" +export NUM_HTTP_SERVERS=1 + +log_sim_settings + +start_simulators + +consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" + +mr_equal ctr_requests 0 60 +dr_equal ctr_published_files 0 60 + +mr_print tc_info +dr_print tc_info +drr_print tc_info + +start_dfc 0 + +dr_equal ctr_published_files 1 60 + +sleep_wait 30 + +dr_equal ctr_published_files 1 + +mr_greater ctr_requests 1 + +mr_equal ctr_events 1 +mr_equal ctr_unique_files 1 +mr_equal ctr_unique_PNFs 1 + +dr_equal ctr_publish_query 1 +dr_equal ctr_publish_query_bad_file_prefix 0 +dr_equal ctr_publish_query_published 0 +dr_equal ctr_publish_query_not_published 1 +dr_equal ctr_publish_req 1 +dr_equal ctr_publish_req_bad_file_prefix 0 +dr_equal ctr_publish_req_redirect 1 +dr_equal ctr_publish_req_published 0 +dr_equal ctr_published_files 1 +dr_equal ctr_double_publish 0 + +drr_equal ctr_publish_requests 1 +drr_equal ctr_publish_requests_bad_file_prefix 0 +drr_equal ctr_publish_responses 1 + +drr_equal dwl_volume 5000000 + +check_dfc_logs + +#### TEST COMPLETE #### + +store_logs END + +print_result diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC9.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC9.sh new file mode 100755 index 000000000..f92cb4b1d --- /dev/null +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC9.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +TC_ONELINE_DESCR="One 50MB file from one PNF in one event using HTTP, from poll to publish" + +. ../common/testcase_common.sh $1 $2 + +#### TEST BEGIN #### + +clean_containers + +export MR_TC="--tc302" +export MR_GROUPS="OpenDcae-c12:PM_MEAS_FILES" +export MR_FILE_PREFIX_MAPPING="PM_MEAS_FILES:A" + +export DR_TC="--tc normal" +export DR_FEEDS="2:A" + +export DR_REDIR_TC="--tc normal" +export DR_REDIR_FEEDS="2:A" + +export NUM_HTTPFILES="10" +export NUM_PNFS="1" +export FILE_SIZE="50MB" +export HTTP_TYPE="HTTP" +export HTTP_FILE_PREFIXES="A" +export NUM_HTTP_SERVERS=1 + +log_sim_settings + +start_simulators + +consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" + +mr_equal ctr_requests 0 60 +dr_equal ctr_published_files 0 60 + +mr_print tc_info +dr_print tc_info +drr_print tc_info + +start_dfc 0 + +dr_equal ctr_published_files 1 60 + +sleep_wait 30 + +dr_equal ctr_published_files 1 + +mr_greater ctr_requests 1 + +mr_equal ctr_events 1 +mr_equal ctr_unique_files 1 +mr_equal ctr_unique_PNFs 1 + +dr_equal ctr_publish_query 1 +dr_equal ctr_publish_query_bad_file_prefix 0 +dr_equal ctr_publish_query_published 0 +dr_equal ctr_publish_query_not_published 1 +dr_equal ctr_publish_req 1 +dr_equal ctr_publish_req_bad_file_prefix 0 +dr_equal ctr_publish_req_redirect 1 +dr_equal ctr_publish_req_published 0 +dr_equal ctr_published_files 1 +dr_equal ctr_double_publish 0 + +drr_equal ctr_publish_requests 1 +drr_equal ctr_publish_requests_bad_file_prefix 0 +drr_equal ctr_publish_responses 1 + +drr_equal dwl_volume 50000000 + +check_dfc_logs + +#### TEST COMPLETE #### + +store_logs END + +print_result diff --git a/test/mocks/datafilecollector-testharness/auto-test/SingleFileSuite.sh b/test/mocks/datafilecollector-testharness/auto-test/SingleFileSuite.sh index 9e3d59c84..bfe9b9e4f 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/SingleFileSuite.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/SingleFileSuite.sh @@ -14,6 +14,9 @@ suite_setup ./FTC4.sh $1 $2 ./FTC5.sh $1 $2 ./FTC6.sh $1 $2 +./FTC7.sh $1 $2 +./FTC8.sh $1 $2 +./FTC9.sh $1 $2 ########################################## diff --git a/test/mocks/datafilecollector-testharness/common/README.md b/test/mocks/datafilecollector-testharness/common/README.md index f0e3eec79..dc57e09df 100644 --- a/test/mocks/datafilecollector-testharness/common/README.md +++ b/test/mocks/datafilecollector-testharness/common/README.md @@ -1,205 +1,216 @@ ## Common test scripts and env file for test -**test_env.sh**</br> -Common env variables for test in the auto-test dir. Used by the auto test cases/suites but could be used for other test script as well. +**test_env.sh**: Common env variables for test in the auto-test dir. +Used by the auto test cases/suites but could be used for other test script as well. -**testcase_common.sh**</br> -Common functions for auto test cases in the auto-test dir. A subset of the functions could be used in other test scripts as well. +**testcase_common.sh**: Common functions for auto test cases in the auto-test dir. +A subset of the functions could be used in other test scripts as well. -**testsuite_common.sh**</br> -Common functions for auto test suites in the auto-test dir. +**testsuite_common.sh**: Common functions for auto test suites in the auto-test dir. ## Descriptions of functions in testcase_common.sh The following is a list of the available functions in a test case file. Please see some of the defined test cases for examples. -**log_sim_settings**</br> +**log_sim_settings**: Print the env variables needed for the simulators and their setup -**clean_containers**</br> +**clean_containers**: Stop and remove all containers including dfc apps and simulators -**start_simulators**</br> +**start_simulators**: Start all simulators in the simulator group -**start_dfc <dfc-instance-id>**</br> -Start the dfc application. The arg shall be an integer from 0 to 5 representing the dfc instance to start. DFC app will get a name like 'dfc_app0' to 'dfc_app4'. +**start_dfc \<dfc-instance-id>**: +Start the dfc application. The arg shall be an integer from 0 to 5 representing the +dfc instance to start. DFC app will get a name like 'dfc_app0' to 'dfc_app4'. -**kill_dfc <dfc-instance-id> **</br> +**kill_dfc \<dfc-instance-id>**: Stop and remove the dfc app container with the instance id. -**consul_config_app <dfc-instance-id> <json-file-path>**</br> -Configure consul with json file with app config for a dfc instance using the dfc instance id and the json file. +**consul_config_app \<dfc-instance-id> \<json-file-path>**: +Configure consul with json file with app config for a dfc instance using the dfc +instance id and the json file. -**consul_config_dmaap <dfc-instance-id> <json-file-path>**</br> -Configure consul with json file with dmaap config for a dfc instance using the dfc instance id and the json file. +**consul_config_dmaap \<dfc-instance-id> \<json-file-path>**: +Configure consul with json file with dmaap config for a dfc instance using the dfc +instance id and the json file. -**kill_dr**</br> +**kill_dr**: Stop and remove the DR simulator container -**kill_drr**</br> +**kill_drr**: Stop and remove the DR redir simulator container -**kill_mr**</br> +**kill_mr**: Stop and remove the MR simulator container -**kill_sftp <sftp-instance-id>**</br> +**kill_sftp \<sftp-instance-id>**: Stop and remove a SFTP container with the supplied instance id (0-5). -**stop_sftp <sftp-instance-id>**</br> +**stop_sftp \<sftp-instance-id>**: Stop a SFTP container with the supplied instance id (0-5). -**start_sftp <sftp-instance-id>**</br> +**start_sftp \<sftp-instance-id>**: Start a previously stopped SFTP container with the supplied instance id (0-5). -**kill_ftpes <ftpes-instance-id>**</br> +**kill_ftpes \<ftpes-instance-id>**: Stop and remove a FTPES container with the supplied instance id (0-5). -**stop_ftpes <ftpes-instance-id>**</br> +**stop_ftpes \<ftpes-instance-id>**: Stop a FTPES container with the supplied instance id (0-5). -**start_ftpes <ftpes-instance-id>**</br> +**start_ftpes \<ftpes-instance-id>**: Start a previously stopped FTPES container with the supplied instance id (0-5). -**mr_print <vaiable-name>**</br> +**kill_http \<http-instance-id>**: +Stop and remove a HTTP container with the supplied instance id (0-5). + +**stop_http \<http-instance-id>**: +Stop a HTTP container with the supplied instance id (0-5). + +**start_http \<http-instance-id>**: +Start a previously stopped HTTP container with the supplied instance id (0-5). + +**mr_print \<variable-name>**: Print a variable value from the MR simulator. -**dr_print <vaiable-name>**</br> +**dr_print \<variable-name>**: Print a variable value from the DR simulator. -**drr_print <vaiable-name>**</br> +**drr_print \<variable-name>**: Print a variable value from the DR redir simulator. -**dfc_print <dfc-instance-id> <vaiable-name>**</br> +**dfc_print \<dfc-instance-id> <variable-name>**: Print a variable value from an dfc instance with the supplied instance id (0-5). -**mr_read <vaiable-name>**</br> +**mr_read \<variable-name>**: Read a variable value from MR sim and send to stdout -**dr_read <vaiable-name>**</br> +**dr_read \<variable-name>**: Read a variable value from DR sim and send to stdout -**drr_read <vaiable-name>**</br> +**drr_read \<variable-name>**: Read a variable value from DR redir sim and send to stdout -**sleep_wait <sleep-time-in-sec>**</br> +**sleep_wait \<sleep-time-in-sec>**: Sleep for a number of seconds -**sleep_heartbeat <sleep-time-in-sec>**</br> +**sleep_heartbeat \<sleep-time-in-sec>**: Sleep for a number of seconds and prints dfc heartbeat output every 30 sec -**mr_equal <variable-name> <target-value> [<timeout-in-sec>]**</br> +**mr_equal \<variable-name> \<target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the MR simulator is equal to a target value and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is equal to the targer or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value becomes equal to the target value or not. -**mr_greater <variable-name> <target-value> [<timeout-in-sec>]**</br> +**mr_greater \<variable-name> \<target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the MR simulator is greater than a target value and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is greater the target or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value is greater than the target value or not. -**mr_less <variable-name> <target-value> [<timeout-in-sec>]**</br> +**mr_less \<variable-name> \<target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the MR simulator is less than a target value and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is less than the target or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value is less than the target value or not. -**mr_contain_str <variable-name> <target-value> [<timeout-in-sec>]**</br> +**mr_contain_str \<variable-name> \<target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the MR simulator contains a substring target and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable contains +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable contains the target substring or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value contains the target substring or not. -**dr_equal <variable-name> <target-value> [<timeout-in-sec>]**</br> +**dr_equal <variable-name> <target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the DR simulator is equal to a target value and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is equal to the target or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value becomes equal to the target value or not. -**dr_greater <variable-name> <target-value> [<timeout-in-sec>]**</br> +**dr_greater <variable-name> <target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the DR simulator is greater than a target value and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is greater the target or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value is greater than the target value or not. -**dr_less <variable-name> <target-value> [<timeout-in-sec>]**</br> +**dr_less <variable-name> <target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the DR simulator is less than a target value and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is less than the target or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value is less than the target value or not. -**dr_contain_str <variable-name> <target-value> [<timeout-in-sec>]**</br> +**dr_contain_str \<variable-name> \<target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the DR simulator contains a substring target and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable contains +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable contains the target substring or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value contains the target substring or not. -**drr_equal <variable-name> <target-value> [<timeout-in-sec>]**</br> +**drr_equal \<variable-name> \<target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the DR Redir simulator is equal to a target value and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is equal to the target or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value becomes equal to the target value or not. -**drr_greater <variable-name> <target-value> [<timeout-in-sec>]**</br> +**drr_greater \<variable-name> \<target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the DR Redir simulator is greater than a target value and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is greater the target or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value is greater than the target value or not. -**drr_less <variable-name> <target-value> [<timeout-in-sec>]**</br> +**drr_less \<variable-name> \<target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the DR Redir simulator is less than a target value and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable is less than the target or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value is less than the target value or not. -**drr_contain_str <variable-name> <target-value> [<timeout-in-sec>]**</br> +**drr_contain_str \<variable-name> \<target-value> \[\<timeout-in-sec>]**: Tests if a variable value in the DR Redir simulator contains a substring target and an optional timeout. -</br>Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable contains +:Arg: `<variable-name> <target-value>` - This test set pass or fail depending on if the variable contains the target substring or not. -</br>Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds +:Arg: `<variable-name> <target-value> <timeout-in-sec>` - This test waits up to the timeout seconds before setting pass or fail depending on if the variable value contains the target substring or not. -**dfc_contain_str <variable-name> <substring-in-quotes>**</br> +**dfc_contain_str \<variable-name> \<substring-in-quotes>**: Test if a variable in the DFC contains a substring. -**store_logs <log-prefix>**</br> +**store_logs \<log-prefix>**: Store all dfc app and simulators log to the test case log dir. All logs get a prefix to separate logs stored at different steps in the test script. If logs need to be stored in several locations, use different prefix to easily identify the location when the logs where taken. -**check_dfc_log**</br> +**check_dfc_log**: Check the dfc application log for WARN and ERR messages and print the count. -**print_result**</br> +**print_result**: Print the test result. Only once at the very end of the script. -**print_all**</br> +**print_all**: Print all variables from the simulators and the dfc heartbeat. In addition, comment in the file can be added using the normal comment sign in bash '#'. @@ -209,11 +220,11 @@ Comments that shall be visible on the screen as well as in the test case log, us The following is a list of the available functions in a test suite file. Please see a existing test suite for examples. -**suite_setup**</br> +**suite_setup**: Sets up the test suite and print out a heading. -**run_tc <tc-script> <$1 from test suite script> <$2 from test suite script>**</br> +**run_tc \<tc-script> <$1 from test suite script> <$2 from test suite script>**: Execute a test case with arg from test suite script -**suite_complete**</br> +**suite_complete**: Print out the overall result of the executed test cases. diff --git a/test/mocks/datafilecollector-testharness/common/test_env.sh b/test/mocks/datafilecollector-testharness/common/test_env.sh index cc96ce8b9..a5ea8a17e 100644 --- a/test/mocks/datafilecollector-testharness/common/test_env.sh +++ b/test/mocks/datafilecollector-testharness/common/test_env.sh @@ -36,8 +36,11 @@ DFC_MAX_NUM=5 #Max number of dfc containers to run in p DFC_MAX_IDX=$(($DFC_MAX_NUM - 1)) #Max index of the dfc containers SFTP_BASE="dfc_sftp-server" #Base name of the dfc_sftp-server containers. Instance 0 will be named dfc_sftp-server0, instance 1 will named dfc_sftp-server1 etc FTPES_BASE="dfc_ftpes-server-vsftpd" #Base name of the dfc_ftpes-server-vsftpd containers. Instance 0 will be named dfc_ftpes-server-vsftpd0, instance 1 will named dfc_ftpes-server-vsftpd1 etc +HTTP_BASE="dfc_http-server" #Base name of the dfc_http-server containers. Instance 0 will be named dfc_http-server0, instance 1 will named dfc_http-server1 etc FTP_MAX_NUM=5 #Max number of sftp and ftpes containers to run in paralell in auto test +HTTP_MAX_NUM=5 #Max number of http containers to run in paralell in auto test FTP_MAX_IDX=$(($FTP_MAX_NUM - 1)) #Max index of sftp and ftpes containers +HTTP_MAX_IDX=$(($HTTP_MAX_NUM - 1)) #Max index of http containers #List of sftp server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc and the simulators in a private docker network SFTP_SIMS_CONTAINER="sftp-server0:22,sftp-server1:22,sftp-server2:22,sftp-server3:22,sftp-server4:22" @@ -45,14 +48,21 @@ SFTP_SIMS_CONTAINER="sftp-server0:22,sftp-server1:22,sftp-server2:22,sftp-server #List of sftp server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc and the simulators in a private docker network FTPES_SIMS_CONTAINER="ftpes-server-vsftpd0:21,ftpes-server-vsftpd1:21,ftpes-server-vsftpd2:21,ftpes-server-vsftpd3:21,ftpes-server-vsftpd4:21" +#List of http server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc and the simulators in a private docker network +HTTP_SIMS_CONTAINER="http-server0:80,http-server1:80,http-server2:80,http-server3:80,http-server4:80" + #List of sftp server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc as stand along app and the simulators in a private docker network SFTP_SIMS_LOCALHOST="localhost:1022,localhost:1023,localhost:1024,localhost:1025,localhost:1026" #List of ftpes server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc as stand along app and the simulators in a private docker network FTPES_SIMS_LOCALHOST="localhost:1032,localhost:1033,localhost:1034,localhost:1035,localhost:1036" +#List of http server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc as stand along app and the simulators in a private docker network +HTTP_SIMS_LOCALHOST="localhost:81,localhost:82,localhost:83,localhost:84,localhost:85" + export SFTP_SIMS=$SFTP_SIMS_CONTAINER #This env will be set to SFTP_SIMS_LOCALHOST if auto test is executed with 'manual-app' export FTPES_SIMS=$FTPES_SIMS_CONTAINER #This env will be set to FTPES_SIMS_LOCALHOST if auto test is executed with 'manual-app' +export HTTP_SIMS=$HTTP_SIMS_CONTAINER #This env will be set to HTTP_SIMS_LOCALHOST if auto test is executed with 'manual-app' #Host name of the DR redirect simulator export DR_REDIR_SIM="drsim_redir" #This env will be set to 'localhost' if auto test is executed with arg 'manual-app' diff --git a/test/mocks/datafilecollector-testharness/common/testcase_common.sh b/test/mocks/datafilecollector-testharness/common/testcase_common.sh index 61f3c5d43..9f156a901 100755 --- a/test/mocks/datafilecollector-testharness/common/testcase_common.sh +++ b/test/mocks/datafilecollector-testharness/common/testcase_common.sh @@ -151,7 +151,7 @@ fi echo "" -echo "Building images for the simulators if needed, MR, DR, DR Redir and FTPES simulators" +echo "Building images for the simulators if needed, MR, DR, DR Redir, FTPES and HTTP simulators" curdir=$PWD cd $SIM_GROUP cd ../dr-sim @@ -160,6 +160,8 @@ cd ../mr-sim docker build -t mrsim:latest . &> /dev/null cd ../ftpes-sftp-server docker build -t ftpes_vsftpd:latest -f Dockerfile-ftpes . &> /dev/null +cd ../http-https-server +docker build -t http_httpd:latest -f Dockerfile-http . &> /dev/null cd $curdir echo "" @@ -170,6 +172,7 @@ echo "DR simulator: " $(docker images | grep drsim_common) echo "DR redir simulator: " $(docker images | grep drsim_common) echo "SFTP: " $(docker images | grep atmoz/sftp) echo "FTPES: " $(docker images | grep ftpes_vsftpd) +echo "HTTP: " $(docker images | grep http_httpd) echo "Consul: " $(docker images | grep consul) echo "CBS: " $(docker images | grep platform.configbinding.app) echo "" @@ -179,6 +182,7 @@ echo "" if [ $START_ARG == "manual-app" ]; then export SFTP_SIMS=$SFTP_SIMS_LOCALHOST export FTPES_SIMS=$FTPES_SIMS_LOCALHOST + export HTTP_SIMS=$HTTP_SIMS_LOCALHOST export DR_REDIR_SIM="localhost" fi #else @@ -501,13 +505,18 @@ log_sim_settings() { echo "DR_REDIR_FEEDS= "$DR_REDIR_FEEDS echo "NUM_FTPFILES= "$NUM_FTPFILES + echo "NUM_HTTPFILES= "$NUM_HTTPFILES echo "NUM_PNFS= "$NUM_PNFS echo "FILE_SIZE= "$FILE_SIZE echo "FTP_TYPE= "$FTP_TYPE + echo "HTTP_TYPE= "$HTTP_TYPE echo "FTP_FILE_PREFIXES= "$FTP_FILE_PREFIXES + echo "HTTP_FILE_PREFIXES= "$HTTP_FILE_PREFIXES echo "NUM_FTP_SERVERS= "$NUM_FTP_SERVERS + echo "NUM_HTTP_SERVERS= "$NUM_HTTP_SERVERS echo "SFTP_SIMS= "$SFTP_SIMS echo "FTPES_SIMS= "$FTPES_SIMS + echo "HTTP_SIMS= "$HTTP_SIMS echo "" } @@ -757,6 +766,64 @@ start_ftpes() { __docker_start $appname } +# Stop and remove the HTTP container, arg: <http-instance-id> +kill_http() { + + if [ $# != 1 ]; then + __print_err "need one arg, <http-instance-id>" + exit 1 + fi + + if [ $1 -lt 0 ] || [ $1 -gt $HTTP_MAX_IDX ]; then + __print_err "arg should be 0.."$HTTP_MAX_IDX + exit 1 + fi + appname=$HTTP_BASE$1 + + echo "Killing HTTP, instance id: "$1 + + __docker_stop $appname + __docker_rm $appname +} + +# Stop HTTP container, arg: <http-instance-id> +stop_http() { + + if [ $# != 1 ]; then + __print_err "need one arg, <http-instance-id>" + exit 1 + fi + + if [ $1 -lt 0 ] || [ $1 -gt $HTTP_MAX_IDX ]; then + __print_err "arg should be 0.."$HTTP_MAX_IDX + exit 1 + fi + appname=$HTTP_BASE$1 + + echo "Stopping HTTP, instance id: "$1 + + __docker_stop $appname +} + +# Starts a stopped HTTP container, arg: <http-instance-id> +start_http() { + + if [ $# != 1 ]; then + __print_err "need one arg, <http-instance-id>" + exit 1 + fi + + if [ $1 -lt 0 ] || [ $1 -gt $HTTP_MAX_IDX ]; then + __print_err "arg should be 0.."$HTTP_MAX_IDX + exit 1 + fi + appname=$HTTP_BASE$1 + + echo "Starting HTTP, instance id: "$1 + + __docker_start $appname +} + # Print a variable value from the MR simulator. Arg: <variable-name> mr_print() { if [ $# != 1 ]; then @@ -1132,6 +1199,11 @@ store_logs() { docker logs $appname > $TESTLOGS/$ATC/${1}_${appname}.log 2>&1 done + for (( i=0; i<=$HTTP_MAX_IDX; i++ )); do + appname=$HTTP_BASE$i + docker logs $appname > $TESTLOGS/$ATC/${1}_${appname}.log 2>&1 + done + docker logs dfc_consul > $TESTLOGS/$ATC/$1_consul.log 2>&1 docker logs dfc_cbs > $TESTLOGS/$ATC/$1_cbs.log 2>&1 } diff --git a/test/mocks/datafilecollector-testharness/http-https-server/.gitignore b/test/mocks/datafilecollector-testharness/http-https-server/.gitignore new file mode 100644 index 000000000..bd6c5bed8 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/http-https-server/.gitignore @@ -0,0 +1 @@ +files/onap/* diff --git a/test/mocks/datafilecollector-testharness/http-https-server/Dockerfile-http b/test/mocks/datafilecollector-testharness/http-https-server/Dockerfile-http new file mode 100644 index 000000000..d3601b865 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/http-https-server/Dockerfile-http @@ -0,0 +1,8 @@ +FROM httpd:alpine + +COPY configuration/htpasswd /usr/local/apache2/conf/.htpasswd +COPY configuration/httpd.conf /usr/local/apache2/conf/httpd.conf +RUN chmod 644 /usr/local/apache2/conf/.htpasswd +RUN chmod 644 /usr/local/apache2/conf/httpd.conf +RUN chown root:root /usr/local/apache2/conf/.htpasswd +RUN chown root:root /usr/local/apache2/conf/httpd.conf diff --git a/test/mocks/datafilecollector-testharness/http-https-server/configuration/htpasswd b/test/mocks/datafilecollector-testharness/http-https-server/configuration/htpasswd new file mode 100644 index 000000000..07f1978e6 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/http-https-server/configuration/htpasswd @@ -0,0 +1 @@ +demo:$apr1$wRR00QBi$GvxjeaEqidmeSae9hE9AZ1 diff --git a/test/mocks/datafilecollector-testharness/http-https-server/configuration/httpd.conf b/test/mocks/datafilecollector-testharness/http-https-server/configuration/httpd.conf new file mode 100644 index 000000000..88081b441 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/http-https-server/configuration/httpd.conf @@ -0,0 +1,553 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See <URL:http://httpd.apache.org/docs/2.4/> for detailed information. +# In particular, see +# <URL:http://httpd.apache.org/docs/2.4/mod/directives.html> +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "logs/access_log" +# with ServerRoot set to "/usr/local/apache2" will be interpreted by the +# server as "/usr/local/apache2/logs/access_log", whereas "/logs/access_log" +# will be interpreted as '/logs/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot "/usr/local/apache2" + +# +# Mutex: Allows you to set the mutex mechanism and mutex file directory +# for individual mutexes, or change the global defaults +# +# Uncomment and change the directory if mutexes are file-based and the default +# mutex file directory is not on a local disk or is not appropriate for some +# other reason. +# +# Mutex default:logs + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the <VirtualHost> +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen 80 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +LoadModule mpm_event_module modules/mod_mpm_event.so +#LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +#LoadModule mpm_worker_module modules/mod_mpm_worker.so +LoadModule authn_file_module modules/mod_authn_file.so +#LoadModule authn_dbm_module modules/mod_authn_dbm.so +#LoadModule authn_anon_module modules/mod_authn_anon.so +#LoadModule authn_dbd_module modules/mod_authn_dbd.so +#LoadModule authn_socache_module modules/mod_authn_socache.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_groupfile_module modules/mod_authz_groupfile.so +LoadModule authz_user_module modules/mod_authz_user.so +#LoadModule authz_dbm_module modules/mod_authz_dbm.so +#LoadModule authz_owner_module modules/mod_authz_owner.so +#LoadModule authz_dbd_module modules/mod_authz_dbd.so +LoadModule authz_core_module modules/mod_authz_core.so +#LoadModule authnz_ldap_module modules/mod_authnz_ldap.so +#LoadModule authnz_fcgi_module modules/mod_authnz_fcgi.so +LoadModule access_compat_module modules/mod_access_compat.so +LoadModule auth_basic_module modules/mod_auth_basic.so +#LoadModule auth_form_module modules/mod_auth_form.so +#LoadModule auth_digest_module modules/mod_auth_digest.so +#LoadModule allowmethods_module modules/mod_allowmethods.so +#LoadModule isapi_module modules/mod_isapi.so +#LoadModule file_cache_module modules/mod_file_cache.so +#LoadModule cache_module modules/mod_cache.so +#LoadModule cache_disk_module modules/mod_cache_disk.so +#LoadModule cache_socache_module modules/mod_cache_socache.so +#LoadModule socache_shmcb_module modules/mod_socache_shmcb.so +#LoadModule socache_dbm_module modules/mod_socache_dbm.so +#LoadModule socache_memcache_module modules/mod_socache_memcache.so +#LoadModule socache_redis_module modules/mod_socache_redis.so +#LoadModule watchdog_module modules/mod_watchdog.so +#LoadModule macro_module modules/mod_macro.so +#LoadModule dbd_module modules/mod_dbd.so +#LoadModule bucketeer_module modules/mod_bucketeer.so +#LoadModule dumpio_module modules/mod_dumpio.so +#LoadModule echo_module modules/mod_echo.so +#LoadModule example_hooks_module modules/mod_example_hooks.so +#LoadModule case_filter_module modules/mod_case_filter.so +#LoadModule case_filter_in_module modules/mod_case_filter_in.so +#LoadModule example_ipc_module modules/mod_example_ipc.so +#LoadModule buffer_module modules/mod_buffer.so +#LoadModule data_module modules/mod_data.so +#LoadModule ratelimit_module modules/mod_ratelimit.so +LoadModule reqtimeout_module modules/mod_reqtimeout.so +#LoadModule ext_filter_module modules/mod_ext_filter.so +#LoadModule request_module modules/mod_request.so +#LoadModule include_module modules/mod_include.so +LoadModule filter_module modules/mod_filter.so +#LoadModule reflector_module modules/mod_reflector.so +#LoadModule substitute_module modules/mod_substitute.so +#LoadModule sed_module modules/mod_sed.so +#LoadModule charset_lite_module modules/mod_charset_lite.so +#LoadModule deflate_module modules/mod_deflate.so +#LoadModule xml2enc_module modules/mod_xml2enc.so +#LoadModule proxy_html_module modules/mod_proxy_html.so +#LoadModule brotli_module modules/mod_brotli.so +LoadModule mime_module modules/mod_mime.so +#LoadModule ldap_module modules/mod_ldap.so +LoadModule log_config_module modules/mod_log_config.so +#LoadModule log_debug_module modules/mod_log_debug.so +#LoadModule log_forensic_module modules/mod_log_forensic.so +#LoadModule logio_module modules/mod_logio.so +#LoadModule lua_module modules/mod_lua.so +LoadModule env_module modules/mod_env.so +#LoadModule mime_magic_module modules/mod_mime_magic.so +#LoadModule cern_meta_module modules/mod_cern_meta.so +#LoadModule expires_module modules/mod_expires.so +LoadModule headers_module modules/mod_headers.so +#LoadModule ident_module modules/mod_ident.so +#LoadModule usertrack_module modules/mod_usertrack.so +#LoadModule unique_id_module modules/mod_unique_id.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule version_module modules/mod_version.so +#LoadModule remoteip_module modules/mod_remoteip.so +#LoadModule proxy_module modules/mod_proxy.so +#LoadModule proxy_connect_module modules/mod_proxy_connect.so +#LoadModule proxy_ftp_module modules/mod_proxy_ftp.so +#LoadModule proxy_http_module modules/mod_proxy_http.so +#LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so +#LoadModule proxy_scgi_module modules/mod_proxy_scgi.so +#LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so +#LoadModule proxy_fdpass_module modules/mod_proxy_fdpass.so +#LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so +#LoadModule proxy_ajp_module modules/mod_proxy_ajp.so +#LoadModule proxy_balancer_module modules/mod_proxy_balancer.so +#LoadModule proxy_express_module modules/mod_proxy_express.so +#LoadModule proxy_hcheck_module modules/mod_proxy_hcheck.so +#LoadModule session_module modules/mod_session.so +#LoadModule session_cookie_module modules/mod_session_cookie.so +#LoadModule session_crypto_module modules/mod_session_crypto.so +#LoadModule session_dbd_module modules/mod_session_dbd.so +#LoadModule slotmem_shm_module modules/mod_slotmem_shm.so +#LoadModule slotmem_plain_module modules/mod_slotmem_plain.so +#LoadModule ssl_module modules/mod_ssl.so +#LoadModule optional_hook_export_module modules/mod_optional_hook_export.so +#LoadModule optional_hook_import_module modules/mod_optional_hook_import.so +#LoadModule optional_fn_import_module modules/mod_optional_fn_import.so +#LoadModule optional_fn_export_module modules/mod_optional_fn_export.so +#LoadModule dialup_module modules/mod_dialup.so +#LoadModule http2_module modules/mod_http2.so +#LoadModule proxy_http2_module modules/mod_proxy_http2.so +#LoadModule md_module modules/mod_md.so +#LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so +#LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so +#LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so +#LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so +LoadModule unixd_module modules/mod_unixd.so +#LoadModule heartbeat_module modules/mod_heartbeat.so +#LoadModule heartmonitor_module modules/mod_heartmonitor.so +#LoadModule dav_module modules/mod_dav.so +LoadModule status_module modules/mod_status.so +LoadModule autoindex_module modules/mod_autoindex.so +#LoadModule asis_module modules/mod_asis.so +#LoadModule info_module modules/mod_info.so +#LoadModule suexec_module modules/mod_suexec.so +<IfModule !mpm_prefork_module> + #LoadModule cgid_module modules/mod_cgid.so +</IfModule> +<IfModule mpm_prefork_module> + #LoadModule cgi_module modules/mod_cgi.so +</IfModule> +#LoadModule dav_fs_module modules/mod_dav_fs.so +#LoadModule dav_lock_module modules/mod_dav_lock.so +#LoadModule vhost_alias_module modules/mod_vhost_alias.so +#LoadModule negotiation_module modules/mod_negotiation.so +LoadModule dir_module modules/mod_dir.so +#LoadModule imagemap_module modules/mod_imagemap.so +#LoadModule actions_module modules/mod_actions.so +#LoadModule speling_module modules/mod_speling.so +#LoadModule userdir_module modules/mod_userdir.so +LoadModule alias_module modules/mod_alias.so +#LoadModule rewrite_module modules/mod_rewrite.so + +<IfModule unixd_module> +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User daemon +Group daemon + +</IfModule> + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# <VirtualHost> definition. These values also provide defaults for +# any <VirtualHost> containers you may define later in the file. +# +# All of these directives may appear inside <VirtualHost> containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin you@example.com + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +#ServerName www.example.com:80 + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# <Directory> blocks below. +# +<Directory /> + AllowOverride none + Require all denied +</Directory> + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/usr/local/apache2/htdocs" +<Directory "/usr/local/apache2/htdocs"> + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options Indexes FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # AllowOverride FileInfo AuthConfig Limit + # + AllowOverride None + + # + # Controls who can get stuff from this server. + # + AuthType Basic + AuthName "Restricted Content" + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user +</Directory> + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# +<IfModule dir_module> + DirectoryIndex index.html +</IfModule> + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# +<Files ".ht*"> + Require all denied +</Files> + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a <VirtualHost> +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a <VirtualHost> +# container, that host's errors will be logged there and not here. +# +ErrorLog /proc/self/fd/2 + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + +<IfModule log_config_module> + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + <IfModule logio_module> + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + </IfModule> + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a <VirtualHost> + # container, they will be logged here. Contrariwise, if you *do* + # define per-<VirtualHost> access logfiles, transactions will be + # logged therein and *not* in this file. + # + CustomLog /proc/self/fd/1 common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + #CustomLog "logs/access_log" combined +</IfModule> + +<IfModule alias_module> + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a <Directory> section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/" + +</IfModule> + +<IfModule cgid_module> + # + # ScriptSock: On threaded servers, designate the path to the UNIX + # socket used to communicate with the CGI daemon of mod_cgid. + # + #Scriptsock cgisock +</IfModule> + +# +# "/usr/local/apache2/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# +<Directory "/usr/local/apache2/cgi-bin"> + AllowOverride None + Options None + Require all granted +</Directory> + +<IfModule headers_module> + # + # Avoid passing HTTP_PROXY environment to CGI's on this or any proxied + # backend servers which have lingering "httpoxy" defects. + # 'Proxy' request header is undefined by the IETF, not listed by IANA + # + RequestHeader unset Proxy early +</IfModule> + +<IfModule mime_module> + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig conf/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + #AddType text/html .shtml + #AddOutputFilter INCLUDES .shtml +</IfModule> + +# +# The mod_mime_magic module allows the server to use various hints from the +# contents of the file itself to determine its type. The MIMEMagicFile +# directive tells the module where the hint definitions are located. +# +#MIMEMagicFile conf/magic + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# MaxRanges: Maximum number of Ranges in a request before +# returning the entire resource, or one of the special +# values 'default', 'none' or 'unlimited'. +# Default setting is to accept 200 Ranges. +#MaxRanges unlimited + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +#EnableSendfile on + +# Supplemental configuration +# +# The configuration files in the conf/extra/ directory can be +# included to add extra features or to modify the default configuration of +# the server, or you may simply copy their contents here and change as +# necessary. + +# Server-pool management (MPM specific) +#Include conf/extra/httpd-mpm.conf + +# Multi-language error messages +#Include conf/extra/httpd-multilang-errordoc.conf + +# Fancy directory listings +#Include conf/extra/httpd-autoindex.conf + +# Language settings +#Include conf/extra/httpd-languages.conf + +# User home directories +#Include conf/extra/httpd-userdir.conf + +# Real-time info on requests and configuration +#Include conf/extra/httpd-info.conf + +# Virtual hosts +#Include conf/extra/httpd-vhosts.conf + +# Local access to the Apache HTTP Server Manual +#Include conf/extra/httpd-manual.conf + +# Distributed authoring and versioning (WebDAV) +#Include conf/extra/httpd-dav.conf + +# Various default settings +#Include conf/extra/httpd-default.conf + +# Configure mod_proxy_html to understand HTML4/XHTML1 +<IfModule proxy_html_module> +Include conf/extra/proxy-html.conf +</IfModule> + +# Secure (SSL/TLS) connections +#Include conf/extra/httpd-ssl.conf +# +# Note: The following must must be present to support +# starting without SSL on platforms with no /dev/random equivalent +# but a statically compiled-in mod_ssl. +# +<IfModule ssl_module> +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin +</IfModule> diff --git a/test/mocks/datafilecollector-testharness/http-https-server/docker-compose.yml b/test/mocks/datafilecollector-testharness/http-https-server/docker-compose.yml new file mode 100644 index 000000000..537cfcfaf --- /dev/null +++ b/test/mocks/datafilecollector-testharness/http-https-server/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3' + +services: + + http-server: + container_name: http-server-httpd + image: httpd:alpine + ports: + - "80:80" + volumes: + - ./configuration/htpasswd:/usr/local/apache2/conf/.htpasswd:ro + - ./configuration/httpd.conf:/usr/local/apache/conf/httpd.conf:ro + + - ./files/onap/http/:/usr/local/apache2/htdocs + restart: on-failure diff --git a/test/mocks/datafilecollector-testharness/http-https-server/prepare.sh b/test/mocks/datafilecollector-testharness/http-https-server/prepare.sh new file mode 100755 index 000000000..937033c90 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/http-https-server/prepare.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# EXAMPLE: Run test case TC2 using the command "./prepare.sh TC2" +MAIN_DIRECTORY=./files/onap +TEST_FILE=./test_cases.yml +TEST=$1 +echo "Generating files for test case:" "$TEST" + +sf=$(sed -n '/'$TEST'/,$p' $TEST_FILE | grep -m 1 'size_files') +sf=${sf//*size_files: /} +sf_array=($sf) +echo "size_files=""$sf" + +nf=$(sed -n '/'$TEST'/,$p' $TEST_FILE | grep -m 1 'number_files') +nf=${nf//*number_files: /} +nf_array=($nf) +echo "number_files=""$nf" + +df=$(sed -n '/'$TEST'/,$p' $TEST_FILE | grep -m 1 'directory_files') +df=${df//*directory_files: /} +df_array=($df) +echo "directory_files=""$df" + +rm -rf $MAIN_DIRECTORY/* +if [ "${#sf_array[@]}" = "${#nf_array[@]}" ] && [ "${#nf_array[@]}" = "${#df_array[@]}" ]; +then + N_ELEMENTS=${#df_array[@]} + for ((n=0;n<$N_ELEMENTS;n++)) + do + # Create directory + DIRECTORY=$MAIN_DIRECTORY/${df_array[$n]} + mkdir -p "$DIRECTORY" + + # Create original file + FILE_SIZE=${sf_array[$n]} + FILE_NAME=$FILE_SIZE"MB.tar.gz" + dd if=/dev/urandom of=$DIRECTORY/$FILE_NAME bs=1k count=$(echo $FILE_SIZE*1000/1 | bc) + + # Create symlinks + N_SYMLINKS=${nf_array[$n]}-1 + for ((l=0;l<=$N_SYMLINKS;l++)) + do + SYMLINK_NAME=$FILE_SIZE"MB_"$l".tar.gz" + ln -s ./$FILE_NAME $DIRECTORY/$SYMLINK_NAME + done + done +else +echo "ERROR: The number of parameters in size_files, number_files, and directory_files must be equal!" +fi + diff --git a/test/mocks/datafilecollector-testharness/http-https-server/test_cases.yml b/test/mocks/datafilecollector-testharness/http-https-server/test_cases.yml new file mode 100644 index 000000000..08f632e28 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/http-https-server/test_cases.yml @@ -0,0 +1,16 @@ +# EXAMPLE: TC1 generates (i) 10 files of 1 MB in http directory, +# (ii) 30 files of 5 MB in sftp directory, and (iii) 10 files of 10 MB in http directory +TC1: + size_files: 1 5 10 + number_files: 10 30 10 + directory_files: http + +TC2: + size_files: 0.5 1 5 + number_files: 2 3 1 + directory_files: http + +TC_10000: + size_files: 1 1 5 5 50 50 + number_files: 10000 10000 10000 10000 1 1 + directory_files: http diff --git a/test/mocks/datafilecollector-testharness/mr-sim/README.md b/test/mocks/datafilecollector-testharness/mr-sim/README.md index 056136f94..055e7d914 100644 --- a/test/mocks/datafilecollector-testharness/mr-sim/README.md +++ b/test/mocks/datafilecollector-testharness/mr-sim/README.md @@ -10,7 +10,7 @@ This readme contains: The MR-sim is a python script delivering batches of events including one or more fileReady for one or more PNFs. It is possible to configure number of events, PNFs, consumer groups, exising or missing files, file prefixes and change identifier. -In addition, MR sim can be configured to deliver file url for up to 5 FTP servers (simulating the PNFs). +In addition, MR sim can be configured to deliver file url for up to 5 FTP and 5 HTTP servers (simulating the PNFs). ## Building and running @@ -32,8 +32,10 @@ The following envrionment vaiables are used: - **FTPES_SIMS** - A comma-separated list of hostname:port for the FTP servers to generate ftpes file urls for. If not set MR sim will assume 'localhost:21'. Minimum 1 and maximum 5 host-port pairs can be given. - **SFTP_SIMS** - A comma-separated list of hostname:port for the FTP servers to generate sftp file urls for. If not set MR sim will assume 'localhost:1022'. Minimum 1 and maximum 5 host-port pairs can be given. +- **HTTP_SIMS** - A comma-separated list of hostname:port for the HTTP servers to generate http file urls for. If not set MR sim will assume 'localhost:81'. Minimum 1 and maximum 5 host-port pairs can be given. - **NUM_FTP_SERVERS** - Number of FTP servers to use out of those specified in the envrioment variables above. The number shall be in the range 1-5. -- **MR_GROUPS** - A comma-separated list of consummer-group:changeId[:changeId]\*. Defines which change identifier that should be used for each consumer gropu. If not set the MR-sim will assume 'OpenDcae-c12:PM_MEAS_FILES'. +- **NUM_HTTP_SERVERS** - Number of HTTP servers to use out of those specified in the envrioment variables above. The number shall be in the range 1-5. +- **MR_GROUPS** - A comma-separated list of consummer-group:changeId\[:changeId]\*. Defines which change identifier that should be used for each consumer group. If not set the MR-sim will assume 'OpenDcae-c12:PM_MEAS_FILES'.not set the MR-sim will assume 'OpenDcae-c12:PM_MEAS_FILES'. - **MR_FILE_PREFIX_MAPPING** - A comma-separated list of changeId:filePrefix. Defines which file prefix to use for each change identifier, needed to distinguish files for each change identifiers. If not set the MR-sim will assume 'PM_MEAS_FILES:A ## Statistics read-out and commands @@ -109,7 +111,7 @@ The simulator can be queried for statistics and started/stopped (use curl from ## Common TC info File names for 1MB, 5MB and 50MB files -Files in the format: <size-in-mb>MB\_<sequence-number>.tar.gz Ex. for 5MB file with sequence number 12: 5MB_12.tar.gz +Files in the format: <size-in-mb>MB\_<sequence-number>.tar.gz Ex. for 5MB file with sequence number 12: 5MB\_12.tar.gz The sequence numbers are stepped so that all files have unique names Missing files (files that are not expected to be found in the ftp server. Format: MissingFile\*<sequence-number>.tar.gz @@ -175,9 +177,9 @@ TC1302 - 700 ME, SFTP, 50MB files, 100 files per event, endless number of events TC1500 - 700 ME, SFTP, 1MB files, 100 files per event, 35 events per poll, simulating 25h backlog of decreasing number of outdated files and then 20 event polls every 15min for 1h -Changing the first digit in tc number will change the test case to run FTPES instead. Eg. TC201 is FTPES version of TC101. +Changing the first digit in tc number will change the test case to run FTPES or HTTP instead. Eg. TC201 is FTPES version of TC101. -TC2XX is same as TC1XX but with FTPES +TC2XX is same as TC1XX but with FTPES, TC3XX is same as TC1XX but with HTTP TC6XX is same as TC5XX but with FTPES @@ -193,7 +195,7 @@ TC2XXX is same as TC1XXX but with FTPES 4. `pip3 freeze | grep -v "pkg-resources" > requirements.txt` #to create a req file 5. `FLASK_APP=mr-sim.py flask run` or - `python3 mr-sim.py ` + ` python3 mr-sim.py ` 6. Check/lint/format the code before commit/amed by `autopep8 --in-place --aggressive --aggressive mr-sim.py` ## User workflow on \*NIX @@ -201,8 +203,8 @@ TC2XXX is same as TC1XXX but with FTPES When cloning/fetching from the repository first time: 1. `git clone` -2. `cd "..." ` #navigate to this folder -3. `source setup.sh ` #setting up virtualenv and install requirements +2. ` cd "..." ` #navigate to this folder +3. ` source setup.sh ` #setting up virtualenv and install requirements you'll get a sourced virtualenv shell here, check prompt 4. `(env) $ python3 mr-sim.py --help` alternatively diff --git a/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py b/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py index 982f4b8b3..809f18661 100644 --- a/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py +++ b/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py @@ -17,7 +17,10 @@ sftp_hosts = [] sftp_ports = [] ftpes_hosts = [] ftpes_ports = [] +http_hosts = [] +http_ports = [] num_ftp_servers = 1 +num_http_servers = 1 def sumList(ctrArray): @@ -515,11 +518,17 @@ def MR_reply(consumerGroup, consumerId): elif args.tc810: return tc710(groupIndex, changeId, filePrefix, "ftpes") + elif args.tc300: + return tc100(groupIndex, changeId, filePrefix, "http", "1MB") + elif args.tc301: + return tc100(groupIndex, changeId, filePrefix, "http", "5MB") + elif args.tc302: + return tc100(groupIndex, changeId, filePrefix, "http", "50MB") #### Test case functions -def tc100(groupIndex, changeId, filePrefix, ftpType, fileSize): +def tc100(groupIndex, changeId, filePrefix, schemeType, fileSize): global ctr_responses global ctr_events @@ -532,8 +541,11 @@ def tc100(groupIndex, changeId, filePrefix, ftpType, fileSize): nodeIndex = 0 nodeName = createNodeName(nodeIndex) fileName = createFileName(groupIndex, filePrefix, nodeName, seqNr, fileSize) - msg = getEventHead(groupIndex, changeId, nodeName) + getEventName(fileName, ftpType, "onap", "pano", + msg = getEventHead(groupIndex, changeId, nodeName) + getEventName(fileName, schemeType, "onap", "pano", nodeIndex) + getEventEnd() + if schemeType == "http": + msg = getEventHead(groupIndex, changeId, nodeName) + getEventName(fileName, schemeType, "demo", "demo123456!", + nodeIndex) + getEventEnd() fileMap[groupIndex][seqNr * hash(filePrefix)] = seqNr ctr_events[groupIndex] = ctr_events[groupIndex] + 1 return buildOkResponse("[" + msg + "]") @@ -1183,6 +1195,10 @@ def getEventName(fn, type, user, passwd, nodeIndex): if (type == "ftpes"): port = ftpes_ports[nodeIndex] ip = ftpes_hosts[nodeIndex] + elif (type == "http"): + nodeIndex = nodeIndex % num_http_servers + port = http_ports[nodeIndex] + ip = http_hosts[nodeIndex] nameStr = """{ "name": \"""" + fn + """", @@ -1231,11 +1247,15 @@ if __name__ == "__main__": # IP addresses to use for ftp servers, using localhost if not env var is set sftp_sims = os.environ.get('SFTP_SIMS', 'localhost:1022') ftpes_sims = os.environ.get('FTPES_SIMS', 'localhost:21') + http_sims = os.environ.get('HTTP_SIMS', 'localhost:81') num_ftp_servers = int(os.environ.get('NUM_FTP_SERVERS', 1)) + num_http_servers = int(os.environ.get('NUM_HTTP_SERVERS', 1)) print("Configured sftp sims: " + sftp_sims) print("Configured ftpes sims: " + ftpes_sims) + print("Configured http sims: " + http_sims) print("Configured number of ftp servers: " + str(num_ftp_servers)) + print("Configured number of http servers: " + str(num_http_servers)) tmp = sftp_sims.split(',') for i in range(len(tmp)): @@ -1249,6 +1269,12 @@ if __name__ == "__main__": ftpes_hosts.append(hp[0]) ftpes_ports.append(hp[1]) + tmp = http_sims.split(',') + for i in range(len(tmp)): + hp = tmp[i].split(':') + http_hosts.append(hp[0]) + http_ports.append(hp[1]) + groups = os.environ.get('MR_GROUPS', 'OpenDcae-c12:PM_MEAS_FILES') print("Groups detected: " + groups) configuredPrefixes = os.environ.get('MR_FILE_PREFIX_MAPPING', 'PM_MEAS_FILES:A') @@ -1597,6 +1623,20 @@ if __name__ == "__main__": action='store_true', help='TC810 - 700 MEs, FTPES, 1MB files, 100 files per event, 3500 events, 35 event per poll.') + # HTTP TCs with single ME + parser.add_argument( + '--tc300', + action='store_true', + help='TC300 - One ME, HTTP, 1 1MB file, 1 event') + parser.add_argument( + '--tc301', + action='store_true', + help='TC301 - One ME, HTTP, 1 5MB file, 1 event') + parser.add_argument( + '--tc302', + action='store_true', + help='TC302 - One ME, HTTP, 1 50MB file, 1 event') + args = parser.parse_args() if args.tc100: @@ -1731,6 +1771,13 @@ if __name__ == "__main__": elif args.tc810: tc_num = "TC# 810" + elif args.tc300: + tc_num = "TC# 300" + elif args.tc301: + tc_num = "TC# 301" + elif args.tc302: + tc_num = "TC# 302" + else: print("No TC was defined") print("use --help for usage info") @@ -1746,7 +1793,12 @@ if __name__ == "__main__": print("Using " + str(ftpes_hosts[i]) + ":" + str(ftpes_ports[i]) + " for ftpes server with index " + str( i) + " for ftpes server address and port in file urls.") + for i in range(len(http_hosts)): + print("Using " + str(http_hosts[i]) + ":" + str(http_ports[i]) + " for http server with index " + str( + i) + " for http server address and port in file urls.") + print("Using up to " + str(num_ftp_servers) + " ftp servers, for each protocol for PNFs.") + print("Using up to " + str(num_http_servers) + " http servers, for each protocol for PNFs.") def https_app(**kwargs): diff --git a/test/mocks/datafilecollector-testharness/simulator-group/README.md b/test/mocks/datafilecollector-testharness/simulator-group/README.md index a37c2fb8d..75abec9f1 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/README.md +++ b/test/mocks/datafilecollector-testharness/simulator-group/README.md @@ -16,7 +16,7 @@ In general these steps are needed to run the simulator group and dfc # Overview of the simulators. -There are 5 different types of simulators. For futher details, see the README.md in each simulator dir. +There are 6 different types of simulators. For futher details, see the README.md in each simulator dir. 1. The MR simulator emits fileready events, upon poll requests, with new and historice file references It is possible to configire the change identifier and file prefixes for these identifiers and for which consumer groups @@ -34,6 +34,7 @@ There are 5 different types of simulators. For futher details, see the README.md possible to configure the distrubution of files over these 5 servers (from 1 up to 5 severs). At start of the server, the server is populated with files to download. 5. The FTPES simulator(s) is the same as the SFTP except that it using the FTPES protocol. +6. The HTTP simulator(s) is the same as SFTP except that it using the HTTP protocol. # Build the simulator images @@ -72,8 +73,8 @@ Do the manual steps to prepare the simulator images: Edit the `docker-compose-setup.sh` (or create a copy) to setup the env variables to the desired test behavior for each simulators. See each simulator to find a description of the available settings (DR_TC, DR_REDIR_TC and MR_TC). The following env variables shall be set (example values). -Note that NUM_FTPFILES and NUM_PNFS controls the number of ftp files created in the ftp servers. -A total of NUM_FTPFILES \* NUM_PNFS ftp files will be created in each ftp server (4 files in the below example). +Note that NUM_FTPFILES, NUM_HTTPFILES and NUM_PNFS controls the number of ftp/http files created in the ftp/http servers. +A total of NUM_FTPFILES \* NUM_PNFS (or NUM_HTTPFILES \* NUM_PNFS) ftp/http files will be created in each ftp/http server (4 files in the below example for ftp server). Large settings will be time consuming at start of the servers. Note that the number of files must match the number of file references emitted from the MR sim. @@ -94,23 +95,24 @@ FILE_SIZE="1MB" #File size for FTP file (1KB, 1MB, 5MB, 50MB or AL FTP_TYPE="SFTP" #Type of FTP files to generate (SFTP, FTPES or ALL) If `FTP_TYPE` is set to `ALL`, both ftp servers will be populated with the same files. If set to `SFTP` or `FTPES` then only the server serving that protocol will be populated with files. +`HTTP_TYPE` for now is only prepared for `HTTP` protocol. `HTTPS` protocol will be served in the future Run the script `docker-compose-setup.sh`to create a docker-compose with the desired settings. The desired setting in the script need to be manually adapted to for each specific simulator behavior according to the above. Check each simulator for available parameters. All simulators will be started with the generated docker-compose.yml file -To generate ftp url with IP different from localhost, set SFTP_SIM_IP and/or FTPES_SIM_IP env variables to the addreses of the ftp servers before starting. +To generate ftp/http url with IP different from localhost, set SFTP_SIM_IP and/or FTPES_SIM_IP and/or HTTP_SIM_IP env variables to the addreses of the ftp servers before starting. So farm, this only works when the simulator python script is started from the command line. Kill all the containers with `simulators-kill.se` `simulators_start.sh` is for CSIT test and requires the env variables for test setting to be present in the shell. -`setup-ftp-files.for-image.sh` is for CSIT and executed when the ftp servers are started from the docker-compose-setup.sh\`. +`setup-ftp-files.for-image.sh` and `setup-http-files-for-image.sh` is for CSIT and executed when the ftp/http servers are started from the docker-compose-setup.sh\`. To make DFC to be able to connect to the simulator containers, DFC need to run in host mode. -Start DFC by the following cmd: `docker run -d --network="host" --name dfc_app <dfc-image> ` +Start DFC by the following cmd: ` docker run -d --network="host" --name dfc_app <dfc-image> ` `<dfc-image>` could be either the locally built image `onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server` or the one in nexus `nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server`. diff --git a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh index e6df022ce..ad45559eb 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh @@ -13,15 +13,24 @@ export DR_FEEDS="1:A,2:B,3:C,4:D" # Comma-separated of <fee export DR_REDIR_TC="--tc normal" # Test behaviour for DR redir sim export DR_REDIR_FEEDS="1:A,2:B,3:C,4:D" # Comma-separated of <feed-id>:<file-name-prefixes> for DR redir sim +export NUM_PNFS="700" # Number of unuqie PNFs to generate file for +export FILE_SIZE="1MB" # File size for file (1KB, 1MB, 5MB, 50MB or ALL) + export NUM_FTPFILES="105" # Number of FTP files to generate per PNF -export NUM_PNFS="700" # Number of unuqie PNFs to generate FTP file for -export FILE_SIZE="1MB" # File size for FTP file (1KB, 1MB, 5MB, 50MB or ALL) export FTP_TYPE="SFTP" # Type of FTP files to generate (SFTP, FTPES or ALL) export FTP_FILE_PREFIXES="A,B,C,D" # Comma separated list of file name prefixes for ftp files export NUM_FTP_SERVERS=1 # Number of FTP server to distribute the PNFs (Max 5) +export NUM_HTTPFILES="105" # Number of HTTP files to generate per PNF +export HTTP_TYPE="HTTP" # Type of HTTP files to generate (HTTP, HTTPS or ALL) +export HTTP_FILE_PREFIXES="A,B,C,D" # Comma separated list of file name prefixes for http files +export NUM_HTTP_SERVERS=1 # Number of HTTP server to distribute the PNFs (Max 5) +export BASIC_AUTH_LOGIN=demo +export BASIC_AUTH_PASSWORD=demo123456! + export SFTP_SIMS="localhost:21,localhost:22,localhost:23,localhost:24,localhost:25" # Comma separated list for SFTP servers host:port export FTPES_SIMS="localhost:1022,localhost:1023,localhost:1024,localhost:1026,localhost:1026" # Comma separated list for FTPES servers host:port +export HTTP_SIMS="localhost:81,localhost:82,localhost:83,localhost:84,localhost:85" # Comma separated list for HTTP servers host:port export DR_REDIR_SIM="localhost" # Hostname of DR redirect server diff --git a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml index 41cafc386..c4ed2fec3 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml +++ b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml @@ -77,7 +77,9 @@ services: environment: SFTP_SIMS: ${SFTP_SIMS} FTPES_SIMS: ${FTPES_SIMS} + HTTP_SIMS: ${HTTP_SIMS} NUM_FTP_SERVERS: ${NUM_FTP_SERVERS} + NUM_HTTP_SERVERS: ${NUM_HTTP_SERVERS} MR_GROUPS: ${MR_GROUPS} MR_FILE_PREFIX_MAPPING: ${MR_FILE_PREFIX_MAPPING} image: mrsim:latest @@ -225,3 +227,48 @@ services: - ./tls:/etc/ssl/private/:rw depends_on: - tls-init-container + + http-server0: + networks: + - dfcnet + container_name: dfc_http-server0 + image: http_httpd:latest + ports: + - "81:80" + restart: on-failure + + http-server1: + networks: + - dfcnet + container_name: dfc_http-server1 + image: http_httpd:latest + ports: + - "82:80" + restart: on-failure + + http-server2: + networks: + - dfcnet + container_name: dfc_http-server2 + image: http_httpd:latest + ports: + - "83:80" + restart: on-failure + + http-server3: + networks: + - dfcnet + container_name: dfc_http-server3 + image: http_httpd:latest + ports: + - "84:80" + restart: on-failure + + http-server4: + networks: + - dfcnet + container_name: dfc_http-server4 + image: http_httpd:latest + ports: + - "85:80" + restart: on-failure diff --git a/test/mocks/datafilecollector-testharness/simulator-group/prepare-images.sh b/test/mocks/datafilecollector-testharness/simulator-group/prepare-images.sh index 11e4c0e11..fa56724bb 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/prepare-images.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/prepare-images.sh @@ -17,3 +17,7 @@ cd ../ftpes-sftp-server docker build -t ftpes_vsftpd:latest -f Dockerfile-ftpes . +#Build image for http server +cd ../http-https-server + +docker build -t http_httpd:latest -f Dockerfile-http . diff --git a/test/mocks/datafilecollector-testharness/simulator-group/setup-http-files-for-image.sh b/test/mocks/datafilecollector-testharness/simulator-group/setup-http-files-for-image.sh new file mode 100755 index 000000000..9d54bc3ef --- /dev/null +++ b/test/mocks/datafilecollector-testharness/simulator-group/setup-http-files-for-image.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env sh + +# Script to create files for the HTTP server to return upon request. +# The file names matches the files names in the events polled from the MR simulator. +# Intended for execution in the running http containers in the http-root dir. + +NUM=200 #Default number of files +PNFS=1 #Default number of PNFs +FSIZE="ALL" +PREFIXES="A" +HTTP_SERV_INDEX=0 +NUM_HTTP_SERVERS=1 + +if [ $# -ge 1 ]; then + NUM=$1 +fi +if [ $# -ge 2 ]; then + PNFS=$2 +fi +if [ $# -ge 3 ]; then + FSIZE=$3 + if [ $3 != "1KB" ] && [ $3 != "1MB" ] && [ $3 != "5MB" ] && [ $3 != "50MB" ] && [ $3 != "ALL" ]; then + echo "File size shall be 1KB|1MB|5MB|50MB|ALL" + exit + fi +fi +if [ $# -ge 4 ]; then + PREFIXES=$4 +fi +if [ $# -ge 5 ]; then + NUM_HTTP_SERVERS=$5 +fi +if [ $# -ge 6 ]; then + HTTP_SERV_INDEX=$6 +fi +if [ $# -lt 1 ] || [ $# -gt 6 ]; then + echo "Wrong args, usage: setup-http-files-for-image.sh [ <num-files> [ <num-PNFs> [ 1KB|1MB|5MB|50MB [ <comma-separated-file-name-prefixs> [ <number-of-http-servers> <http-server-index> ] ] ] ] ] ]" + exit +fi + +echo "Running http file creations. " $PNFS " PNFs and " $NUM " files for each PNF with file size(s) " $FSIZE "and file prefixe(s) " $PREFIXES " in http servers with index " $HTTP_SERV_INDEX + +truncate -s 1KB 1KB.tar.gz +truncate -s 1MB 1MB.tar.gz +truncate -s 5MB 5MB.tar.gz +truncate -s 50MB 50MB.tar.gz + +for fnp in ${PREFIXES//,/ } +do + p=0 + while [ $p -lt $PNFS ]; do + if [[ $(($p%$NUM_HTTP_SERVERS)) == $HTTP_SERV_INDEX ]]; then + i=0 + while [ $i -lt $NUM ]; do #Problem with for loop and var substituion in curly bracket....so used good old style loop + if [ $FSIZE = "ALL" ] || [ $FSIZE = "1KB" ]; then ln -s 1KB.tar.gz $fnp'20000626.2315+0200-2330+0200_PNF'$p'-'$i'-1KB.tar.gz' >& /dev/null; fi + if [ $FSIZE = "ALL" ] || [ $FSIZE = "1MB" ]; then ln -s 1MB.tar.gz $fnp'20000626.2315+0200-2330+0200_PNF'$p'-'$i'-1MB.tar.gz' >& /dev/null; fi + if [ $FSIZE = "ALL" ] || [ $FSIZE = "5MB" ]; then ln -s 5MB.tar.gz $fnp'20000626.2315+0200-2330+0200_PNF'$p'-'$i'-5MB.tar.gz' >& /dev/null; fi + if [ $FSIZE = "ALL" ] || [ $FSIZE = "50MB" ]; then ln -s 50MB.tar.gz $fnp'20000626.2315+0200-2330+0200_PNF'$p'-'$i'-50MB.tar.gz' >& /dev/null; fi + let i=i+1 + done + fi + let p=p+1 + done +done diff --git a/test/mocks/datafilecollector-testharness/simulator-group/simulators-kill.sh b/test/mocks/datafilecollector-testharness/simulator-group/simulators-kill.sh index 32045ea56..697850339 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/simulators-kill.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/simulators-kill.sh @@ -16,6 +16,11 @@ docker kill dfc_ftpes-server-vsftpd1 docker kill dfc_ftpes-server-vsftpd2 docker kill dfc_ftpes-server-vsftpd3 docker kill dfc_ftpes-server-vsftpd4 +docker kill dfc_http-server0 +docker kill dfc_http-server1 +docker kill dfc_http-server2 +docker kill dfc_http-server3 +docker kill dfc_http-server4 docker kill dfc_cbs docker kill dfc_consul @@ -33,7 +38,12 @@ docker rm dfc_ftpes-server-vsftpd1 docker rm dfc_ftpes-server-vsftpd2 docker rm dfc_ftpes-server-vsftpd3 docker rm dfc_ftpes-server-vsftpd4 +docker rm dfc_http-server0 +docker rm dfc_http-server1 +docker rm dfc_http-server2 +docker rm dfc_http-server3 +docker rm dfc_http-server4 docker rm dfc_cbs docker rm dfc_consul -echo "done"
\ No newline at end of file +echo "done" diff --git a/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh b/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh index 25750d38f..895949caa 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh @@ -12,6 +12,18 @@ server_check() { echo "Simulator " $1 " on localhost:$2$3 - no response" } +basic_auth_server_check() { + for i in {1..10}; do + res=$(curl -s -o /dev/null -w "%{http_code}" http://$BASIC_AUTH_LOGIN:$BASIC_AUTH_PASSWORD@localhost:$2$3) + if [ $res -gt 199 ] && [ $res -lt 300 ]; then + echo "Simulator " $1 " on localhost:$2$3 responded ok" + return + fi + sleep 1 + done + echo "Simulator " $1 " on localhost:$2$3 - no response" +} + server_check_https() { for i in {1..10}; do res=$(curl -k -s -o /dev/null -w "%{http_code}" https://localhost:$2$3) @@ -55,6 +67,16 @@ DOCKER_SIM_NWNAME="dfcnet" echo "Creating docker network $DOCKER_SIM_NWNAME, if needed" docker network ls| grep $DOCKER_SIM_NWNAME > /dev/null || docker network create $DOCKER_SIM_NWNAME +if [ -z "$NUM_FTP_SERVERS" ] + then + export NUM_FTP_SERVERS=1 +fi + +if [ -z "$NUM_HTTP_SERVERS" ] + then + export NUM_HTTP_SERVERS=1 +fi + docker-compose -f docker-compose-template.yml config > docker-compose.yml docker-compose up -d @@ -64,6 +86,7 @@ sudo chown $(id -u):$(id -g) consul/consul/ declare -a SFTP_SIM declare -a FTPES_SIM +declare -a HTTP_SIM DR_SIM="$(docker ps -q --filter='name=dfc_dr-sim')" DR_RD_SIM="$(docker ps -q --filter='name=dfc_dr-redir-sim')" @@ -78,6 +101,11 @@ FTPES_SIM[1]="$(docker ps -q --filter='name=dfc_ftpes-server-vsftpd1')" FTPES_SIM[2]="$(docker ps -q --filter='name=dfc_ftpes-server-vsftpd2')" FTPES_SIM[3]="$(docker ps -q --filter='name=dfc_ftpes-server-vsftpd3')" FTPES_SIM[4]="$(docker ps -q --filter='name=dfc_ftpes-server-vsftpd4')" +HTTP_SIM[0]="$(docker ps -q --filter='name=dfc_http-server0')" +HTTP_SIM[1]="$(docker ps -q --filter='name=dfc_http-server1')" +HTTP_SIM[2]="$(docker ps -q --filter='name=dfc_http-server2')" +HTTP_SIM[3]="$(docker ps -q --filter='name=dfc_http-server3')" +HTTP_SIM[4]="$(docker ps -q --filter='name=dfc_http-server4')" CBS_SIM="$(docker ps -q --filter='name=dfc_cbs')" CONSUL_SIM="$(docker ps -q --filter='name=dfc_consul')" @@ -96,6 +124,11 @@ if [ $(docker inspect --format '{{ .State.Running }}' $DR_SIM) ] && \ [ $(docker inspect --format '{{ .State.Running }}' ${FTPES_SIM[2]}) ] && \ [ $(docker inspect --format '{{ .State.Running }}' ${FTPES_SIM[3]}) ] && \ [ $(docker inspect --format '{{ .State.Running }}' ${FTPES_SIM[4]}) ] && \ +[ $(docker inspect --format '{{ .State.Running }}' ${HTTP_SIM[0]}) ] && \ +[ $(docker inspect --format '{{ .State.Running }}' ${HTTP_SIM[1]}) ] && \ +[ $(docker inspect --format '{{ .State.Running }}' ${HTTP_SIM[2]}) ] && \ +[ $(docker inspect --format '{{ .State.Running }}' ${HTTP_SIM[3]}) ] && \ +[ $(docker inspect --format '{{ .State.Running }}' ${HTTP_SIM[4]}) ] && \ [ $(docker inspect --format '{{ .State.Running }}' $CBS_SIM) ] && \ [ $(docker inspect --format '{{ .State.Running }}' $CONSUL_SIM) ] then @@ -107,6 +140,16 @@ if [ $(docker inspect --format '{{ .State.Running }}' $DR_SIM) ] && \ fi done +if [ -z "$BASIC_AUTH_LOGIN" ] + then + BASIC_AUTH_LOGIN=demo +fi + +if [ -z "$BASIC_AUTH_PASSWORD" ] + then + BASIC_AUTH_PASSWORD=demo123456! +fi + server_check "cbs " 10000 "/healthcheck" server_check "consul " 8500 "/v1/catalog/service/agent" server_check "DR sim " 3906 "/" @@ -125,6 +168,11 @@ sftp_server_check "SFTP server 1" 1023 sftp_server_check "SFTP server 2" 1024 sftp_server_check "SFTP server 3" 1025 sftp_server_check "SFTP server 4" 1026 +basic_auth_server_check "HTTP server 0" 81 +basic_auth_server_check "HTTP server 1" 82 +basic_auth_server_check "HTTP server 2" 83 +basic_auth_server_check "HTTP server 3" 84 +basic_auth_server_check "HTTP server 4" 85 echo "" @@ -150,10 +198,6 @@ if [ -z "$FTP_FILE_PREFIXES" ] FTP_FILE_PREFIXES="A" fi -if [ -z "$NUM_FTP_SERVERS" ] - then - NUM_FTP_SERVERS=1 -fi if [ $FTP_TYPE = "ALL" ] || [ $FTP_TYPE = "SFTP" ]; then @@ -176,4 +220,29 @@ if [ $FTP_TYPE = "ALL" ] || [ $FTP_TYPE = "FTPES" ]; then let p=p+1 done fi + +#Populate the http server with files. Note some common variables with ftp files! +if [ -z "$NUM_HTTPFILES" ] + then + NUM_HTTPFILES=200 +fi +if [ -z "$HTTP_TYPE" ] + then + HTTP_TYPE="ALL" +fi +if [ -z "$HTTP_FILE_PREFIXES" ] + then + HTTP_FILE_PREFIXES="A" +fi + +if [ $HTTP_TYPE = "ALL" ] || [ $HTTP_TYPE = "HTTP" ]; then + echo "Creating files for HTTP server, may take time...." + p=0 + while [ $p -lt $NUM_HTTP_SERVERS ]; do + docker cp setup-http-files-for-image.sh ${HTTP_SIM[$p]}:/tmp/setup-http-files-for-image.sh + #Double slash needed for docker on win... + docker exec -w //usr//local//apache2//htdocs ${HTTP_SIM[$p]} //tmp/setup-http-files-for-image.sh $NUM_HTTPFILES $NUM_PNFS $FILE_SIZE $HTTP_FILE_PREFIXES $NUM_HTTP_SERVERS $p #>/dev/null 2>&1 + let p=p+1 + done +fi echo "Done: All simulators started and configured" |