diff options
Diffstat (limited to 'test/mocks')
54 files changed, 582 insertions, 582 deletions
diff --git a/test/mocks/datafilecollector-testharness/auto-test/25h_backlog_1MB_FTPS.sh b/test/mocks/datafilecollector-testharness/auto-test/25h_backlog_1MB_FTPS.sh index 76afc8c7c..dd1daea54 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/25h_backlog_1MB_FTPS.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/25h_backlog_1MB_FTPS.sh @@ -31,7 +31,6 @@ start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/25h_backlog_1MB_SFTP.sh b/test/mocks/datafilecollector-testharness/auto-test/25h_backlog_1MB_SFTP.sh index 2722182d9..15852057f 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/25h_backlog_1MB_SFTP.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/25h_backlog_1MB_SFTP.sh @@ -31,7 +31,6 @@ start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/AutoTestTest.sh b/test/mocks/datafilecollector-testharness/auto-test/AutoTestTest.sh index 462a7dd37..7c7d3543f 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/AutoTestTest.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/AutoTestTest.sh @@ -32,19 +32,14 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed1_PM_feed2_CTR.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed1_2_3_4_5.json" consul_config_app 1 "../simulator-group/consul/c13_feed2_CTR.json" -consul_config_dmaap 1 "../simulator-group/consul/dmaap_feed1_2_3_4_5.json" consul_config_app 2 "../simulator-group/consul/c14_feed3_LOG.json" -consul_config_dmaap 2 "../simulator-group/consul/dmaap_feed1_2_3_4_5.json" consul_config_app 3 "../simulator-group/consul/c15_feed1_PM_feed4_TEST.json" -consul_config_dmaap 3 "../simulator-group/consul/dmaap_feed1_2_3_4_5.json" consul_config_app 2 "../simulator-group/consul/c16_feed4_TEST_feed5_TEMP.json" -consul_config_dmaap 4 "../simulator-group/consul/dmaap_feed1_2_3_4_5.json" mr_print "" @@ -432,31 +427,34 @@ start_dfc 2 start_dfc 3 start_dfc 4 -dr_equal ctr_published_files 1 60 +dr_greater ctr_published_files 1 60 sleep_wait 30 -dr_equal ctr_published_files 1 - mr_greater ctr_requests 1 -mr_equal ctr_events 1 -mr_equal ctr_unique_files 1 -mr_equal ctr_unique_PNFs 1 +mr_greater ctr_events 1 +mr_greater ctr_unique_files 1 +mr_greater ctr_unique_PNFs 1 -dr_equal ctr_publish_query 1 +dr_greater ctr_publish_query 1 dr_equal ctr_publish_query_published 0 -dr_equal ctr_publish_query_not_published 1 -dr_equal ctr_publish_req 1 -dr_equal ctr_publish_req_redirect 1 +dr_greater ctr_publish_query_not_published 1 +dr_greater ctr_publish_req 1 +dr_greater ctr_publish_req_redirect 1 dr_equal ctr_publish_req_published 0 -dr_equal ctr_published_files 1 +dr_greater ctr_published_files 1 dr_equal ctr_double_publish 0 -drr_equal ctr_publish_requests 1 -drr_equal ctr_publish_responses 1 +drr_greater ctr_publish_requests 1 +drr_greater ctr_publish_responses 1 + +drr_greater dwl_volume 1000000 + + +####There is a risk of double publishing when running multiple DFCs. +####The related counters ctr_publish_query_published and ctr_double_publish may be non-zero. -drr_equal dwl_volume 1000000 check_dfc_logs diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC1.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC1.sh index 88f63c378..f1d6f093c 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC1.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC1.sh @@ -29,13 +29,12 @@ log_sim_settings start_simulators -consul_config_app 0 "../simulator-group/consul/c12_feed2_PM_secureMR.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" +consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -mr_secure_equal ctr_requests 0 60 +mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 -mr_secure_print tc_info +mr_print tc_info dr_print tc_info drr_print tc_info @@ -47,11 +46,11 @@ sleep_wait 30 dr_equal ctr_published_files 1 -mr_secure_greater ctr_requests 1 +mr_greater ctr_requests 1 -mr_secure_equal ctr_events 1 -mr_secure_equal ctr_unique_files 1 -mr_secure_equal ctr_unique_PNFs 1 +mr_equal ctr_events 1 +mr_equal ctr_unique_files 1 +mr_equal ctr_unique_PNFs 1 dr_equal ctr_publish_query 1 dr_equal ctr_publish_query_bad_file_prefix 0 @@ -76,4 +75,4 @@ check_dfc_logs store_logs END -print_result
\ No newline at end of file +print_result diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC10.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC10.sh index 918906fa7..c162a2a16 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC10.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC10.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC100.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC100.sh index 77477a82a..9d9665bb2 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC100.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC100.sh @@ -1,6 +1,6 @@ #!/bin/bash -TC_ONELINE_DESCR="100 event with 1 1MB file in each evewnt from one PNF in one event using SFTP with feed reconfigure" +TC_ONELINE_DESCR="100 events with 1 1MB file in each event from one PNF using SFTP with feed reconfigure" . ../common/testcase_common.sh $1 $2 @@ -13,10 +13,10 @@ export MR_GROUPS="OpenDcae-c12:PM_MEAS_FILES" export MR_FILE_PREFIX_MAPPING="PM_MEAS_FILES:A" export DR_TC="--tc normal" -export DR_FEEDS="2:A" +export DR_FEEDS="1:A,2:A" export DR_REDIR_TC="--tc normal" -export DR_REDIR_FEEDS="2:A" +export DR_REDIR_FEEDS="1:A,2:A" export NUM_FTPFILES="200" export NUM_PNFS="1" @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 @@ -44,26 +43,11 @@ drr_contain_str feeds "2:A" start_dfc 0 -dr_equal ctr_published_files 50 900 - - -export DR_TC="--tc normal" -export DR_FEEDS="1:A" - -export DR_REDIR_TC="--tc normal" -export DR_REDIR_FEEDS="1:A" +dr_equal ctr_published_files 5 900 consul_config_app 0 "../simulator-group/consul/c12_feed1_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed1.json" - -kill_dr -kill_drr - -log_sim_settings - -start_simulators -mr_equal ctr_events 100 900 +mr_equal ctr_events 100 1800 mr_equal ctr_unique_files 100 mr_equal ctr_unique_PNFs 1 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC11.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC11.sh index d6560a3c5..18db3b288 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC11.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC11.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC12.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC12.sh index adafb2fed..a33f37c22 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC12.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC12.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC13.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC13.sh index 12e9ddc88..93e348e12 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC13.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC13.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC14.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC14.sh index 27d51f659..99646b369 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC14.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC14.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC15.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC15.sh index 060f3c364..44238c31d 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC15.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC15.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC2.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC2.sh index e498f42ef..cb2f71a25 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC2.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC2.sh @@ -29,13 +29,12 @@ log_sim_settings start_simulators -consul_config_app 0 "../simulator-group/consul/c12_feed2_PM_secureMR.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" +consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -mr_secure_equal ctr_requests 0 60 +mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 -mr_secure_print tc_info +mr_print tc_info dr_print tc_info drr_print tc_info @@ -47,11 +46,11 @@ sleep_wait 30 dr_equal ctr_published_files 1 -mr_secure_greater ctr_requests 1 +mr_greater ctr_requests 1 -mr_secure_equal ctr_events 1 -mr_secure_equal ctr_unique_files 1 -mr_secure_equal ctr_unique_PNFs 1 +mr_equal ctr_events 1 +mr_equal ctr_unique_files 1 +mr_equal ctr_unique_PNFs 1 dr_equal ctr_publish_query 1 dr_equal ctr_publish_query_bad_file_prefix 0 @@ -76,4 +75,4 @@ check_dfc_logs store_logs END -print_result
\ No newline at end of file +print_result diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC20.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC20.sh index a61b1a429..9eef5ae95 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC20.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC20.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC200.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC200.sh index 8060ddfc2..0b1828966 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC200.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC200.sh @@ -31,7 +31,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC21.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC21.sh index fb18d7644..df9b57d3f 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC21.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC21.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC210.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC210.sh index 46fff9d5a..5291b6815 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC210.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC210.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC220.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC220.sh index 08222d2ec..2eb9abc97 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC220.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC220.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 30 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC3.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC3.sh index 1f7a8e9a1..84db1d8c8 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC3.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC3.sh @@ -29,13 +29,12 @@ log_sim_settings start_simulators -consul_config_app 0 "../simulator-group/consul/c12_feed2_PM_secureMR.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" +consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -mr_secure_equal ctr_requests 0 60 +mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 -mr_secure_print tc_info +mr_print tc_info dr_print tc_info drr_print tc_info @@ -47,11 +46,11 @@ sleep_wait 30 dr_equal ctr_published_files 1 -mr_secure_greater ctr_requests 1 +mr_greater ctr_requests 1 -mr_secure_equal ctr_events 1 -mr_secure_equal ctr_unique_files 1 -mr_secure_equal ctr_unique_PNFs 1 +mr_equal ctr_events 1 +mr_equal ctr_unique_files 1 +mr_equal ctr_unique_PNFs 1 dr_equal ctr_publish_query 1 dr_equal ctr_publish_query_bad_file_prefix 0 @@ -77,4 +76,4 @@ check_dfc_logs store_logs END -print_result
\ No newline at end of file +print_result diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC30.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC30.sh index e29d948bf..380d3ed0f 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC30.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC30.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC31.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC31.sh index ad71d3043..2776399c3 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC31.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC31.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC32.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC32.sh index 3d608351a..b1ab48224 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC32.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC32.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC33.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC33.sh index 1b1ae452c..338a20da0 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC33.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC33.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC4.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC4.sh index bb3d2ba70..93dd69c0c 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC4.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC4.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed1_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed1.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC40.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC40.sh index 25e68e725..f7b67d51b 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC40.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC40.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC5.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC5.sh index f5996213b..3de577eee 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC5.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC5.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed1_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed1.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC50.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC50.sh index 7e6288b84..6efa32244 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC50.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC50.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC6.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC6.sh index 10de5c7fe..fd3977348 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC6.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC6.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed1_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed1.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC60.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC60.sh index 8be18cbcd..cc3839bec 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC60.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC60.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC61.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC61.sh index 861e03532..f16c442f2 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC61.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC61.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC70.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC70.sh index 0c21b3e01..0a5b3f1d4 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC70.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC70.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC71.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC71.sh index c69958d72..add145492 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC71.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC71.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC80.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC80.sh index 134c87c73..960ea9679 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC80.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC80.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_CTR_feed3_LOG_TEMP.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed1_2_3_4.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC81.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC81.sh index 442bdce88..9734d9714 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC81.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC81.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed3_PM_CTR.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed3.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/FTC90.sh b/test/mocks/datafilecollector-testharness/auto-test/FTC90.sh index 780fbf724..50da063a4 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/FTC90.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/FTC90.sh @@ -30,9 +30,7 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed1_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed1.json" consul_config_app 1 "../simulator-group/consul/c13_feed2_CTR.json" -consul_config_dmaap 1 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/MaxFiles1MB_FTPS_24h.sh b/test/mocks/datafilecollector-testharness/auto-test/MaxFiles1MB_FTPS_24h.sh index 4c3fca0d7..08d4d9ea2 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/MaxFiles1MB_FTPS_24h.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/MaxFiles1MB_FTPS_24h.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/MaxFiles1MB_SFTP_24h.sh b/test/mocks/datafilecollector-testharness/auto-test/MaxFiles1MB_SFTP_24h.sh index feae9d5ae..1bc88ef95 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/MaxFiles1MB_SFTP_24h.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/MaxFiles1MB_SFTP_24h.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 diff --git a/test/mocks/datafilecollector-testharness/auto-test/README.md b/test/mocks/datafilecollector-testharness/auto-test/README.md index a94076f58..b73067dee 100644 --- a/test/mocks/datafilecollector-testharness/auto-test/README.md +++ b/test/mocks/datafilecollector-testharness/auto-test/README.md @@ -1,6 +1,8 @@ ## Running automated test case and test suites Test cases run a single test case and test suites run one or more test cases in a sequence. +The test cases and test suites are possible to run on both Ubuntu and Mac-OS. + ##Overall structure and setup Test cases and test suites are written as bash scripts which call predefined functions in two other bash scripts located in ../common dir. @@ -69,201 +71,8 @@ print_result ``` ----------------------------------------------------------- -The ../common/testcase_common.sh contains all functions needed for the test case file. - -The following is a list of the available functions in a test case file. Please see some of the defined test case for examples. - -**log_sim_settings**</br> -Print the env variables needed for the simulators and their setup - -**clean_containers**</br> -Stop and remove all containers including dfc apps and simulators - -**start_simulators**</br> -Start all simulators in the simulator group - -**start_dfc <dfc-instance-id>**</br> -Start the dfc application. The arg shall be an integer from 0 to 5 reprenting the dfc instance to start. DFC app will get a name like 'dfc_app0' to 'dfc_app4'. - -**kill_dfc <dfc-instance-id> **</br> -Stop and remove the dfc app container with the instance id. - -**consul_config_app <dfc-instance-id> <json-file-path>**</br> -Configure consul with json file with app config for a dfc instance using the dfc instance id and the json file. - -**consul_config_dmaap <dfc-instance-id> <json-file-path>**</br> -Configure consul with json file with dmaap config for a dfc instance using the dfc instance id and the json file. +The ../common/testcase_common.sh contains all functions needed for the test case file. See the README.md file in the ../common dir for a description of all available functions. -**kill_dr**</br> -Stop and remove the DR simulator container - -**kill_drr**</br> -Stop and remove the DR redir simulator container - -**kill_mr**</br> -Stop and remove the MR simulator container - -**kill_sftp <sftp-instance-id>**</br> -Stop and remove a SFTP container with the supplied instance id (0-5). - -**stop_sftp <sftp-instance-id>**</br> -Stop a SFTP container with the supplied instance id (0-5). - -**start_sftp <sftp-instance-id>**</br> -Start a previously stopped SFTP container with the supplied instance id (0-5). - -**kill_ftps <ftps-instance-id>**</br> -Stop and remove a FTPS container with the supplied instance id (0-5). - -**stop_ftps <ftps-instance-id>**</br> -Stop a FTPS container with the supplied instance id (0-5). - -**start_ftps <ftps-instance-id>**</br> -Start a previously stopped FTPS container with the supplied instance id (0-5). - -**mr_print <vaiable-name>**</br> -Print a variable value from the MR simulator. - -**dr_print <vaiable-name>**</br> -Print a varialle value from the DR simulator. - -**drr_print <vaiable-name>**</br> -Print a variable value from the DR redir simulator. - -**dfc_print <dfc-instance-id> <vaiable-name>**</br> -Print a variable value from an dfc instance with the supplied instance id (0-5). - -**mr_read <vaiable-name>**</br> -Read a variable value from MR sim and send to stdout - -**dr_read <vaiable-name>**</br> -Read a variable value from DR sim and send to stdout - -**drr_read <vaiable-name>**</br> -Read a variable value from DR redir sim and send to stdout - -**sleep_wait <sleep-time-in-sec>**</br> -Sleep for a number of seconds - -**sleep_heartbeat <sleep-time-in-sec>**</br> -Sleep for a number of seconds and prints dfc heartbeat output every 30 sec - -**mr_equal <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the MR simulator is equal to a target value and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is -equal to the targer or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value becomes equal to the target -value or not. - -**mr_greater <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the MR simulator is greater than a target value and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is -greater the target or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value is greater than the target -value or not. - -**mr_less <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the MR simulator is less than a target value and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is -less than the target or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value is less than the target -value or not. - -**mr_contain_str <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the MR simulator contains a substring target and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable contains -the target substring or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value contains the target -substring or not. - -**dr_equal <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the DR simulator is equal to a target value and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is -equal to the target or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value becomes equal to the target -value or not. - -**dr_greater <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the DR simulator is greater than a target value and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is -greater the target or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value is greater than the target -value or not. - -**dr_less <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the DR simulator is less than a target value and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is -less than the target or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value is less than the target -value or not. - -**dr_contain_str <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the DR simulator contains a substring target and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable contains -the target substring or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value contains the target -substring or not. - -**drr_equal <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the DR Redir simulator is equal to a target value and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is -equal to the target or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value becomes equal to the target -value or not. - -**drr_greater <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the DR Redir simulator is greater than a target value and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is -greater the target or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value is greater than the target -value or not. - -**drr_less <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the DR Redir simulator is less than a target value and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is -less than the target or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value is less than the target -value or not. - -**drr_contain_str <variable-name> <target-value> [<timeout-in-sec>]**</br> -Tests if a variable value in the DR Redir simulator contains a substring target and and optional timeout. -</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable contains -the target substring or not. -</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds -before setting pass or fail depending on if the variable value contains the target -substring or not. - -**dfc_contain_str <variable-name> <substring-in-quotes>**</br> -Test is a variable in the DFC contains a substring. - -**store_logs <log-prefix>**</br> -Store all dfc app and simulators log to the test case log dir. All logs gets a prefix to -separate logs stored at different steps in the test script. -If logs need to be stored in several locations, use different prefix to easily identify the location -when the logs where taken. - -**check_dfc_log**</br> -Check the dfc application log for WARN and ERR messages and print the count. - -**print_result**</br> -Print the test result. Only once at the very end of the script. - -**print_all**</br> -Print all variables from the simulators and the dfc heartbeat. - -In addition, comment in the file can be added using the normal comment sign in bash '#'. -Comments that shall be visible on the screen as well as in the test case log, use ``echo "<msg>"``. ##Test suite files## A test suite file contains one or more test cases to run in sequence. @@ -297,15 +106,15 @@ suite_complete ``` ----------------------------------------------------------- -The ../common/testsuite_common.sh contains all functions needed for a test suite file. - -The following is a list of the available functions in a test case file. Please see a existing test suite for examples. - -**suite_setup**</br> -Sets up the test suite and print out a heading. +The ../common/testsuite_common.sh contains all functions needed for a test suite file. See the README.md file in the ../common dir for a description of all available functions. -**run_tc <tc-script> <$1 from test suite script> <$2 from test suite script>**</br> -Execute a test case with arg from test suite script +##Known limitations## +When DFC has polled a new event from the MR simulator, DFC starts to check each file whether it has been already published or not. This check is done per file towards the DR simulator. +If the event contains a large amount of files, there is a risk that DFC will flood the DR simulator with requests for these checks. The timeout in DFC for the response is currently 4 sec and the DR simulator may not be able to answer all request within the timeout. +DR simulator is single threaded. This seem to be a problem only for the first polled event. For subsequent events these requests seem to be spread out in time by DFC so the DR simulator can respond in time. +The problem is visible in the DR simulator counters `ctr_publish_query` and`ctr_publish_query_not_published` in the auto-test scripts. They will have a count slightly less (1 to 5) than the actual number of files in the event. The application log in DFC also prints a timeout error for each failed request. +A number of the test script will report failure due to this limitation in the DR simulator. -**suite_complete**</br> -Print out the overall result of the executed test cases.
\ No newline at end of file +The FTP servers may deny connection when too many file download requests are made in a short time from DFC. +This is visible in the DFC application log as WARNINGs for failed downloads. However, DFC always retry the failed download a number of times to +minimize the risk of giving up download completely for these files.
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/auto-test/Stability1MB_FTPS_72h.sh b/test/mocks/datafilecollector-testharness/auto-test/Stability1MB_FTPS_72h.sh index f221ac5f5..6e3368518 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/Stability1MB_FTPS_72h.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/Stability1MB_FTPS_72h.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 @@ -62,7 +61,7 @@ TARGET_EVENTS=$((TARGET_FILES-70000+700)) #First event from a PNF is 100 new fi TARGET_VOLUME=$((TARGET_FILES*1000000)) #Maximum number of configured FTP files, if DFC reach this then the NUM_FTPSFILES need to be increased. -MAX_FILES=$((NUM_FTPFILE*NUM_PNFS)) +MAX_FILES=$((NUM_FTPFILES*NUM_PNFS)) #Wait remaining time upto 15 min for DFC to download all consumed events sleep_wait 870 diff --git a/test/mocks/datafilecollector-testharness/auto-test/Stability1MB_SFTP_72h.sh b/test/mocks/datafilecollector-testharness/auto-test/Stability1MB_SFTP_72h.sh index bc0b2efb5..baafc906d 100755 --- a/test/mocks/datafilecollector-testharness/auto-test/Stability1MB_SFTP_72h.sh +++ b/test/mocks/datafilecollector-testharness/auto-test/Stability1MB_SFTP_72h.sh @@ -30,7 +30,6 @@ log_sim_settings start_simulators consul_config_app 0 "../simulator-group/consul/c12_feed2_PM.json" -consul_config_dmaap 0 "../simulator-group/consul/dmaap_feed2.json" mr_equal ctr_requests 0 60 dr_equal ctr_published_files 0 60 @@ -62,7 +61,7 @@ TARGET_EVENTS=$((TARGET_FILES-70000+700)) #First event from a PNF is 100 new TARGET_VOLUME=$((TARGET_FILES*1000000)) #Maximum number of configured FTP files, if DFC reach this then the NUM_FTPSFILES need to be increased. -MAX_FILES=$((NUM_FTPFILE*NUM_PNFS)) +MAX_FILES=$((NUM_FTPFILES*NUM_PNFS)) #Wait remaining time upto 15 min for DFC to download all consumed events sleep_wait 870 diff --git a/test/mocks/datafilecollector-testharness/common/README.md b/test/mocks/datafilecollector-testharness/common/README.md index b5cda65ef..bcd345739 100644 --- a/test/mocks/datafilecollector-testharness/common/README.md +++ b/test/mocks/datafilecollector-testharness/common/README.md @@ -7,4 +7,214 @@ Common env variables for test in the auto-test dir. Used by the auto test cases/ Common functions for auto test cases in the auto-test dir. A subset of the functions could be used in other test scripts as well. **testsuite_common.sh**</br> -Common functions for auto test suites in the auto-test dir.
\ No newline at end of file +Common functions for auto test suites in the auto-test dir. + +##Descriptions of functions in testcase_common.sh + +The following is a list of the available functions in a test case file. Please see some of the defined test cases for examples. + +**log_sim_settings**</br> +Print the env variables needed for the simulators and their setup + +**clean_containers**</br> +Stop and remove all containers including dfc apps and simulators + +**start_simulators**</br> +Start all simulators in the simulator group + +**start_dfc <dfc-instance-id>**</br> +Start the dfc application. The arg shall be an integer from 0 to 5 representing the dfc instance to start. DFC app will get a name like 'dfc_app0' to 'dfc_app4'. + +**kill_dfc <dfc-instance-id> **</br> +Stop and remove the dfc app container with the instance id. + +**consul_config_app <dfc-instance-id> <json-file-path>**</br> +Configure consul with json file with app config for a dfc instance using the dfc instance id and the json file. + +**consul_config_dmaap <dfc-instance-id> <json-file-path>**</br> +Configure consul with json file with dmaap config for a dfc instance using the dfc instance id and the json file. + +**kill_dr**</br> +Stop and remove the DR simulator container + +**kill_drr**</br> +Stop and remove the DR redir simulator container + +**kill_mr**</br> +Stop and remove the MR simulator container + +**kill_sftp <sftp-instance-id>**</br> +Stop and remove a SFTP container with the supplied instance id (0-5). + +**stop_sftp <sftp-instance-id>**</br> +Stop a SFTP container with the supplied instance id (0-5). + +**start_sftp <sftp-instance-id>**</br> +Start a previously stopped SFTP container with the supplied instance id (0-5). + +**kill_ftps <ftps-instance-id>**</br> +Stop and remove a FTPS container with the supplied instance id (0-5). + +**stop_ftps <ftps-instance-id>**</br> +Stop a FTPS container with the supplied instance id (0-5). + +**start_ftps <ftps-instance-id>**</br> +Start a previously stopped FTPS container with the supplied instance id (0-5). + +**mr_print <vaiable-name>**</br> +Print a variable value from the MR simulator. + +**dr_print <vaiable-name>**</br> +Print a variable value from the DR simulator. + +**drr_print <vaiable-name>**</br> +Print a variable value from the DR redir simulator. + +**dfc_print <dfc-instance-id> <vaiable-name>**</br> +Print a variable value from an dfc instance with the supplied instance id (0-5). + +**mr_read <vaiable-name>**</br> +Read a variable value from MR sim and send to stdout + +**dr_read <vaiable-name>**</br> +Read a variable value from DR sim and send to stdout + +**drr_read <vaiable-name>**</br> +Read a variable value from DR redir sim and send to stdout + +**sleep_wait <sleep-time-in-sec>**</br> +Sleep for a number of seconds + +**sleep_heartbeat <sleep-time-in-sec>**</br> +Sleep for a number of seconds and prints dfc heartbeat output every 30 sec + +**mr_equal <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the MR simulator is equal to a target value and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is +equal to the targer or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value becomes equal to the target +value or not. + +**mr_greater <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the MR simulator is greater than a target value and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is +greater the target or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value is greater than the target +value or not. + +**mr_less <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the MR simulator is less than a target value and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is +less than the target or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value is less than the target +value or not. + +**mr_contain_str <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the MR simulator contains a substring target and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable contains +the target substring or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value contains the target +substring or not. + +**dr_equal <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the DR simulator is equal to a target value and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is +equal to the target or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value becomes equal to the target +value or not. + +**dr_greater <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the DR simulator is greater than a target value and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is +greater the target or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value is greater than the target +value or not. + +**dr_less <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the DR simulator is less than a target value and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is +less than the target or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value is less than the target +value or not. + +**dr_contain_str <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the DR simulator contains a substring target and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable contains +the target substring or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value contains the target +substring or not. + +**drr_equal <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the DR Redir simulator is equal to a target value and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is +equal to the target or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value becomes equal to the target +value or not. + +**drr_greater <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the DR Redir simulator is greater than a target value and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is +greater the target or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value is greater than the target +value or not. + +**drr_less <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the DR Redir simulator is less than a target value and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable is +less than the target or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value is less than the target +value or not. + +**drr_contain_str <variable-name> <target-value> [<timeout-in-sec>]**</br> +Tests if a variable value in the DR Redir simulator contains a substring target and an optional timeout. +</br>Arg: ``<variable-name> <target-value>`` - This test set pass or fail depending on if the variable contains +the target substring or not. +</br>Arg: ``<variable-name> <target-value> <timeout-in-sec>`` - This test waits up to the timeout seconds +before setting pass or fail depending on if the variable value contains the target +substring or not. + +**dfc_contain_str <variable-name> <substring-in-quotes>**</br> +Test if a variable in the DFC contains a substring. + +**store_logs <log-prefix>**</br> +Store all dfc app and simulators log to the test case log dir. All logs get a prefix to +separate logs stored at different steps in the test script. +If logs need to be stored in several locations, use different prefix to easily identify the location +when the logs where taken. + +**check_dfc_log**</br> +Check the dfc application log for WARN and ERR messages and print the count. + +**print_result**</br> +Print the test result. Only once at the very end of the script. + +**print_all**</br> +Print all variables from the simulators and the dfc heartbeat. + +In addition, comment in the file can be added using the normal comment sign in bash '#'. +Comments that shall be visible on the screen as well as in the test case log, use ``echo "<msg>"``. + + +##Descriptions of functions in testsuite_common.sh + +The following is a list of the available functions in a test suite file. Please see a existing test suite for examples. + +**suite_setup**</br> +Sets up the test suite and print out a heading. + +**run_tc <tc-script> <$1 from test suite script> <$2 from test suite script>**</br> +Execute a test case with arg from test suite script + +**suite_complete**</br> +Print out the overall result of the executed test cases.
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/common/test_env.sh b/test/mocks/datafilecollector-testharness/common/test_env.sh index 7ba35ef27..1a97ffc73 100644 --- a/test/mocks/datafilecollector-testharness/common/test_env.sh +++ b/test/mocks/datafilecollector-testharness/common/test_env.sh @@ -1,6 +1,6 @@ #!/bin/bash -# This env variable is only needed if the auto test scripts tests are executed in a different folder than 'auto-test' in the integration repo +# This env variable is only needed if the auto test scripts tests are executed in a different folder than 'auto-test' in the integration repo # Change '<local-path>' to your path to the integration repo. In addition to the auto-test, the 'common' dir is needed if not executed in the # integration repo. # @@ -18,33 +18,41 @@ export DFC_LOCAL_IMAGE=onap/org.onap.dcaegen2.collectors.datafile.datafile-app-s # Common env var for auto-test. -DFC_PORT=8100 -DFC_PORT_SECURE=8433 -DFC_LOGPATH="/var/log/ONAP/application.log" -DOCKER_SIM_NWNAME="dfcnet" -CONSUL_HOST="consul-server" -CONSUL_PORT=8500 -CONFIG_BINDING_SERVICE="config-binding-service" -MR_PORT=2222 -MR_PORT_SECURE=2223 -DR_PORT=3906 -DR_PORT_SECURE=3907 -DRR_PORT=3908 -DRR_PORT_SECURE=3909 -DFC_APP_BASE="dfc_app" -DFC_MAX_NUM=5 -DFC_MAX_IDX=$(($DFC_MAX_NUM - 1)) -SFTP_BASE="dfc_sftp-server" -FTPS_BASE="dfc_ftpes-server-vsftpd" -FTP_MAX_NUM=5 -FTP_MAX_IDX=$(($FTP_MAX_NUM - 1)) +DFC_PORT=8100 #Up to five dfc apps can be used, dfc_app0 will be mapped to 8100 on local machine for http, dfc_app1 mapped to 8101 etc +DFC_PORT_SECURE=8433 #Up to five dfc apps can be used, dfc_app0 will be mapped to 8433 on local machine for hhtps, dfc_app1 mapped to 8434 etc +DFC_LOGPATH="/var/log/ONAP/application.log" #Path the application log in the dfc container +DOCKER_SIM_NWNAME="dfcnet" #Name of docker private network +CONSUL_HOST="consul-server" #Host name of consul +CONSUL_PORT=8500 #Port number of consul +CONFIG_BINDING_SERVICE="config-binding-service" #Host name of CBS +MR_PORT=2222 #MR simulator port number http +DR_PORT=3906 #DR simulator port number http +DR_PORT_SECURE=3907 #DR simulator port number for https +DRR_PORT=3908 #DR Redirect simulator port number for http +DRR_PORT_SECURE=3909 #DR Redirect simulator port number for https +DFC_APP_BASE="dfc_app" #Base name of the dfc containers. Instance 0 will be named dfc_app0, instance 1 will named dfc_app1 etc +DFC_MAX_NUM=5 #Max number of dfc containers to run in paralell in auto test +DFC_MAX_IDX=$(($DFC_MAX_NUM - 1)) #Max index of the dfc containers +SFTP_BASE="dfc_sftp-server" #Base name of the dfc_sftp-server containers. Instance 0 will be named dfc_sftp-server0, instance 1 will named dfc_sftp-server1 etc +FTPS_BASE="dfc_ftpes-server-vsftpd" #Base name of the dfc_ftpes-server-vsftpd containers. Instance 0 will be named dfc_ftpes-server-vsftpd0, instance 1 will named dfc_ftpes-server-vsftpd1 etc +FTP_MAX_NUM=5 #Max number of sftp and ftps containers to run in paralell in auto test +FTP_MAX_IDX=$(($FTP_MAX_NUM - 1)) #Max index of sftp and ftps containers + +#List of sftp server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc and the simulators in a private docker network SFTP_SIMS_CONTAINER="sftp-server0:22,sftp-server1:22,sftp-server2:22,sftp-server3:22,sftp-server4:22" + +#List of sftp server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc and the simulators in a private docker network FTPS_SIMS_CONTAINER="ftpes-server-vsftpd0:21,ftpes-server-vsftpd1:21,ftpes-server-vsftpd2:21,ftpes-server-vsftpd3:21,ftpes-server-vsftpd4:21" + +#List of sftp server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc as stand along app and the simulators in a private docker network SFTP_SIMS_LOCALHOST="localhost:1022,localhost:1023,localhost:1024,localhost:1025,localhost:1026" + +#List of ftps server name and port number, used by MR sim to produce file urls. Theses server names and ports are used when running dfc as stand along app and the simulators in a private docker network FTPS_SIMS_LOCALHOST="localhost:1032,localhost:1033,localhost:1034,localhost:1035,localhost:1036" export SFTP_SIMS=$SFTP_SIMS_CONTAINER #This env will be set to SFTP_SIMS_LOCALHOST if auto test is executed with 'manual-app' export FTPS_SIMS=$FTPS_SIMS_CONTAINER #This env will be set to FTPS_SIMS_LOCALHOST if auto test is executed with 'manual-app' -export DR_REDIR_SIM="drsim_redir" #This env will be set to 'localhost' if auto test is executed with 'manual-app' +#Host name of the DR redirect simulator +export DR_REDIR_SIM="drsim_redir" #This env will be set to 'localhost' if auto test is executed with arg 'manual-app' diff --git a/test/mocks/datafilecollector-testharness/common/testcase_common.sh b/test/mocks/datafilecollector-testharness/common/testcase_common.sh index deed26dfd..a1e092157 100755 --- a/test/mocks/datafilecollector-testharness/common/testcase_common.sh +++ b/test/mocks/datafilecollector-testharness/common/testcase_common.sh @@ -557,35 +557,28 @@ start_dfc() { fi } -# Configure consul with dfc config, args <app|dmaap> <dfc-instance-id> <json-file-path> +# Configure consul with dfc config, args <dfc-instance-id> <json-file-path> # Not intended to be called directly by test scripts. __consul_config() { - if [ $# != 3 ]; then - __print_err "need three args, <app|dmaap> <dfc-instance-id> <json-file-path>" + if [ $# != 2 ]; then + __print_err "need two args, <dfc-instance-id> <json-file-path>" exit 1 fi - if [ $2 -lt 0 ] || [ $2 -gt $DFC_MAX_IDX ]; then + if [ $1 -lt 0 ] || [ $1 -gt $DFC_MAX_IDX ]; then __print_err "dfc-instance-id should be 0.."$DFC_MAX_IDX exit 1 fi - if ! [ -f $3 ]; then - __print_err "json file does not extis: "$3 + if ! [ -f $2 ]; then + __print_err "json file does not extis: "$2 exit 1 fi - if [ $1 == "app" ]; then - appname=$DFC_APP_BASE$2 - elif [ $1 == "dmaap" ]; then - appname=$DFC_APP_BASE$2":dmaap" - else - __print_err "config type should be 'app' or 'dmaap'" - exit 1 - fi + appname=$DFC_APP_BASE$1 - echo "Configuring consul for " $appname " from " $3 - curl -s http://127.0.0.1:${CONSUL_PORT}/v1/kv/${appname}?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary "@"$3 >/dev/null + echo "Configuring consul for " $appname " from " $2 + curl -s http://127.0.0.1:${CONSUL_PORT}/v1/kv/${appname}?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary "@"$2 >/dev/null } # Configure consul with dfc app config, args <dfc-instance-id> <json-file-path> @@ -593,20 +586,11 @@ consul_config_app() { if [ $START_ARG == "manual-app" ]; then echo "Replacing 'mrsim' with 'localhost' in json app config for consul" sed 's/mrsim/localhost/g' $2 > .tmp_app.json - __consul_config app $1 .tmp_app.json - else - __consul_config app $1 $2 - fi -} - -# Configure consul with dfc dmaap config, args <dfc-instance-id> <json-file-path> -consul_config_dmaap() { - if [ $START_ARG == "manual-app" ]; then echo "Replacing 'drsim' with 'localhost' in json dmaap config for consul" - sed 's/drsim/localhost/g' $2 > .tmp_dmaap.json - __consul_config dmaap $1 .tmp_dmaap.json + sed 's/drsim/localhost/g' .tmp_app.json > .app.json + __consul_config $1 .app.json else - __consul_config dmaap $1 $2 + __consul_config $1 $2 fi } diff --git a/test/mocks/datafilecollector-testharness/dr-sim/README.md b/test/mocks/datafilecollector-testharness/dr-sim/README.md index 9b5b5eb6d..a258ed46d 100644 --- a/test/mocks/datafilecollector-testharness/dr-sim/README.md +++ b/test/mocks/datafilecollector-testharness/dr-sim/README.md @@ -1,59 +1,183 @@ -###Alternative to running python (as described below) on your machine, use the docker files. +###Run DR simulators as docker container 1. Build docker container with ```docker build -t drsim_common:latest .``` 2. Run the container ```docker-compose up``` 3. For specific behavior of of the simulators, add arguments to the `command` entries in the `docker-compose.yml`. For example `command: node dmaapDR.js --tc no_publish` . (No argument will assume '--tc normal'). Run `node dmaapDR.js --printtc` -and `node dmaapDR-redir.js --printtc` for details. - +and `node dmaapDR-redir.js --printtc` for details or see further below for the list of possible arg to the simulator +###Run DR simulators and all other simulators as one group +See the README in the 'simulator-group' dir. +###Run DR simulators from cmd line 1. install nodejs 2. install npm Make sure that you run these commands in the application directory "dr-sim" 3. `npm install express` 4. `npm install argparse` -5. `node dmaapDR.js` #keep it in the foreground, see item 3 in the above list for arg to the simulator -6. `node dmaapDR_redir.js` #keep it in the foreground, see item 3 in the above list for arg to the simulator +5. `node dmaapDR.js` #keep it in the foreground, see below for a list for arg to the simulator +6. `node dmaapDR_redir.js` #keep it in the foreground, see below for a list for arg to the simulator + +###Arg to control the behavior of the simulators + +**DR** + + --tc tc_normal Normal case, query response based on published files. Publish respond with ok/redirect depending on if file is published or not.</br> + --tc tc_none_published Query respond 'ok'. Publish respond with redirect.</br> + --tc tc_all_published Query respond with filename. Publish respond with 'ok'.</br> + --tc tc_10p_no_response 10% % no response for query and publish. Otherwise normal case.</br> + --tc tc_10first_no_response 10 first queries and requests gives no response for query and publish. Otherwise normal case.</br> + --tc tc_100first_no_response 100 first queries and requests gives no response for query and publish. Otherwise normal case.</br> + --tc tc_all_delay_1s All responses delayed 1s (both query and publish).</br> + --tc tc_all_delay_10s All responses delayed 10s (both query and publish).</br> + --tc tc_10p_delay_10s 10% of responses delayed 10s, (both query and publish).</br> + --tc tc_10p_error_response 10% error response for query and publish. Otherwise normal case.</br> + --tc tc_10first_error_response 10 first queries and requests gives no response for query and publish. Otherwise normal case.</br> + --tc tc_100first_error_response 100 first queries and requests gives no response for query and publish. Otherwise normal case.</br> + + +**DR Redirect** + + --tc_normal Normal case, all files publish and DR updated.</br> + --tc_no_publish Ok response but no files published.</br> + --tc_10p_no_response 10% % no response (file not published).</br> + --tc_10first_no_response 10 first requests give no response (files not published).</br> + --tc_100first_no_response 100 first requests give no response (files not published).</br> + --tc_all_delay_1s All responses delayed 1s, normal publish.</br> + --tc_all_delay_10s All responses delayed 10s, normal publish.</br> + --tc_10p_delay_10s 10% of responses delayed 10s, normal publish.</br> + --tc_10p_error_response 10% error response (file not published).</br> + --tc_10first_error_response 10 first requests give error response (file not published).</br> + --tc_100first_error_response 100 first requests give error responses (file not published).</br> + + +###Needed environment + +DR + + DRR_SIM_IP Set to host name of the DR Redirect simulator "drsim_redir" if running the simulators in a docker private network. Otherwise to "localhost" + DR_FEEDS A comma separated list of configured feednames and filetypes. Example "1:A,2:B:C" - Feed 1 for filenames beginning with A and feed2 for filenames beginning with B or C. + +`DRR_SIM_IP` is needed for the redirected publish request to be redirected to the DR redirect server. + +DR Redirect (DRR for short) + DR_SIM_IP Set to host name of the DR simulator "drsim" if running the simulators in a docker private network. Otherwise to "localhost" + DR_REDIR_FEEDS Same contentd as DR_FEEDS for DR. -The dmaapDR_redir server send callback to dmaapDR server to update the list of successfully published files. -As default, the ip for dmaapDR is set to work when running as container (using an ip address from the dfc_net docker network) . When running the servers from command line, set the env variable DR_SIM_IP=localhost +The DR Redirect server send callback to DR server to update the list of successfully published files. +When running as container (using an ip address from the `dfc_net` docker network) the env shall be set to 'drsim'. . When running the servers from command line, set the env variable `DR_SIM_IP=localhost` +###APIs for statistic readout The simulator can be queried for statistics (use curl from cmd line or open in browser, curl used below): DR -`curl localhost:3906/ctr_publish_query` - returns the number of publish queries -`curl localhost:3906/ctr_publish_query_published` - returns the number of responses where the files was published +`curl localhost:3906/` - returns 'ok' + +`curl localhost:3906/tc_info` - returns the tc id + +`curl localhost:3906/execution_time` - returns the execution time in the format mm_ss + +`curl localhost:3906/feeds` - returns the list of configured feeds + +`curl localhost:3906/ctr_publish_query` - returns the number of publish queries for all feeds + +`curl localhost:3906/feeds/ctr_publish_query` -returns a list of number of publish queries in each feed + +`curl localhost:3906/ctr_publish_query/<feed>` - returns the number of publish queries for a feed + +`curl localhost:3906/ctr_publish_query_published` - returns the number of query responses for all feeds where the files were published + +`curl localhost:3906/feeds/ctr_publish_query_published` - returns a list of the number of query responses for each feed where the files were published + +`curl localhost:3906/ctr_publish_query_published/<feed>` - returns the number of query responses for a feed where the files were published + +`curl localhost:3906/ctr_publish_query_not_published` - returns the number of query responses for all feed where the files were not published + +`curl localhost:3906/feeds/ctr_publish_query_not_published` - returns a list of the number of query responses for each feed where the files were not published + +`curl localhost:3906/ctr_publish_query_not_published/<feed>` - returns the number of query responses for a feed where the files were not published + +`curl localhost:3906/ctr_publish_req` - returns the number of publish requests for all feeds + +`curl localhost:3906/feeds/ctr_publish_req` - returns a list of the number of publish requests for each feed + +`curl localhost:3906/ctr_publish_req/<feed>` - returns the number of publish requests for feed + +`curl localhost:3906/ctr_publish_req_redirect` - returns the number of publish responses with redirect for all feeds + +`curl localhost:3906/feeds/ctr_publish_req_redirect` - returns a list of the number of publish responses with redirect for each feed + +`curl localhost:3906/ctr_publish_req_redirect/<feed>` - returns the number of publish responses with redirect for a feed + +`curl localhost:3906/ctr_publish_req_published` - returns the number of publish responses where files have been published (no redirect) for all feeds + +`curl localhost:3906/feeds/ctr_publish_req_published` - returns a list of the number of publish responses where files have been published (no redirect) for each feeds + +`curl localhost:3906/ctr_publish_req_published/<feed>` - returns the number of publish responses where files have been published (no redirect) for a feed + +`curl localhost:3906/ctr_published_files` - returns the number of published files for all feeds + +`curl localhost:3906/feeds/ctr_published_files` - returns a list of the number of published files for each feed + +`curl localhost:3906/ctr_published_files/<feed>` - returns the number of published files for a feed + +`curl localhost:3906/ctr_double_publish` - returns the number of double published files for all feeds + +`curl localhost:3906/feeds/ctr_double_publish` - returns a list of the number of double published files for each feed + +`curl localhost:3906/ctr_double_publish/<feed>` - returns a list of the number of double published files for a feed + +`curl localhost:3906/ctr_publish_query_bad_file_prefix` - returns the number of publish queries with bad file prefix for all feeds + +`curl localhost:3906/feeds/ctr_publish_query_bad_file_prefix` - returns a list of the number of publish queries with bad file prefix for each feed + +`curl localhost:3906/ctr_publish_query_bad_file_prefix/<feed>` - returns a list of the number of publish queries with bad file prefix for a feed + + +DR Redirect + + +`curl localhost:3908/` - returns 'ok' + +`curl localhost:3908/tc_info` - returns the tc id + +`curl localhost:3908/execution_time` - returns the execution time in the format mm:ss + +`curl localhost:3908/feeds` - returns the list of configured feeds + +`curl localhost:3908/speed` - returns the speed in published files per second + +`curl localhost:3908/ctr_publish_requests` - returns the number of publish requests for all feeds + +`curl localhost:3908/feeds/ctr_publish_requests` - returns a list of the number of publish requests for each feed -`curl localhost:3906/ctr_publish_query_not_published` - returns the number of responses where the file was not published +`curl localhost:3908/ctr_publish_requests/<feed>` - returns the number of publish requests for a feed -`curl localhost:3906/ctr_publish_req` - returns the number of publish requests +`curl localhost:3908/ctr_publish_requests_bad_file_prefix` - returns the number of publish requests with bad file prefix for all feeds -`curl localhost:3906/ctr_publish_req_redirect` - returns the number of publish responses with redirect +`curl localhost:3908/feeds/ctr_publish_requests_bad_file_prefix` - returns a list of the number of publish requests with bad file prefix for each feed -`curl localhost:3906/ctr_publish_req_published` - returns the number of publish responses without redirect +`curl localhost:3908/ctr_publish_requests_bad_file_prefix/<feed>` - returns the number of publish requests with bad file prefix for a feed -`curl localhost:3906/ctr_published_files` - returns the number of unique published files +`curl localhost:3908/ctr_publish_responses` - returns the number of publish responses for all feeds -`curl localhost:3906/tc_info` - returns the tc name (argument on the command line) +`curl localhost:3908/feeds/ctr_publish_responses` - returns a list of the number of publish responses for each feed -`curl localhost:3906/execution_time` - returns the execution times in mm:ss +`curl localhost:3908/ctr_publish_responses/<feed>` - returns the number of publish responses for a feed -`curl localhost:3906/ctr_double_publish` - returns the number of double published files +`curl localhost:3908/time_lastpublish` - returns the time of the last successful publish in the format mm:ss for any feed +`curl localhost:3908/feeds/time_lastpublish` - returns a list of the time of the last successful publish in the format mm:ss for each feed -DR REDIR +`curl localhost:3908/time_lastpublish/<feed>` - returns the time of the last successful publish in the format mm:ss for a feed -`curl localhost:3908/ctr_publish_requests` - returns the number of publish queries +`curl localhost:3908/dwl_volume` - returns the number of bytes of published files for all feeds -`curl localhost:3908/ctr_publish_responses` - returns the number of publish responses +`curl localhost:3908/feeds/dwl_volume` - returns a list of the number of bytes of the published files for each feed -`curl localhost:3908/tc_info` - returns the tc name (argument on the command line) +`curl localhost:3908/dwl_volume/<feed>` - returns the number of bytes of the published files for a feed -`curl localhost:3908/execution_time` - returns the execution times in mm:ss -`curl localhost:3908/time_lastpublish` - returns the time (mm:ss) for the latest publish -`curl localhost:3908/dwl_volume` - returns the total received data volume of file data diff --git a/test/mocks/datafilecollector-testharness/mr-sim/README.md b/test/mocks/datafilecollector-testharness/mr-sim/README.md index 5e6ac6d88..d3ca91c87 100644 --- a/test/mocks/datafilecollector-testharness/mr-sim/README.md +++ b/test/mocks/datafilecollector-testharness/mr-sim/README.md @@ -26,10 +26,15 @@ All other configuration is done via envrionment variables. The simulator listens to port 2222. The following envrionment vaiables are used: + **FTPS_SIMS** - A comma-separated list of hostname:port for the FTP servers to generate ftps file urls for. If not set MR sim will assume 'localhost:21'. Minimum 1 and maximum 5 host-port pairs can be given. + **SFTP_SIMS** - A comma-separated list of hostname:port for the FTP servers to generate sftp file urls for. If not set MR sim will assume 'localhost:1022'. Minimum 1 and maximum 5 host-port pairs can be given. + **NUM_FTP_SERVERS** - Number of FTP servers to use out of those specified in the envrioment variables above. The number shall be in the range 1-5. + **MR_GROUPS** - A comma-separated list of consummer-group:changeId[:changeId]*. Defines which change identifier that should be used for each consumer gropu. If not set the MR-sim will assume 'OpenDcae-c12:PM_MEAS_FILES'. + **MR_FILE_PREFIX_MAPPING** - A comma-separated list of changeId:filePrefix. Defines which file prefix to use for each change identifier, needed to distinguish files for each change identifiers. If not set the MR-sim will assume 'PM_MEAS_FILES:A @@ -39,77 +44,78 @@ The simulator can be queried for statistics and started/stopped (use curl from `curl localhost:2222` - Just returns 'Hello World'. -`curl localhost:2222/groups` - Return a comma-separated list of configured consumer groups.. +`curl localhost:2222/tc_info` - returns the tc string (as given on the cmd line) -`curl localhost:2222/changeids` - Return a commar-separated list of configured change id sets, where each set is a list of colon-separated change for each configured consumer group. +`curl localhost:2222/start` - start event delivery (default status). -`curl localhost:2222/fileprefixes` - Return the setting of env var MR_FILE_PREFIX_MAPPING. +`curl localhost:2222/stop` - stop event delivery. -`curl localhost:2222/ctr_requests` - return an integer of the number of get request to the event poll path +`curl localhost:2222/status` - returns the status, started or stopped . -`curl localhost:2222/groups/ctr_requests` - return an integer of the number of get requests, for all consumer groups, to the event poll path +`curl localhost:2222/execution_time` - returns the execution time in mm:ss -`curl localhost:2222/ctr_requests/<consumer-group>` - return an integer of the number of get requests, for the specified consumer group, to the event poll path +`curl localhost:2222/groups` - returns a comma-separated list of configured consumer groups.. +`curl localhost:2222/changeids` - returns a commar-separated list of configured change id sets, where each set is a list of colon-separated change for each configured consumer group. +`curl localhost:2222/fileprefixes` - returns the setting of env var MR_FILE_PREFIX_MAPPING. +`curl localhost:2222/ctr_requests` - returns an integer of the number of get requests, for all groups, to the event poll path +`curl localhost:2222/groups/ctr_requests` - returns a list of integers of the number of get requests, for each consumer group, to the event poll path +`curl localhost:2222/ctr_requests/<consumer-group>` - returns an integer of the number of get requests, for the specified consumer group, to the event poll path -`curl localhost:2222/ctr_responses` - return an integer of the number of get responses to the event poll path -`curl localhost:2222/ctr_files` - returns an integer or the number files. +`curl localhost:2222/ctr_responses` - returns an integer of the number of get responses, for all groups, to the event poll path -`curl localhost:2222/ctr_unique_files` - returns an integer or the number of unique files. A unique file is the combination of node+file_sequence_number +`curl localhost:2222/groups/ctr_responses` - returns a list of integers of the number of get responses, for each consumer group, to the event poll path -`curl localhost:2222/tc_info` - returns the tc string (as given on the cmd line) +`curl localhost:2222/ctr_responses/<consumer-group>` - returns an integer of the number of get responses, for the specified consumer group, to the event poll path -`curl localhost:2222/ctr_events` - returns the total number of events -`curl localhost:2222/execution_time` - returns the execution time in mm:ss +`curl localhost:2222/ctr_files` - returns an integer of the number generated files for all groups -`curl localhost:2222/exe_time_first_poll` - returns the execution time in mm:ss from the first poll +`curl localhost:2222/groups/ctr_files` - returns a list of integers of the number generated files for each group -`curl localhost:2222/ctr_unique_PNFs` - return the number of unique PNFS in alla events. +`curl localhost:2222/ctr_files/<consumer-group>` - returns an integer or the number generated files for the specified group -`curl localhost:2222/start` - start event delivery (default status). -`curl localhost:2222/stop` - stop event delivery. +`curl localhost:2222/ctr_unique_files` - returns an integer of the number generated unique files for all groups -`curl localhost:2222/status` - Return the started or stopped status. +`curl localhost:2222/groups/ctr_unique_files` - returns a list of integers of the number generated unique files for each group +`curl localhost:2222/ctr_unique_files/<consumer-group>` - returns an integer or the number generated unique files for the specified group -#Alternative to running python (as described below) on your machine, use the docker files. -1. Build docker container with ```docker build -t mrsim:latest .``` -2. Run the container ```docker-compose up``` -The behavior can be changed by argument to the python script in the docker-compose.yml -The simulator can be queried for statistics and started/stopped (use curl from cmd line or open in browser, curl used below): -`curl localhost:2222/ctr_requests` - return an integer of the number of get request to the event poll path +`curl localhost:2222/ctr_events` - returns the total number of events for all groups -`curl localhost:2222/ctr_responses` - return an integer of the number of get responses to the event poll path +`curl localhost:2222/groups/ctr_events` - returns a list the integer of the total number of events for each group -`curl localhost:2222/ctr_files` - returns an integer or the number files. +`curl localhost:2222/ctr_events/<consumer-group>` - returns the total number of events for a specified group -`curl localhost:2222/ctr_unique_files` - returns an integer or the number of unique files. A unique file is the combination of node+file_sequence_number -`curl localhost:2222/tc_info` - returns the tc string (as given on the cmd line) +`curl localhost:2222/exe_time_first_poll` - returns the execution time in mm:ss from the first poll -`curl localhost:2222/ctr_events` - returns the total number of events +`curl localhost:2222/groups/exe_time_first_poll` - returns a list of the execution time in mm:ss from the first poll for each group -`curl localhost:2222/execution_time` - returns the execution time in mm:ss +`curl localhost:2222/exe_time_first_poll/<consumer-group>` - returns the execution time in mm:ss from the first poll for the specified group -`curl localhost:2222/exe_time_first_poll` - returns the execution time in mm:ss from the first poll -`curl localhost:2222/ctr_unique_PNFs` - return the number of unique PNFS in alla events. +`curl localhost:2222/ctr_unique_PNFs` - returns the number of unique PNFS in all events. -`curl localhost:2222/start` - start event delivery (default status). +`curl localhost:2222/groups/ctr_unique_PNFs` - returns a list of the number of unique PNFS in all events for each group. -`curl localhost:2222/stop` - stop event delivery. +`curl localhost:2222/ctr_unique_PNFs/<consumer-group>` - returns the number of unique PNFS in all events for the specified group. + + +#Alternative to running python (as described below) on your machine, use the docker files. +1. Build docker container with ```docker build -t mrsim:latest .``` +2. Run the container ```docker-compose up``` +The behavior can be changed by argument to the python script in the docker-compose.yml -`curl localhost:2222/status` - Return the started or stopped status. ##Common TC info File names for 1MB, 5MB and 50MB files @@ -117,8 +123,8 @@ Files in the format: <size-in-mb>MB_<sequence-number>.tar.gz Ex. for 5MB file The sequence numbers are stepped so that all files have unique names Missing files (files that are not expected to be found in the ftp server. Format: MissingFile_<sequence-number>.tar.gz -Limited event streams -When the number of events are exhausted, empty replies are returned '[]' +When the number of events are exhausted, empty replies are returned '[]', for the limited test cases. For endless tc no empty replies will be given. +Test cases are limited unless noted as 'endless'. TC100 - One ME, SFTP, 1 1MB file, 1 event @@ -139,25 +145,56 @@ TC120 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per po TC121 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per poll. 10% missing files -TC122 - One ME, SFTP, 1MB files, 100 files per event, 100 events. 1 event per poll. All files with identical name. +TC122 - One ME, SFTP, 1MB files, 100 files per event, 100 events. 1 event per poll. All files with identical name. -TC510 - 700 MEs, SFTP, 1MB files, 1 file per event, 3500 events, 700 event per poll. +TC500 - 700 MEs, SFTP, 1MB files, 2 new files per event, 700 events, all event in one poll -TC200-TC202 same as TC100-TC102 but with FTPS +TC501 - 700 MEs, SFTP, 5MB files, 2 new files per event, 700 events, all event in one poll -TC210-TC213 same as TC110-TC113 but with FTPS +TC502 - 700 MEs, SFTP, 50MB files, 2 new files per event, 700 events, all event in one poll -TC2000-TC2001 same as TC1000-TC1001 but with FTPS +TC510 - 700 MEs, SFTP, 1MB files, 1 file per event, 3500 events, 700 event per poll. -TC610 same as TC510 but with FTPS +TC511 - 700 MEs, SFTP, 1KB files, 1 file per event, 3500 events, 700 event per poll +TC550 - 700 MEs, SFTP, 50MB files, 1 file per event, 3500 events, 700 event per poll -Endless event streams +TC710 - 700 MEs, SFTP, 1MB files, 100 files per event, 3500 events, 35 event per poll TC1000 - One ME, SFTP, 1MB files, 100 files per event, endless number of events, 1 event per poll TC1001 - One ME, SFTP, 5MB files, 100 files per event, endless number of events, 1 event per poll +TC1100 - 700 ME, SFTP, 1MB files, 100 files per event, endless number of events, 35 event per poll + +TC1101 - 700 ME, SFTP, 5MB files, 100 files per event, endless number of events, 35 event per poll + +TC1102 - 700 ME, SFTP, 50MB files, 100 files per event, endless number of events, 35 event per poll + +TC1200 - 700 ME, SFTP, 1MB files, 100 new files per event, endless number of events, 35 event per poll + +TC1201 - 700 ME, SFTP, 5MB files, 100 new files per event, endless number of events, 35 event per poll + +TC1202 - 700 ME, SFTP, 50MB files, 100 new files per event, endless number of events, 35 event per poll + +TC1300 - 700 ME, SFTP, 1MB files, 100 files per event, endless number of events, 35 event per poll, 20 event polls every 15min + +TC1301 - 700 ME, SFTP, 5MB files, 100 files per event, endless number of events, 35 event per poll, 20 event polls every 15min + +TC1302 - 700 ME, SFTP, 50MB files, 100 files per event, endless number of events, 35 event per poll, 20 event polls every 15min + +TC1500 - 700 ME, SFTP, 1MB files, 100 files per event, 35 events per poll, simulating 25h backlog of decreasing number of outdated files and then 20 event polls every 15min for 1h + +Changing the first digit in tc number will change the test case to run FTPS instead. Eg. TC201 is FTPS version of TC101. + +TC2XX is same as TC1XX but with FTPS + +TC6XX is same as TC5XX but with FTPS + +TC8XX is same as TC7XX but with FTPS + +TC2XXX is same as TC1XXX but with FTPS + ## Developer workflow diff --git a/test/mocks/datafilecollector-testharness/simulator-group/README.md b/test/mocks/datafilecollector-testharness/simulator-group/README.md index 5a51d8ace..55a2467ae 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/README.md +++ b/test/mocks/datafilecollector-testharness/simulator-group/README.md @@ -45,6 +45,8 @@ Run the script `prepare-images.sh` to build the docker images for MR, DR and FTP ###Summary of scripts and files `consul_config.sh` - Convert a json config file to work with dfc when manually started as java-app or container and then add that json to Consul. +`dfc-internal-stats.sh` - Periodically extract jvm data and dfc internal data and print to console/file. + `docker-compose-setup.sh` - Sets environment variables for the simulators and start the simulators with that settings. `docker-compose-template.yml` - A docker compose template with environment variables setting. Used for producing a docker-compose file to defined the simulator containers. @@ -76,16 +78,9 @@ cd ../dr-sim Run the docker build command to build the image for the DR simulators: `docker build -t drsim_common:latest .' - -cd ../simulator-group - -Copy the 'configuration' and 'tls' catalogues from the ftps-sftp-server dir. - +cd ../ftps-sftp-server Check the README.md in ftps-sftp-server dir in case the cert need to be updated. - -cp -r ../ftps-sftp-server/configuration . - -cp -r ../ftps-sftp-server/tls . +Run the docker build command to build the image for the DR simulators: `docker build -t ftps_vsftpd:latest -f Dockerfile-ftps .' ###Execution diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed1_PM_feed2_CTR.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed1_PM_feed2_CTR.json index 75637ce79..c320eda7a 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed1_PM_feed2_CTR.json +++ b/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed1_PM_feed2_CTR.json @@ -18,7 +18,7 @@ "password": "password", "publisher_id": "972.360gm" }, - "CTR_FILES": { + "CTR_MEAS_FILES": { "type": "data_router", "dmaap_info": { "feed02": { diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed3_PM_CTR.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed3_PM_CTR.json index 2f5287bbc..489580a4f 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed3_PM_CTR.json +++ b/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed3_PM_CTR.json @@ -20,7 +20,7 @@ "publisher_id": "972.360gm" } }, - "CTR_FILES":{ + "CTR_MEAS_FILES":{ "type":"data_router", "dmaap_info":{ "username": "user", diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1.json deleted file mode 100644 index e6769d877..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "feed01": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/1", - "publish_url": "https://drsim:3907/publish/1", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -}
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4.json deleted file mode 100644 index aa2e6a056..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "feed01": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/1", - "publish_url": "https://drsim:3907/publish/1", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed02": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/2", - "publish_url": "https://drsim:3907/publish/2", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed03": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/3", - "publish_url": "https://drsim:3907/publish/3", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed04": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/4", - "publish_url": "https://drsim:3907/publish/4", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -}
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4_5.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4_5.json deleted file mode 100644 index 6f28f39c3..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4_5.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "feed01": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/1", - "publish_url": "https://drsim:3907/publish/1", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed02": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/2", - "publish_url": "https://drsim:3907/publish/2", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed03": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/3", - "publish_url": "https://drsim:3907/publish/3", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed04": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/4", - "publish_url": "https://drsim:3907/publish/4", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed05": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/5", - "publish_url": "https://drsim:3907/publish/4", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -}
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2.json deleted file mode 100644 index bea360c44..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "feed02": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/2", - "publish_url": "https://drsim:3907/publish/2", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -}
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2_3.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2_3.json deleted file mode 100644 index a84bf3391..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2_3.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "feed02": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/2", - "publish_url": "https://drsim:3907/publish/2", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed03": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/3", - "publish_url": "https://drsim:3907/publish/3", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -}
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed3.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed3.json deleted file mode 100644 index 11138e354..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed3.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "feed03": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/3", - "publish_url": "https://drsim:3907/publish/3", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -}
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/dfc-internal-stats.sh b/test/mocks/datafilecollector-testharness/simulator-group/dfc-internal-stats.sh index 9fbe9615c..e0d7c33b7 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/dfc-internal-stats.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/dfc-internal-stats.sh @@ -34,7 +34,7 @@ else print_usage exit 1 fi -fileoutput="./.tmp_stats.txt" +fileoutput=".tmp_stats.txt" echo "Stats piped to file: "$fileoutput @@ -75,29 +75,41 @@ do_curl_status() { } +OK=0 # Flag for DFC response (0==no response, 1==reponse ok and logging can start) while [ true ]; do - if [ $heading -eq 1 ]; then - echo -n "date" >> $fileoutput - else - ds=$(date) - echo -n $ds >> $fileoutput - fi - if [ $stat == "all" ] || [ $stat == "jvm" ]; then - echo "========= DFC JVM Stats =========" - do_curl_actuator jvm.threads.live - do_curl_actuator jvm.threads.peak - do_curl_actuator process.files.open - do_curl_actuator process.files.max - do_curl_actuator jvm.memory.used - do_curl_actuator jvm.memory.max - fi + if [ $OK -eq 0 ]; then + test=$(curl -s localhost:${dfcport}/status) + if [ -z "$test" ] && [ $heading -eq 1 ]; then + echo "No response from dfc on port: ${dfcport}. Retrying..." + else + echo "Response from dfc on port: ${dfcport}. Starts logging." + OK=1 + fi + fi + if [ $OK -eq 1 ]; then + if [ $heading -eq 1 ]; then + echo -n "date" >> $fileoutput + else + ds=$(date) + echo -n $ds >> $fileoutput + fi + if [ $stat == "all" ] || [ $stat == "jvm" ]; then + echo "========= DFC JVM Stats =========" + do_curl_actuator jvm.threads.live + do_curl_actuator jvm.threads.peak + do_curl_actuator process.files.open + do_curl_actuator process.files.max + do_curl_actuator jvm.memory.used + do_curl_actuator jvm.memory.max + fi - if [ $stat == "all" ] || [ $stat == "internal" ]; then - echo "========= DFC internal Stats =========" - do_curl_status - fi - echo "" >> $fileoutput - heading=0 + if [ $stat == "all" ] || [ $stat == "internal" ]; then + echo "========= DFC internal Stats =========" + do_curl_status + fi + echo "" >> $fileoutput + heading=0 + fi sleep 5 done
\ No newline at end of file |