From 69311bf8082471be45574c9a99b23e31e606669c Mon Sep 17 00:00:00 2001 From: maximesson Date: Fri, 30 Aug 2019 14:05:15 +0000 Subject: Updated README files and adaptation of consul/cbs config Change-Id: I81494f56978a3d0ff06ec0d66968f33f08114103 Issue-ID: DCAEGEN2-1719 Signed-off-by: maximesson --- .../simulator-group/README.md | 13 ++--- .../consul/c12_feed1_PM_feed2_CTR.json | 2 +- .../simulator-group/consul/c12_feed3_PM_CTR.json | 2 +- .../simulator-group/consul/dmaap_feed1.json | 10 ---- .../simulator-group/consul/dmaap_feed1_2_3_4.json | 34 ------------- .../consul/dmaap_feed1_2_3_4_5.json | 42 ---------------- .../simulator-group/consul/dmaap_feed2.json | 10 ---- .../simulator-group/consul/dmaap_feed2_3.json | 18 ------- .../simulator-group/consul/dmaap_feed3.json | 10 ---- .../simulator-group/dfc-internal-stats.sh | 56 +++++++++++++--------- 10 files changed, 40 insertions(+), 157 deletions(-) delete mode 100644 test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1.json delete mode 100644 test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4.json delete mode 100644 test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4_5.json delete mode 100644 test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2.json delete mode 100644 test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2_3.json delete mode 100644 test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed3.json (limited to 'test/mocks/datafilecollector-testharness/simulator-group') diff --git a/test/mocks/datafilecollector-testharness/simulator-group/README.md b/test/mocks/datafilecollector-testharness/simulator-group/README.md index 5a51d8ace..55a2467ae 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/README.md +++ b/test/mocks/datafilecollector-testharness/simulator-group/README.md @@ -45,6 +45,8 @@ Run the script `prepare-images.sh` to build the docker images for MR, DR and FTP ###Summary of scripts and files `consul_config.sh` - Convert a json config file to work with dfc when manually started as java-app or container and then add that json to Consul. +`dfc-internal-stats.sh` - Periodically extract jvm data and dfc internal data and print to console/file. + `docker-compose-setup.sh` - Sets environment variables for the simulators and start the simulators with that settings. `docker-compose-template.yml` - A docker compose template with environment variables setting. Used for producing a docker-compose file to defined the simulator containers. @@ -76,16 +78,9 @@ cd ../dr-sim Run the docker build command to build the image for the DR simulators: `docker build -t drsim_common:latest .' - -cd ../simulator-group - -Copy the 'configuration' and 'tls' catalogues from the ftps-sftp-server dir. - +cd ../ftps-sftp-server Check the README.md in ftps-sftp-server dir in case the cert need to be updated. - -cp -r ../ftps-sftp-server/configuration . - -cp -r ../ftps-sftp-server/tls . +Run the docker build command to build the image for the DR simulators: `docker build -t ftps_vsftpd:latest -f Dockerfile-ftps .' ###Execution diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed1_PM_feed2_CTR.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed1_PM_feed2_CTR.json index 75637ce79..c320eda7a 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed1_PM_feed2_CTR.json +++ b/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed1_PM_feed2_CTR.json @@ -18,7 +18,7 @@ "password": "password", "publisher_id": "972.360gm" }, - "CTR_FILES": { + "CTR_MEAS_FILES": { "type": "data_router", "dmaap_info": { "feed02": { diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed3_PM_CTR.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed3_PM_CTR.json index 2f5287bbc..489580a4f 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed3_PM_CTR.json +++ b/test/mocks/datafilecollector-testharness/simulator-group/consul/c12_feed3_PM_CTR.json @@ -20,7 +20,7 @@ "publisher_id": "972.360gm" } }, - "CTR_FILES":{ + "CTR_MEAS_FILES":{ "type":"data_router", "dmaap_info":{ "username": "user", diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1.json deleted file mode 100644 index e6769d877..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "feed01": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/1", - "publish_url": "https://drsim:3907/publish/1", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -} \ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4.json deleted file mode 100644 index aa2e6a056..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "feed01": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/1", - "publish_url": "https://drsim:3907/publish/1", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed02": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/2", - "publish_url": "https://drsim:3907/publish/2", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed03": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/3", - "publish_url": "https://drsim:3907/publish/3", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed04": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/4", - "publish_url": "https://drsim:3907/publish/4", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -} \ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4_5.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4_5.json deleted file mode 100644 index 6f28f39c3..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed1_2_3_4_5.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "feed01": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/1", - "publish_url": "https://drsim:3907/publish/1", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed02": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/2", - "publish_url": "https://drsim:3907/publish/2", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed03": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/3", - "publish_url": "https://drsim:3907/publish/3", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed04": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/4", - "publish_url": "https://drsim:3907/publish/4", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed05": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/5", - "publish_url": "https://drsim:3907/publish/4", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -} \ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2.json deleted file mode 100644 index bea360c44..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "feed02": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/2", - "publish_url": "https://drsim:3907/publish/2", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -} \ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2_3.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2_3.json deleted file mode 100644 index a84bf3391..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed2_3.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "feed02": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/2", - "publish_url": "https://drsim:3907/publish/2", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - }, - "feed03": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/3", - "publish_url": "https://drsim:3907/publish/3", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -} \ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed3.json b/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed3.json deleted file mode 100644 index 11138e354..000000000 --- a/test/mocks/datafilecollector-testharness/simulator-group/consul/dmaap_feed3.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "feed03": { - "username": "user", - "log_url": "https://drsim:3907/feedlog/3", - "publish_url": "https://drsim:3907/publish/3", - "location": "loc00", - "password": "password", - "publisher_id": "972.360gm" - } -} \ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/dfc-internal-stats.sh b/test/mocks/datafilecollector-testharness/simulator-group/dfc-internal-stats.sh index 9fbe9615c..e0d7c33b7 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/dfc-internal-stats.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/dfc-internal-stats.sh @@ -34,7 +34,7 @@ else print_usage exit 1 fi -fileoutput="./.tmp_stats.txt" +fileoutput=".tmp_stats.txt" echo "Stats piped to file: "$fileoutput @@ -75,29 +75,41 @@ do_curl_status() { } +OK=0 # Flag for DFC response (0==no response, 1==reponse ok and logging can start) while [ true ]; do - if [ $heading -eq 1 ]; then - echo -n "date" >> $fileoutput - else - ds=$(date) - echo -n $ds >> $fileoutput - fi - if [ $stat == "all" ] || [ $stat == "jvm" ]; then - echo "========= DFC JVM Stats =========" - do_curl_actuator jvm.threads.live - do_curl_actuator jvm.threads.peak - do_curl_actuator process.files.open - do_curl_actuator process.files.max - do_curl_actuator jvm.memory.used - do_curl_actuator jvm.memory.max - fi + if [ $OK -eq 0 ]; then + test=$(curl -s localhost:${dfcport}/status) + if [ -z "$test" ] && [ $heading -eq 1 ]; then + echo "No response from dfc on port: ${dfcport}. Retrying..." + else + echo "Response from dfc on port: ${dfcport}. Starts logging." + OK=1 + fi + fi + if [ $OK -eq 1 ]; then + if [ $heading -eq 1 ]; then + echo -n "date" >> $fileoutput + else + ds=$(date) + echo -n $ds >> $fileoutput + fi + if [ $stat == "all" ] || [ $stat == "jvm" ]; then + echo "========= DFC JVM Stats =========" + do_curl_actuator jvm.threads.live + do_curl_actuator jvm.threads.peak + do_curl_actuator process.files.open + do_curl_actuator process.files.max + do_curl_actuator jvm.memory.used + do_curl_actuator jvm.memory.max + fi - if [ $stat == "all" ] || [ $stat == "internal" ]; then - echo "========= DFC internal Stats =========" - do_curl_status - fi - echo "" >> $fileoutput - heading=0 + if [ $stat == "all" ] || [ $stat == "internal" ]; then + echo "========= DFC internal Stats =========" + do_curl_status + fi + echo "" >> $fileoutput + heading=0 + fi sleep 5 done \ No newline at end of file -- cgit 1.2.3-korg