diff options
Diffstat (limited to 'test/mocks/datafilecollector-testharness/simulator-group')
7 files changed, 291 insertions, 33 deletions
diff --git a/test/mocks/datafilecollector-testharness/simulator-group/README.md b/test/mocks/datafilecollector-testharness/simulator-group/README.md index 4d448f8d2..5981c7978 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/README.md +++ b/test/mocks/datafilecollector-testharness/simulator-group/README.md @@ -4,15 +4,17 @@ Mainly this is needed for CSIT tests but can be used also for local testing. ###Preparation +Do the manual steps to prepare the simulator images + Build the mr-sim image. cd ../mr-sim -Run the docker build command to build the image for the MR simulator: 'docker build -t mrsim:latest . +Run the docker build command to build the image for the MR simulator: 'docker build -t mrsim:latest .' cd ../dr-sim -Run the docker build command to build the image for the DR simulators: `docker build -t drsim_common:latest . +Run the docker build command to build the image for the DR simulators: `docker build -t drsim_common:latest .' cd ../simulator-group @@ -21,21 +23,53 @@ Copy the 'configuration' and 'tls' catalogues from the ftps-sftp-server dir. Check the README.md in ftps-sftp-server dir in case the cert need to be updated. -cp -r ./ftps-sftp-server/configuration . +cp -r ../ftps-sftp-server/configuration . cp -r ../ftps-sftp-server/tls . ###Execution -Edit the `docker-compose-setup.sh` to setup the env variables to the desired test behavior for each simulators. +Edit the `docker-compose-setup.sh` (or create a copy) to setup the env variables to the desired test behavior for each simulators. See each simulator to find a description of the available settings. +The following env variables shall be set (example values). +Note that NUM_FTPFILES and NUM_PNFS controls the number of ftp files created in the ftp servers. +A total of NUM_FTPFILES * NUM_PNFS ftp files will be created in each dtp server (4 files in the below example). +Large settings will be time consuming at start of the servers. + +DR_TC="--tc normal" #Normal behavior of the DR sim + +DR_REDIR_TC="--tc normal" #Normal behavior of the DR redirect sim + +MR_TC="--tc100" #One 1 MB file in one event, once. + +BC_TC="" #Not in use yet + +NUM_FTPFILES="2" #Two file for each PNF -Run the script `docker-compose-setup.sh`to create a docker-compose with the desired settings. All simulators -will be started with the generated docker-compose.yml file +NUM_PNFS="2" #Two PNFs + +Run the script `docker-compose-setup.sh`to create a docker-compose with the desired settings. The desired setting +in the script need to be manually adapted to for each specific simulator behavior according to the above. Check each simulator for available +parameters. +All simulators will be started with the generated docker-compose.yml file + +To generate ftp url with IP different from localhost, set SFTP_SIM_IP and/or FTPS_SIM_IP env variables to the addreses of the ftp servers before starting. +So farm, this only works when the simulator python script is started from the command line. Kill all the containers with `simulators-kill.se` `simulators_start.sh` is for CSIT test and requires the env variables for test setting to be present in the shell. `setup-ftp-files.for-image.sh` is for CSIT and executed when the ftp servers are started from the docker-compose-setup.sh`. +To make DFC to be able to connect to the simulator containers, DFC need to run in host mode. +Start DFC by the following cmd: `docker run -d --network="host" --name dfc_app <dfc-image> ` + +`<dfc-image>` could be either the locally built image `onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server` +or the one in nexus `nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server`. + + + +###Simulator monitor +Start the simulator monitor server with `sim-monitor-start.sh` and the open a browser with the url `localhost:9999/mon` +to see the statisics page with data from MR sim, DR sim and DR redir sim. diff --git a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh index d0171a5e3..af36d0562 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh @@ -2,10 +2,13 @@ #Script for manually starting all simulators with test setting below -export DR_TC="" -export DR_REDIR_TC="" +export DR_TC="--tc normal" +export DR_REDIR_TC="--tc normal" export MR_TC="--tc100" -export BC_TC="" -export NUM_FTPFILES="250" +export BC_TC="" #Not in use yet +export NUM_FTPFILES="10" +export NUM_PNFS="700" +export FILE_SIZE="1MB" +export FTP_TYPE="SFTP" source ./simulators-start.sh
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml index e7c700759..89a45a476 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml +++ b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml @@ -1,8 +1,17 @@ version: '2' + +networks: + dfcnet: + ipam: + config: + - subnet: 192.168.100.0/16 + services: drsim: - network_mode: "host" + networks: + dfcnet: + ipv4_address: 192.168.100.2 image: drsim_common:latest ports: - "3906:3906" @@ -11,6 +20,9 @@ services: command: node dmaapDR.js ${DR_TC} drsim_redir: + networks: + dfcnet: + ipv4_address: 192.168.100.3 image: drsim_common:latest ports: - "3908:3908" @@ -19,6 +31,9 @@ services: command: node dmaapDR_redir.js ${DR_REDIR_TC} mrsim: + networks: + dfcnet: + ipv4_address: 192.168.100.1 image: mrsim:latest ports: - "2222:2222" @@ -26,17 +41,17 @@ services: command: python mr-sim.py ${MR_TC} sftp-server: + network_mode: bridge container_name: dfc_sftp-server image: atmoz/sftp:alpine ports: - "1022:22" -# volumes: -# - ./files/onap/sftp/:/home/onap/ restart: on-failure command: onap:pano:1001 ftpes-server-vsftpd: + network_mode: bridge container_name: dfc_ftpes-server-vsftpd image: docker.io/panubo/vsftpd ports: @@ -54,6 +69,5 @@ services: - ./tls/dfc.crt:/etc/ssl/private/dfc.crt:ro - ./configuration/vsftpd_ssl.conf:/etc/vsftpd_ssl.conf:ro -# - ./files/onap/ftps/:/srv/ restart: on-failure
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh b/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh index d8e57a3d8..6c2b4f29f 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh @@ -1,28 +1,47 @@ -#!/bin/bash +#!/usr/bin/env bash # Script to create files for the FTP server to return upon request. # The file names matches the files names in the events polled from the MR simulator. # Intended for execution in the running ftp containers in the ftp-root dir. -echo "Running ftp file creations" - NUM=200 #Default number of files +PNFS=1 #Default number of PNFs +FSIZE="ALL" if [ $# -eq 1 ]; then NUM=$1 +elif [ $# -eq 2 ]; then + NUM=$1 + PNFS=$2 +elif [ $# -eq 3 ]; then + NUM=$1 + PNFS=$2 + FSIZE=$3 + if [ $3 != "1KB" ] && [ $3 != "1MB" ] && [ $3 != "5MB" ] && [ $3 != "50MB" ] && [ $3 != "ALL" ]; then + echo "File size shall be 1KB|1MB|5MB|50MB|ALL" + exit + fi +else + echo "Wrong args, usage: setup-ftp-files-for-image.sh [ <num-files> [ <num-PNFs> [ 1KB|1MB|5MB|50MB ] ] ]" + exit fi +echo "Running ftp file creations. " $PNFS " PNFs and " $NUM " files for each PNF with file size(s) "$FSIZE + +truncate -s 1KB 1KB.tar.gz truncate -s 1MB 1MB.tar.gz truncate -s 5MB 5MB.tar.gz truncate -s 50MB 50MB.tar.gz - -i=0 -while [ $i -lt $NUM ]; do #Problem with for loop and var substituion in curly bracket....so used good old style loop - ln -s 1MB.tar.gz 1MB_$i.tar.gz - ln -s 5MB.tar.gz 5MB_$i.tar.gz - let i=i+1 +p=0 +while [ $p -lt $PNFS ]; do + i=0 + while [ $i -lt $NUM ]; do #Problem with for loop and var substituion in curly bracket....so used good old style loop + if [ $FSIZE = "ALL" ] || [ $FSIZE = "1KB" ]; then ln -s 1KB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-1KB.tar.gz'; fi + if [ $FSIZE = "ALL" ] || [ $FSIZE = "1MB" ]; then ln -s 1MB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-1MB.tar.gz'; fi + if [ $FSIZE = "ALL" ] || [ $FSIZE = "5MB" ]; then ln -s 5MB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-5MB.tar.gz'; fi + if [ $FSIZE = "ALL" ] || [ $FSIZE = "50MB" ]; then ln -s 50MB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-50MB.tar.gz'; fi + let i=i+1 + done + let p=p+1 done - - -ln -s 50MB.tar.gz 50MB_0.tar.gz #Large file, only for single file test diff --git a/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor-start.sh b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor-start.sh new file mode 100755 index 000000000..52c8c1cac --- /dev/null +++ b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor-start.sh @@ -0,0 +1,7 @@ +#/bin/bash + +#Script to start the sim-monitor + +#Re-using modules for dr-sim +cp -r ../dr-sim/node_modules . +node sim-monitor.js
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor.js b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor.js new file mode 100644 index 000000000..e4a19c21e --- /dev/null +++ b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor.js @@ -0,0 +1,165 @@ +var http = require('http'); + +var express = require('express'); +var app = express(); + +//I am alive +app.get("/",function(req, res){ + res.send("ok"); +}) + +//Get parameter valuye from other server +function getSimCtr(url, cb) { + var data = ''; + http.get(url, (resp) => { + // A chunk of data has been recieved. + resp.on('data', (chunk) => { + data += chunk; + }); + + // The whole response has been received. + resp.on('end', () => { + //Pad data to fixed length + var i = 20-data.length; + while(i>0) { + data = data+" "; + i--; + } + cb(data); + }); + + }).on("error", (err) => { + console.log("Error: " + err.message); + cb("no response from simulator"); + }); +}; + +//Status variables, for parameters values fetched from other simulators +var mr1, mr2, mr3, mr4, mr5, mr6, mr7; + +var dr1, dr2, dr3, dr4, dr5, dr6, dr7, dr8, dr9; + +var drr1, drr2, drr3, drr4, drr5, drr6; + +app.get("/mon",function(req, res){ + + //MR + getSimCtr("http://127.0.0.1:2222/ctr_requests", function(data) { + mr1 = data; + }); + getSimCtr("http://127.0.0.1:2222/ctr_responses", function(data) { + mr2 = data; + }); + getSimCtr("http://127.0.0.1:2222/ctr_unique_files", function(data) { + mr3 = data; + }); + getSimCtr("http://127.0.0.1:2222/tc_info", function(data) { + mr4 = data; + }); + getSimCtr("http://127.0.0.1:2222/ctr_events", function(data) { + mr5 = data; + }); + getSimCtr("http://127.0.0.1:2222/execution_time", function(data) { + mr6 = data; + }); + getSimCtr("http://127.0.0.1:2222/ctr_unique_PNFs", function(data) { + mr7 = data; + }); + + //DR + getSimCtr("http://127.0.0.1:3906/ctr_publish_query", function(data) { + dr1 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_query_published", function(data) { + dr2 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_query_not_published", function(data) { + dr3 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_req", function(data) { + dr4 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_req_redirect", function(data) { + dr5 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_req_published", function(data) { + dr6 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_published_files", function(data) { + dr7 = data; + }); + getSimCtr("http://127.0.0.1:3906/tc_info", function(data) { + dr8 = data; + }); + getSimCtr("http://127.0.0.1:3906/execution_time", function(data) { + dr9 = data; + }); + + //DR REDIR + getSimCtr("http://127.0.0.1:3908/ctr_publish_requests", function(data) { + drr1 = data; + }); + getSimCtr("http://127.0.0.1:3908/ctr_publish_responses", function(data) { + drr2 = data; + }); + getSimCtr("http://127.0.0.1:3908/tc_info", function(data) { + drr3 = data; + }); + getSimCtr("http://127.0.0.1:3908/execution_time", function(data) { + drr4 = data; + }); + getSimCtr("http://127.0.0.1:3908/time_lastpublish", function(data) { + drr5 = data; + }); + getSimCtr("http://127.0.0.1:3908/dwl_volume", function(data) { + drr6 = data; + }); + + //Build web page + var str = "<!DOCTYPE html>" + + "<html>" + + "<head>" + + "<meta http-equiv=\"refresh\" content=\"5\">"+ //5 sec auto reefresh + "<title>Simulator monitor</title>"+ + "</head>" + + "<body>" + + "<h3>MR Simulator</h3>" + + "<font face=\"courier\">"+ + "MR TC:........................................." + mr4 + "<br>" + + "Execution time (mm.ss):........................" + mr6 + "<br>" + + "Number of requests (polls):...................." + mr1 + "<br>" + + "Number of responses (polls):..................." + mr2 + "<br>" + + "Number of unique files in all responses:......." + mr3 + "<br>" + + "Number of events..............................." + mr5 + "<br>" + + "Number of unique PNFs.........................." + mr7 + "<br>" + + "</font>"+ + "<h3>DR Simulator</h3>" + + "<font face=\"courier\">"+ + "DR TC:........................................." + dr8 + "<br>" + + "Execution time (mm.ss):........................" + dr9 + "<br>" + + "Number of queries:............................." + dr1 + "<br>" + + "Number of query responses, file published:....." + dr2 + "<br>" + + "Number of query responses, file not published:." + dr3 + "<br>" + + "Number of requests:............................" + dr4 + "<br>" + + "Number of responses with redirect:............." + dr5 + "<br>" + + "Number of responses without redirect:.........." + dr6 + "<br>" + + "Number of published files:....................." + dr7 + "<br>" + + "</font>"+ + "<h3>DR Redirect Simulator</h3>" + + "<font face=\"courier\">"+ + "DR REDIR TC:..................................." + drr3 + "<br>" + + "Execution time (mm.ss):........................" + drr4 + "<br>" + + "Number of requests:............................" + drr1 + "<br>" + + "Number of responses:..........................." + drr2 + "<br>" + + "Downloaded volume (bytes):....................." + drr6 + "<br>" + + "Last publish (mm:ss):.........................." + drr5 + "<br>" + + "</font>"+ + "</body>" + + "</html>"; + res.send(str); +}) + +var httpServer = http.createServer(app); +var httpPort=9999; +httpServer.listen(httpPort); +console.log("Simulator monitor listening (http) at "+httpPort);
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh b/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh index ef1b90a38..cc5ce32ec 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh @@ -35,11 +35,27 @@ if [ -z "$NUM_FTPFILES" ] then NUM_FTPFILES=200 fi +if [ -z "$NUM_PNFS" ] + then + NUM_PNFS=1 +fi +if [ -z "$FILE_SIZE" ] + then + FILE_SIZE="ALL" +fi +if [ -z "$FTP_TYPE" ] + then + FTP_TYPE="ALL" +fi - -docker cp setup-ftp-files-for-image.sh $SFTP_SIM:/tmp/ -docker exec -w /home/onap/ $SFTP_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES >/dev/null 2>&1 - -docker cp setup-ftp-files-for-image.sh $FTPS_SIM:/tmp/setup-ftp-files-for-image.sh -docker exec -w /srv $FTPS_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES >/dev/null 2>&1 - +if [ $FTP_TYPE = "ALL" ] || [ $FTP_TYPE = "SFTP" ]; then + echo "Creating files for SFTP server, may take time...." + docker cp setup-ftp-files-for-image.sh $SFTP_SIM:/tmp/ + docker exec -w /home/onap/ $SFTP_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES $NUM_PNFS $FILE_SIZE #>/dev/null 2>&1 +fi +if [ $FTP_TYPE = "ALL" ] || [ $FTP_TYPE = "FTPS" ]; then + echo "Creating files for FTPS server, may take time...." + docker cp setup-ftp-files-for-image.sh $FTPS_SIM:/tmp/setup-ftp-files-for-image.sh + docker exec -w /srv $FTPS_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES $NUM_PNFS $FILE_SIZE #>/dev/null 2>&1 +fi +echo "Done: All simulators started and configured" |