diff options
13 files changed, 1070 insertions, 159 deletions
diff --git a/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile b/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile index fc903d7e7..3d95492a8 100644 --- a/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile +++ b/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile @@ -12,7 +12,10 @@ COPY cert/ cert/ RUN npm install express RUN npm install argparse -EXPOSE 3906 -EXPOSE 3907 -EXPOSE 3908 -EXPOSE 3909
\ No newline at end of file +#Ports for DR +#EXPOSE 3906 +#EXPOSE 3907 + +#Ports for DR redir +#EXPOSE 3908 +#EXPOSE 3909
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/dr-sim/README.md b/test/mocks/datafilecollector-testharness/dr-sim/README.md index f0cdf589b..8761d0cef 100644 --- a/test/mocks/datafilecollector-testharness/dr-sim/README.md +++ b/test/mocks/datafilecollector-testharness/dr-sim/README.md @@ -1,6 +1,9 @@ -#Alternative to running python (as described below) on your machine, use the docker files. +###Alternative to running python (as described below) on your machine, use the docker files. 1. Build docker container with ```docker build -t drsim_common:latest .``` 2. Run the container ```docker-compose up``` +3. For specific behavior of of the simulators, add arguments to the `command` entries in the `docker-compose.yml`. +For example `command: node dmaapDR.js --tc no_publish` . (No argument will assume '--tc normal'). Run `node dmaapDR.js --printtc` +and `node dmaapDR-redir.js --printtc` for details. @@ -9,5 +12,46 @@ Make sure that you run these commands in the application directory "dr-sim" 3. `npm install express` 4. `npm install argparse` -5. `node dmaapDR.js` #keep it in the foreground -6. `node dmaapDR_redir.js` #keep it in the foreground +5. `node dmaapDR.js` #keep it in the foreground, see item 3 in the above list for arg to the simulator +6. `node dmaapDR_redir.js` #keep it in the foreground, see item 3 in the above list for arg to the simulator + + +The dmaapDR_redir server send callback to dmaapDR server to update the list of successfully published files. +As default, the ip for dmaapDR is set to work when running as container (using an ip address from the dfc_net docker network) . When running the servers from command line, set the env variable DR_SIM_IP=localhost + +The simulator can be queried for statistics (use curl from cmd line or open in browser, curl used below): + +DR + +`curl localhost:3906/ctr_publish_query` - returns the number of publish queries + +`curl localhost:3906/ctr_publish_query_published` - returns the number of responses where the files was published + +`curl localhost:3906/ctr_publish_query_not_published` - returns the number of responses where the file was not published + +`curl localhost:3906/ctr_publish_req` - returns the number of publish requests + +`curl localhost:3906/ctr_publish_req_redirect` - returns the number of publish responses with redirect + +`curl localhost:3906/ctr_publish_req_published` - returns the number of publish responses without redirect + +`curl localhost:3906/ctr_published_files` - returns the number of unique published files + +`curl localhost:3906/tc_info` - returns the tc name (argument on the command line) + +`curl localhost:3906/execution_time` - returns the execution times in mm:ss + + +DR REDIR + +`curl localhost:3908/ctr_publish_requests` - returns the number of publish queries + +`curl localhost:3908/ctr_publish_responses` - returns the number of publish responses + +`curl localhost:3908/tc_info` - returns the tc name (argument on the command line) + +`curl localhost:3908/execution_time` - returns the execution times in mm:ss + +`curl localhost:3908/time_lastpublish` - returns the time (mm:ss) for the latest publish + +`curl localhost:3908/dwl_volume` - returns the total received data volume of file data diff --git a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js index 4e7317473..fffe57ce3 100644 --- a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js +++ b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js @@ -5,12 +5,33 @@ var express = require('express'); const stream = require('stream'); var app = express(); var fs = require('fs'); -var path = require('path'); var privateKey = fs.readFileSync('cert/private.key', 'utf8'); var certificate = fs.readFileSync('cert/certificate.crt', 'utf8'); var credentials = {key: privateKey, cert: certificate}; -const allPublished = "allPublished"; -const nonePublished = "nonePublished"; + +//For execution time calculation +var startTime = Date.now(); + +//Test case constants +const tc_normal = "normal"; +const tc_none_published = "none_published"; +const tc_all_published = "all_published" +const tc_10p_no_response = "10p_no_response"; +const tc_10first_no_response = "10first_no_response"; +const tc_100first_no_response = "100first_no_response"; +const tc_all_delay_10s = "all_delay_10s"; +const tc_10p_delay_10s = "10p_delay_10s"; +const tc_10p_error_response = "10p_error_response"; +const tc_10first_error_response = "10first_error_response"; +const tc_100first_error_response = "100first_error_response"; + +//Counters +var ctr_publish_query = 0; +var ctr_publish_query_published = 0; +var ctr_publish_query_not_published = 0; +var ctr_publish_req = 0; +var ctr_publish_req_redirect = 0; +var ctr_publish_req_published = 0; var parser = new ArgumentParser({ version: '0.0.1', @@ -28,20 +49,59 @@ parser.addArgument('--printtc' , var args = parser.parseArgs(); -if (args.tc=="nonePublished") { - console.log("TC: nonePublished") -} -if (args.tc=="allPublished") { - console.log("TC: allPublished") - //preparations +if (args.tc==tc_normal) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_none_published) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_all_published) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10first_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_100first_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_all_delay_10s) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_delay_10s) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_error_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10first_error_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_100first_error_response) { + console.log("TC: " + args.tc) +} else { + console.log("No TC specified, use: --tc <tc-id>"); + process.exit(0); } if (args.printtc) { - console.log("TC nonePublished: no file has already been published."); - console.log("TC allPublished: whatever is the request, this file is considered as published."); - console.log("No argument passed: normal behaviour, that is publish if not already published"); + console.log("TC " + tc_normal + ": Normal case, query respone based on published files. Publish responde with ok/redirect depending on if file is published or not."); + console.log("TC " + tc_none_published + ": Query responde 'ok'. Publish respond with redirect."); + console.log("TC " + tc_all_published + ": Query respond with filename. Publish respond with 'ok'."); + console.log("TC " + tc_10p_no_response + ": 10% % no response for query and publish. Otherwise normal case."); + console.log("TC " + tc_10first_no_response + ": 10 first queries and requests gives no response for query and publish. Otherwise normal case."); + console.log("TC " + tc_100first_no_response + ": 100 first queries and requests gives no response for query and publish. Otherwise normal case."); + console.log("TC " + tc_all_delay_10s + ": All responses delayed 10s (both query and publish)."); + console.log("TC " + tc_10p_delay_10s + ": 10% of responses delayed 10s, (both query and publish)."); + console.log("TC " + tc_10p_error_response + ": 10% error response for query and publish. Otherwise normal case."); + console.log("TC " + tc_10first_error_response + ": 10 first queries and requests gives no response for query and publish. Otherwise normal case."); + console.log("TC " + tc_100first_error_response + ": 100 first queries and requests gives no response for query and publish. Otherwise normal case."); + process.exit(0); -} + } + var bodyParser = require('body-parser') app.use(bodyParser.urlencoded({ extended: false })) @@ -52,55 +112,179 @@ app.use(bodyParser.json()) // parse application/vnd.api+json as json app.use(bodyParser.json({ type: 'application/vnd.api+json' })) -// parse some custom thing into a Buffer -app.use(bodyParser.raw({limit:1024*1024*20, type: 'application/octet-stream' })) +// parse some custom thing into a Buffer (to cater for 60MB files) +app.use(bodyParser.raw({limit:1024*1024*60, type: 'application/octet-stream' })) // parse an HTML body into a string app.use(bodyParser.text({ type: 'text/html' })) + + + +//Is alive function app.get("/",function(req, res){ res.send("ok"); }) +//Counter readout +app.get("/ctr_publish_query",function(req, res){ + res.send(""+ctr_publish_query); +}) +app.get("/ctr_publish_query_published",function(req, res){ + res.send(""+ctr_publish_query_published); +}) +app.get("/ctr_publish_query_not_published",function(req, res){ + res.send(""+ctr_publish_query_not_published); +}) +app.get("/ctr_publish_req",function(req, res){ + res.send(""+ctr_publish_req); +}) +app.get("/ctr_publish_req_redirect",function(req, res){ + res.send(""+ctr_publish_req_redirect); +}) +app.get("/ctr_publish_req_published",function(req, res){ + res.send(""+ctr_publish_req_published); +}) +app.get("/ctr_published_files",function(req, res){ + res.send(""+published.length); +}) +app.get("/tc_info",function(req, res){ + res.send(args.tc); +}) +function fmtMSS(s){ + return(s-(s%=60))/60+(9<s?':':':0')+s //Format time diff in mm:ss +} +app.get("/execution_time",function(req, res){ + diff = fmtMSS(Math.floor((Date.now()-startTime)/1000)); + res.send(""+diff); +}) +//db of published files var published = []; + app.get('/feedlog/1/',function(req, res){ + console.log("url:"+req.url); + ctr_publish_query++; var filename = req.query.filename; - if(args.tc == allPublished){ - res.send("[" + filename + "]"); - } else if(args.tc == nonePublished){ + console.log(filename); + var qtype = req.query.type; + if(typeof(filename) == 'undefined'){ + res.status(400).send({error: 'No filename provided.'}); + return; + } else if(typeof(qtype) == 'undefined'){ + res.status(400).send({error: 'No type provided.'}); + return; + } + + //Ugly fix, plus signs replaces with spaces in query params....need to put them back + filename = filename.replace(/ /g,"+"); + + if (args.tc==tc_normal) { + //continue + } else if (args.tc==tc_none_published) { + ctr_publish_query_not_published++; res.send("[]"); + return; + } else if (args.tc==tc_all_published) { + ctr_publish_query_published++; + res.send("[" + filename + "]"); + return; + } else if (args.tc==tc_10p_no_response && (ctr_publish_query%10) == 0) { + return; + } else if (args.tc==tc_10first_no_response && ctr_publish_query<11) { + return; + } else if (args.tc==tc_100first_no_response && ctr_publish_query<101) { + return; + } else if (args.tc==tc_all_delay_10s) { + console.log("sleep begin"); + timer(10000).then(_=>console.log("sleeping done")); + } else if (args.tc==tc_10p_delay_10s && (ctr_publish_query%10) == 0) { + console.log("sleep begin"); + timer(10000).then(_=>console.log("sleeping done")); + } else if (args.tc==tc_10p_error_response && (ctr_publish_query%10) == 0) { + res.send(400); + return; + } else if (args.tc==tc_10first_error_response && ctr_publish_query<11) { + res.send(400); + return; + } else if (args.tc==tc_100first_error_response & ctr_publish_query<101) { + res.send(400); + return; + } + + if (published.includes(filename)) { + ctr_publish_query_published++; + res.send("[" + filename + "]"); } else { - if (published.includes(filename)) { - res.send("[" + filename + "]"); - } else { - res.send("[]"); - } + ctr_publish_query_not_published++; + res.send("[]"); } }) +app.put('/publish/1/:filename', function (req, res) { + console.log("url:"+req.url); + console.log("body (first 25 bytes):"+req.body.slice(0,25)); + console.log("headers:"+req.headers); + ctr_publish_req++; -app.put('/publish/1/', function (req, res) { - var filename = req.query.filename; - var type = req.query.type; - if(typeof(filename) == 'undefined'){ - res.status(400).send({error: 'No filename provided.'}); - } else if(typeof(type) == 'undefined'){ - res.status(400).send({error: 'No type provided.'}); + var filename = req.params.filename; + console.log(filename); + + if (args.tc==tc_normal) { + //continue + } else if (args.tc==tc_none_published) { + ctr_publish_req_redirect++; + res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename); + return; + } else if (args.tc==tc_all_published) { + ctr_publish_req_published++; + res.send("ok"); + return; + }else if (args.tc==tc_10p_no_response && (ctr_publish_req%10) == 0) { + return; + } else if (args.tc==tc_10first_no_response && ctr_publish_req<11) { + return; + } else if (args.tc==tc_100first_no_response && ctr_publish_req<101) { + return; + } else if (args.tc==tc_all_delay_10s) { + console.log("sleep begin"); + timer(10000).then(_=>console.log("sleeping done")); + } else if (args.tc==tc_10p_delay_10s && (ctr_publish_req%10) == 0) { + console.log("sleep begin"); + timer(10000).then(_=>console.log("sleeping done")); + } else if (args.tc==tc_10p_error_response && (ctr_publish_req%10) == 0) { + res.send(400); + return; + } else if (args.tc==tc_10first_error_response && ctr_publish_req<11) { + res.send(400); + return; + } else if (args.tc==tc_100first_error_response & ctr_publish_req<101) { + res.send(400); + return; + } + + if (!published.includes(filename)) { + ctr_publish_req_redirect++; + res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename); } else { - if(args.tc == allPublished){ - res.send("[" + filename + "]"); - } else if(args.tc == nonePublished){ - res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename); - } else { - if (!published.includes(filename)) { - published.push(filename); - res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename); - } else { - res.send("ok"); - } - } + ctr_publish_req_published++; + res.send("ok"); } }) +//Callback from DR REDIR server, when file is published ok this PUT request update the list of published files. +app.put('/dr_redir_publish/:filename', function (req, res) { + console.log("url:"+req.url); + var filename = req.params.filename; + console.log(filename); + + if (!published.includes(filename)) { + console.log("File marked as published by callback from DR redir SIM. url: " + req.url); + published.push(filename); + } else { + console.log("File already marked as published. Callback from DR redir SIM. url: " + req.url); + } + + res.send("ok"); +}) var httpServer = http.createServer(app); var httpsServer = https.createServer(credentials, app); diff --git a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js index 5be1f689e..4494e899e 100644 --- a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js +++ b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js @@ -6,11 +6,97 @@ const stream = require('stream'); var app = express(); var fs = require("fs"); var path = require('path'); +var ArgumentParser = require('argparse').ArgumentParser; var privateKey = fs.readFileSync('cert/private.key', 'utf8'); var certificate = fs.readFileSync('cert/certificate.crt', 'utf8'); var credentials = {key: privateKey, cert: certificate}; + var bodyParser = require('body-parser') +var startTime = Date.now(); + +var dr_callback_ip = '192.168.100.2'; //IP for DR when running as container. Can be changed by env DR_SIM_IP + +//Counters +var ctr_publish_requests = 0; +var ctr_publish_responses = 0; +var lastPublish = ""; +var dwl_volume = 0; + +var parser = new ArgumentParser({ + version: '0.0.1', + addHelp:true, + description: 'Datarouter redirect simulator' + }); + +parser.addArgument('--tc' , { help: 'TC $NoOfTc' } ); +parser.addArgument('--printtc' , + { + help: 'Print complete usage help', + action: 'storeTrue' + } + ); + +var args = parser.parseArgs(); +const tc_normal = "normal"; +const tc_no_publish ="no_publish" +const tc_10p_no_response = "10p_no_response"; +const tc_10first_no_response = "10first_no_response"; +const tc_100first_no_response = "100first_no_response"; +const tc_all_delay_10s = "all_delay_10s"; +const tc_10p_delay_10s = "10p_delay_10s"; +const tc_10p_error_response = "10p_error_response"; +const tc_10first_error_response = "10first_error_response"; +const tc_100first_error_response = "100first_error_response"; + +if (args.tc==tc_normal) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_no_publish) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10first_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_100first_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_all_delay_10s) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_delay_10s) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_error_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10first_error_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_100first_error_response) { + console.log("TC: " + args.tc) +} else { + console.log("No TC specified, use: --tc <tc-id>"); + process.exit(0); +} + +if (args.printtc) { + console.log("TC " + tc_normal + ": Normal case, all files publish and DR updated"); + console.log("TC " + tc_no_publish + ": Ok response but no files published"); + console.log("TC " + tc_10p_no_response + ": 10% % no response (file not published)"); + console.log("TC " + tc_10first_no_response + ": 10 first requests give no response (files not published)"); + console.log("TC " + tc_100first_no_response + ": 100 first requests give no response (files not published)"); + console.log("TC " + tc_all_delay_10s + ": All responses delayed 10s, normal publish"); + console.log("TC " + tc_10p_delay_10s + ": 10% of responses delayed 10s, normal publish"); + console.log("TC " + tc_10p_error_response + ": 10% error response (file not published)"); + console.log("TC " + tc_10first_error_response + ": 10 first requests give error response (file not published)"); + console.log("TC " + tc_100first_error_response + ": 100 first requests give error responses (file not published)"); + + process.exit(0); +} // parse application/x-www-form-urlencoded app.use(bodyParser.urlencoded({ extended: false })) @@ -22,26 +108,120 @@ app.use(bodyParser.json()) app.use(bodyParser.json({ type: 'application/vnd.api+json' })) // parse some custom thing into a Buffer -app.use(bodyParser.raw({limit:1024*1024*20, type: 'application/octet-stream' })) +app.use(bodyParser.raw({limit:1024*1024*60, type: 'application/octet-stream' })) // parse an HTML body into a string app.use(bodyParser.text({ type: 'text/html' })) + +//Formatting +function fmtMSS(s){ + return(s-(s%=60))/60+(9<s?':':':0')+s //Format time diff to mm:ss +} +function fmtLargeNumber(x) { + return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, " "); //Format large with space, eg: 1 000 000 +} + +//I'm alive function app.get("/",function(req, res){ res.send("ok"); }) +//Counter readout +app.get("/ctr_publish_requests",function(req, res){ + res.send(""+ctr_publish_requests); +}) +app.get("/ctr_publish_responses",function(req, res){ + res.send(""+ctr_publish_responses); +}) +app.get("/execution_time",function(req, res){ + diff = fmtMSS(Math.floor((Date.now()-startTime)/1000)); + res.send(""+diff); +}) +app.get("/time_lastpublish",function(req, res){ + res.send(""+lastPublish); +}) +app.get("/dwl_volume",function(req, res){ + res.send(""+fmtLargeNumber(dwl_volume)); +}) +app.get("/tc_info",function(req, res){ + res.send(args.tc); +}) + app.put('/publish/1/:filename', function (req, res) { - console.log(req.files); - console.log(req.body) + console.log(req.url); + console.log("First 25 bytes of body: " + req.body.slice(0,25)) console.log(req.headers) - var filename = path.basename(req.params.filename); - filename = path.resolve(__dirname, filename); - console.log(req.params.filename); - fs.writeFile(filename, req.body, function (error) { - if (error) { console.error(error); } + ctr_publish_requests++; + if (args.tc == tc_no_publish) { + tr_publish_responses++; + res.send("ok") + return; + } else if (args.tc==tc_10p_no_response && (ctr_publish_requests%10)==0) { + return; + } else if (args.tc==tc_10first_no_response && ctr_publish_requests<11) { + return; + } else if (args.tc==tc_100first_no_response && ctr_publish_requests<101) { + return; + } else if (args.tc==tc_10p_error_response && (ctr_publish_requests%10)==0) { + tr_publish_responses++; + res.send(400, ""); + return; + } else if (args.tc==tc_10first_error_response && ctr_publish_requests<11) { + tr_publish_responses++; + res.send(400, ""); + return; + } else if (args.tc==tc_100first_error_response && ctr_publish_requests<101) { + tr_publish_responses++; + res.send(400, ""); + return; + } else if (args.tc==tc_10p_delay_10s && (ctr_publish_requests%10)==0) { + console.log("sleep begin"); + timer(10000).then(_=>console.log("sleeping done")); + } else if (args.tc==tc_all_delay_10s) { + //var sleep = require('sleep'); + console.log("sleep begin"); + //sleep.sleep(10); + timer(10000).then(_=>console.log("sleeping done")); + } + + //Remaining part if normal file publish + + var filename = req.params.filename; + console.log(filename); + //Create filename (appending file size to name) to store + var storedFilename = path.resolve(__dirname, filename+"-"+req.body.length); + fs.writeFile(storedFilename, "", function (error) { //Store file with zero size + if (error) { console.error(error); } }); - res.send("ok") -}) + + //Make callback to update list of publish files in DR sim + //Note the hard code ip-adress, DR sim get this ip if simulators started from the + //script in the 'simulatorgroup' dir. + //Work around: Could not get a normal http put to work from nodejs, using curl instead + var util = require('util'); + var exec = require('child_process').exec; + + var command = 'curl -s -X PUT http://' + dr_callback_ip + ':3906/dr_redir_publish/' +req.params.filename; + + console.log("Callback to DR sim to report file published, cmd: " + command); + child = exec(command, function(error, stdout, stderr){ + console.log('stdout: ' + stdout); + console.log('stderr: ' + stderr); + if(error !== null) { + console.log('exec error: ' + error); + } + + }); + + //Update status variables + ctr_publish_responses++; + lastPublish = fmtMSS(Math.floor((Date.now()-startTime)/1000)); + dwl_volume = dwl_volume + req.body.length; + + res.send("ok") +}); + + var httpServer = http.createServer(app); var httpsServer = https.createServer(credentials, app); @@ -52,3 +232,7 @@ console.log("DR-simulator listening (http) at "+httpPort) httpsServer.listen(httpsPort); console.log("DR-simulator listening (https) at "+httpsPort) +if (process.env.DR_SIM_IP) { + dr_callback_ip=process.env.DR_SIM_IP; +} +console.log("Using IP " + dr_callback_ip + " for callback to DR sim"); diff --git a/test/mocks/datafilecollector-testharness/mr-sim/README.md b/test/mocks/datafilecollector-testharness/mr-sim/README.md index 57463453f..8fafdfe1f 100644 --- a/test/mocks/datafilecollector-testharness/mr-sim/README.md +++ b/test/mocks/datafilecollector-testharness/mr-sim/README.md @@ -4,11 +4,21 @@ 2. Run the container ```docker-compose up``` The behavior can be changed by argument to the python script in the docker-compose.yml -The simulator can be queried for statistics -localhost:2222/ctr_requests - return an integer of the number of get request to the event poll path -localhost:2222/ctr_responses - return an integer of the number of get responses to the event poll path -localhost:2222/ctr_unique_files - returns an integer or the number of unique files. A unique file is the combination of node+file_sequence_number +The simulator can be queried for statistics (use curl from cmd line or open in browser, curl used below): +`curl localhost:2222/ctr_requests` - return an integer of the number of get request to the event poll path + +`curl localhost:2222/ctr_responses` - return an integer of the number of get responses to the event poll path + +`curl localhost:2222/ctr_unique_files` - returns an integer or the number of unique files. A unique file is the combination of node+file_sequence_number + +`curl localhost:2222/tc_info` - returns the tc string (as given on the cmd line) + +`curl localhost:2222/ctr_events` - returns the total number of events + +`curl localhost:2222/execution_time` - returns the execution time in mm:ss + +`curl localhost:2222/ctr_unique_PNFs` - return the number of unique PNFS in alla events. ##Common TC info File names for 1MB, 5MB and 50MB files @@ -20,33 +30,44 @@ Limited event streams When the number of events are exhausted, empty replies are returned '[]' TC100 - One ME, SFTP, 1 1MB file, 1 event + TC101 - One ME, SFTP, 1 5MB file, 1 event + TC102 - One ME, SFTP, 1 50MB file, 1 event TC110 - One ME, SFTP, 1MB files, 1 file per event, 100 events, 1 event per poll. + TC111 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per poll. + TC112 - One ME, SFTP, 5MB files, 100 files per event, 100 events, 1 event per poll. + TC113 - One ME, SFTP, 1MB files, 100 files per event, 100 events. All events in one poll. TC120 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per poll. 10% of replies each: no response, empty message, slow response, 404-error, malformed json -TC121 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per poll. 10% missing files -TC122 - One ME, SFTP, 1MB files, 100 files per event, 100 events. 1 event per poll. All files with identical name. - -Endless event streams -TC1000 - One ME, SFTP, 1MB files, 100 files per event, endless number of events, 1 event per poll -TC1001 - One ME, SFTP, 5MB files, 100 files per event, endless number of events, 1 event per poll +TC121 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per poll. 10% missing files -TC510 - 5 ME, SFTP, 1MB files, 1 file per event, 100 events, 1 event per poll. +TC122 - One ME, SFTP, 1MB files, 100 files per event, 100 events. 1 event per poll. All files with identical name. +TC510 - 700 MEs, SFTP, 1MB files, 1 file per event, 3500 events, 700 event per poll. TC200-TC202 same as TC100-TC102 but with FTPS + TC210-TC213 same as TC110-TC113 but with FTPS + TC2000-TC2001 same as TC1000-TC1001 but with FTPS + TC610 same as TC510 but with FTPS +Endless event streams + +TC1000 - One ME, SFTP, 1MB files, 100 files per event, endless number of events, 1 event per poll + +TC1001 - One ME, SFTP, 5MB files, 100 files per event, endless number of events, 1 event per poll + + ## Developer workflow 1. ```sudo apt install python3-venv``` diff --git a/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py b/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py index ef46535f5..c1bed8f63 100644 --- a/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py +++ b/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py @@ -3,15 +3,20 @@ import os from werkzeug import secure_filename from flask import Flask, render_template, request from time import sleep +import time import sys import json from flask import Flask + app = Flask(__name__) #Server info HOST_IP = "0.0.0.0" HOST_PORT = 2222 +SFTP_PORT = 1022 +FTPS_PORT = 21 + #Test function to check server running @app.route('/', methods=['GET']) @@ -46,6 +51,30 @@ def testcase_info(): global tc_num return tc_num +#Returns number of events +@app.route('/ctr_events', + methods=['GET']) +def counter_events(): + global ctr_events + return str(ctr_events) + +#Returns number of events +@app.route('/execution_time', + methods=['GET']) +def exe_time(): + global startTime + + stopTime = time.time() + minutes, seconds = divmod(stopTime-startTime, 60) + return "{:0>2}:{:0>2}".format(int(minutes),int(seconds)) + +#Returns number of unique PNFs +@app.route('/ctr_unique_PNFs', + methods=['GET']) +def counter_uniquePNFs(): + global pnfMap + return str(len(pnfMap)) + #Messages polling function @app.route( "/events/unauthenticated.VES_NOTIFICATION_OUTPUT/OpenDcae-c12/C12", @@ -86,7 +115,12 @@ def MR_reply(): return tc1001("sftp") elif args.tc510: - return tc510("sftp") + return tc510("sftp") + elif args.tc511: + return tc511("sftp") + + elif args.tc710: + return tc710("sftp") elif args.tc200: @@ -118,7 +152,12 @@ def MR_reply(): return tc2001("ftps") elif args.tc610: - return tc510("ftps") + return tc510("ftps") + elif args.tc611: + return tc511("ftps") + + elif args.tc810: + return tc710("ftps") #### Test case functions @@ -127,6 +166,7 @@ def MR_reply(): def tc100(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 @@ -134,28 +174,35 @@ def tc100(ftptype): return buildOkResponse("[]") seqNr = (ctr_responses-1) - msg = getEventHead() + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + getEventEnd() + nodeName = createNodeName(0) + fileName = createFileName(nodeName, seqNr, "1MB") + msg = getEventHead(nodeName) + getEventName(fileName,ftptype,"onap","pano") + getEventEnd() fileMap[seqNr] = seqNr + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc101(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 if (ctr_responses > 1): return buildOkResponse("[]") - + seqNr = (ctr_responses-1) - msg = getEventHead() + getEventName("5MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + getEventEnd() + nodeName = createNodeName(0) + fileName = createFileName(nodeName, seqNr, "5MB") + msg = getEventHead(nodeName) + getEventName(fileName,ftptype,"onap","pano") + getEventEnd() fileMap[seqNr] = seqNr - + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc102(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 @@ -163,14 +210,17 @@ def tc102(ftptype): return buildOkResponse("[]") seqNr = (ctr_responses-1) - msg = getEventHead() + getEventName("50MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + getEventEnd() + nodeName = createNodeName(0) + fileName = createFileName(nodeName, seqNr, "50MB") + msg = getEventHead(nodeName) + getEventName(fileName,ftptype,"onap","pano") + getEventEnd() fileMap[seqNr] = seqNr - + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc110(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 @@ -178,75 +228,89 @@ def tc110(ftptype): return buildOkResponse("[]") seqNr = (ctr_responses-1) - msg = getEventHead() + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + getEventEnd() + nodeName = createNodeName(0) + fileName = createFileName(nodeName, seqNr, "1MB") + msg = getEventHead(nodeName) + getEventName(fileName,ftptype,"onap","pano") + getEventEnd() fileMap[seqNr] = seqNr - + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc111(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 if (ctr_responses > 100): return buildOkResponse("[]") - - msg = getEventHead() + + nodeName = createNodeName(0) + msg = getEventHead(nodeName) for i in range(100): seqNr = i+(ctr_responses-1) if i != 0: msg = msg + "," - msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + fileName = createFileName(nodeName, seqNr, "1MB") + msg = msg + getEventName(fileName,ftptype,"onap","pano") fileMap[seqNr] = seqNr msg = msg + getEventEnd() + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc112(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 if (ctr_responses > 100): return buildOkResponse("[]") - - msg = getEventHead() + + nodeName = createNodeName(0) + msg = getEventHead(nodeName) for i in range(100): seqNr = i+(ctr_responses-1) if i != 0: msg = msg + "," - msg = msg + getEventName("5MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + fileName = createFileName(nodeName, seqNr, "5MB") + msg = msg + getEventName(fileName,ftptype,"onap","pano") fileMap[seqNr] = seqNr msg = msg + getEventEnd() + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc113(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 if (ctr_responses > 1): return buildOkResponse("[]") - + + nodeName = createNodeName(0) msg = "" for evts in range(100): # build 100 evts if (evts > 0): msg = msg + "," - msg = msg + getEventHead() + msg = msg + getEventHead(nodeName) for i in range(100): # build 100 files seqNr = i+evts+100*(ctr_responses-1) if i != 0: msg = msg + "," - msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + fileName = createFileName(nodeName, seqNr, "1MB") + msg = msg + getEventName(fileName,ftptype,"onap","pano") fileMap[seqNr] = seqNr msg = msg + getEventEnd() + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") @@ -254,20 +318,23 @@ def tc113(ftptype): def tc120(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 + nodeName = createNodeName(0) + if (ctr_responses > 100): return buildOkResponse("[]") if (ctr_responses % 10 == 2): return # Return nothing - + if (ctr_responses % 10 == 3): return buildOkResponse("") # Return empty message if (ctr_responses % 10 == 4): - return buildOkResponse(getEventHead()) # Return part of a json event + return buildOkResponse(getEventHead(nodeName)) # Return part of a json event if (ctr_responses % 10 == 5): return buildEmptyResponse(404) # Return empty message with status code @@ -275,64 +342,72 @@ def tc120(ftptype): if (ctr_responses % 10 == 6): sleep(60) - - msg = getEventHead() + + msg = getEventHead(nodeName) for i in range(100): seqNr = i+(ctr_responses-1) if i != 0: msg = msg + "," - msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + fileName = createFileName(nodeName, seqNr, "1MB") + msg = msg + getEventName(fileName,ftptype,"onap","pano") fileMap[seqNr] = seqNr msg = msg + getEventEnd() + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc121(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 if (ctr_responses > 100): - return buildOkResponse("[]") - - msg = getEventHead() + return buildOkResponse("[]") + + nodeName = createNodeName(0) + msg = getEventHead(nodeName) + fileName = "" for i in range(100): seqNr = i+(ctr_responses-1) if (seqNr%10 == 0): # Every 10th file is "missing" - fn = "MissingFile_" + str(seqNr) + ".tar.gz" + fileName = createMissingFileName(nodeName, seqNr, "1MB") else: - fn = "1MB_" + str(seqNr) + ".tar.gz" + fileName = createFileName(nodeName, seqNr, "1MB") fileMap[seqNr] = seqNr if i != 0: msg = msg + "," - msg = msg + getEventName(fn,ftptype,"onap","pano","localhost",1022) - + msg = msg + getEventName(fileName,ftptype,"onap","pano") msg = msg + getEventEnd() + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc122(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 if (ctr_responses > 100): return buildOkResponse("[]") - - msg = getEventHead() + + nodeName = createNodeName(0) + msg = getEventHead(nodeName) for i in range(100): - fn = "1MB_0.tar.gz" # All files identical names + fileName = createFileName(nodeName, 0, "1MB") # All files identical names if i != 0: msg = msg + "," - msg = msg + getEventName(fn,ftptype,"onap","pano","localhost",1022) + msg = msg + getEventName(fileName,ftptype,"onap","pano") fileMap[0] = 0 msg = msg + getEventEnd() + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") @@ -340,42 +415,51 @@ def tc122(ftptype): def tc1000(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 - msg = getEventHead() + nodeName = createNodeName(0) + msg = getEventHead(nodeName) for i in range(100): seqNr = i+(ctr_responses-1) if i != 0: msg = msg + "," - msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + fileName = createFileName(nodeName, seqNr, "1MB") + msg = msg + getEventName(fileName,ftptype,"onap","pano") fileMap[seqNr] = seqNr msg = msg + getEventEnd() + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc1001(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 - msg = getEventHead() + nodeName = createNodeName(0) + msg = getEventHead(nodeName) for i in range(100): seqNr = i+(ctr_responses-1) if i != 0: msg = msg + "," - msg = msg + getEventName("5MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + fileName = createFileName(nodeName, seqNr, "5MB") + msg = msg + getEventName(fileName,ftptype,"onap","pano") fileMap[seqNr] = seqNr msg = msg + getEventEnd() + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") def tc510(ftptype): global ctr_responses global ctr_unique_files + global ctr_events ctr_responses = ctr_responses + 1 @@ -384,18 +468,82 @@ def tc510(ftptype): msg = "" - for evts in range(700): # build events for 5 MEs - if (evts > 0): + for pnfs in range(700): # build events for 700 MEs + if (pnfs > 0): + msg = msg + "," + nodeName = createNodeName(pnfs) + msg = msg + getEventHead(nodeName) + seqNr = (ctr_responses-1) + fileName = createFileName(nodeName, seqNr, "1MB") + msg = msg + getEventName(fileName,ftptype,"onap","pano") + seqNr = seqNr + pnfs*1000000 #Create unique id for this node and file + fileMap[seqNr] = seqNr + msg = msg + getEventEnd() + ctr_events = ctr_events+1 + + return buildOkResponse("["+msg+"]") + +def tc511(ftptype): + global ctr_responses + global ctr_unique_files + global ctr_events + + ctr_responses = ctr_responses + 1 + + if (ctr_responses > 5): + return buildOkResponse("[]") + + msg = "" + + for pnfs in range(700): # build events for 700 MEs + if (pnfs > 0): msg = msg + "," - msg = msg + getEventHeadNodeName("PNF"+str(evts)) + nodeName = createNodeName(pnfs) + msg = msg + getEventHead(nodeName) seqNr = (ctr_responses-1) - msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) - seqNr = seqNr + evts*1000000 #Create unique id for this node and file + fileName = createFileName(nodeName, seqNr, "1KB") + msg = msg + getEventName(fileName,ftptype,"onap","pano") + seqNr = seqNr + pnfs*1000000 #Create unique id for this node and file fileMap[seqNr] = seqNr msg = msg + getEventEnd() + ctr_events = ctr_events+1 return buildOkResponse("["+msg+"]") +def tc710(ftptype): + global ctr_responses + global ctr_unique_files + global ctr_events + + ctr_responses = ctr_responses + 1 + + if (ctr_responses > 100): + return buildOkResponse("[]") + + msg = "" + + batch = (ctr_responses-1)%20; + + for pnfs in range(35): # build events for 35 PNFs at a time. 20 batches -> 700 + if (pnfs > 0): + msg = msg + "," + nodeName = createNodeName(pnfs + batch*35) + msg = msg + getEventHead(nodeName) + + for i in range(100): # 100 files per event + seqNr = i + int((ctr_responses-1)/20); + if i != 0: msg = msg + "," + fileName = createFileName(nodeName, seqNr, "1MB") + msg = msg + getEventName(fileName,ftptype,"onap","pano") + seqNr = seqNr + (pnfs+batch*35)*1000000 #Create unique id for this node and file + fileMap[seqNr] = seqNr + + msg = msg + getEventEnd() + ctr_events = ctr_events+1 + + return buildOkResponse("["+msg+"]") + + #Mapping FTPS TCs def tc200(ftptype): return tc100(ftptype) @@ -419,6 +567,14 @@ def tc221(ftptype): return tc121(ftptype) def tc222(ftptype): return tc122(ftptype) + +def tc610(ftptype): + return tc510(ftptype) +def tc611(ftptype): + return tc511(ftptype) + +def tc810(ftptype): + return tc710(ftptype) def tc2000(ftptype): return tc1000(ftptype) @@ -427,11 +583,21 @@ def tc2001(ftptype): #### Functions to build json messages and respones #### +def createNodeName(index): + return "PNF"+str(index); + +def createFileName(nodeName, index, size): + return "A20000626.2315+0200-2330+0200_" + nodeName + "-" + str(index) + "-" +size + ".tar.gz"; + +def createMissingFileName(nodeName, index, size): + return "AMissingFile_" + nodeName + "-" + str(index) + "-" +size + ".tar.gz"; + + # Function to build fixed beginning of an event -def getEventHead(): - return getEventHeadNodeName("oteNB5309") -def getEventHeadNodeName(nodename): +def getEventHead(nodename): + global pnfMap + pnfMap.add(nodename) headStr = """ { "event": { @@ -461,7 +627,13 @@ def getEventHeadNodeName(nodename): return headStr # Function to build the variable part of an event -def getEventName(fn,type,user,passwd,ip,port): +def getEventName(fn,type,user,passwd): + port = SFTP_PORT + ip = sftp_ip + if (type == "ftps"): + port = FTPS_PORT + ip = ftps_ip + nameStr = """{ "name": \"""" + fn + """", "hashMap": { @@ -501,15 +673,26 @@ def buildEmptyResponse(status_code): if __name__ == "__main__": - + + # IP addresses to use for ftp servers, using localhost if not env var is set + sftp_ip = os.environ.get('SFTP_SIM_IP', 'localhost') + ftps_ip = os.environ.get('FTPS_SIM_IP', 'localhost') + + + #Counters ctr_responses = 0 ctr_requests = 0 ctr_unique_files = 0 + ctr_events = 0 + startTime = time.time() - #Keeps all reponded file names + #Keeps all responded file names fileMap = {} + #Keeps all responded PNF names + pnfMap = set() + tc_num = "Not set" tc_help = "Not set" @@ -572,9 +755,17 @@ if __name__ == "__main__": parser.add_argument( '--tc510', action='store_true', - help='TC510 - 5 MEs, SFTP, 1MB files, 1 file per event, 100 events, 1 event per poll.') - + help='TC510 - 700 MEs, SFTP, 1MB files, 1 file per event, 3500 events, 700 event per poll.') + + parser.add_argument( + '--tc511', + action='store_true', + help='TC511 - 700 MEs, SFTP, 1KB files, 1 file per event, 3500 events, 700 event per poll.') + parser.add_argument( + '--tc710', + action='store_true', + help='TC710 - 700 MEs, SFTP, 1MB files, 100 files per event, 3500 events, 35 event per poll.') # FTPS TCs with single ME parser.add_argument( @@ -632,11 +823,21 @@ if __name__ == "__main__": parser.add_argument( '--tc610', action='store_true', - help='TC510 - 5 MEs, FTPS, 1MB files, 1 file per event, 100 events, 1 event per poll.') + help='TC610 - 700 MEs, FTPS, 1MB files, 1 file per event, 3500 events, 700 event per poll.') + + parser.add_argument( + '--tc611', + action='store_true', + help='TC611 - 700 MEs, FTPS, 1KB files, 1 file per event, 3500 events, 700 event per poll.') + + parser.add_argument( + '--tc810', + action='store_true', + help='TC810 - 700 MEs, FTPS, 1MB files, 100 files per event, 3500 events, 35 event per poll.') args = parser.parse_args() - + if args.tc100: tc_num = "TC# 100" @@ -668,6 +869,11 @@ if __name__ == "__main__": elif args.tc510: tc_num = "TC# 510" + elif args.tc511: + tc_num = "TC# 511" + + elif args.tc710: + tc_num = "TC# 710" elif args.tc200: tc_num = "TC# 200" @@ -697,14 +903,25 @@ if __name__ == "__main__": elif args.tc2001: tc_num = "TC# 2001" + elif args.tc610: tc_num = "TC# 610" + elif args.tc611: + tc_num = "TC# 611" + + elif args.tc810: + tc_num = "TC# 810" else: print("No TC was defined") print("use --help for usage info") sys.exit() - print(tc_num) - + print("TC num: " + tc_num) + + + print("Using " + sftp_ip + " for sftp server address in file urls.") + print("Using " + ftps_ip + " for ftps server address in file urls.") + app.run(port=HOST_PORT, host=HOST_IP) + diff --git a/test/mocks/datafilecollector-testharness/simulator-group/README.md b/test/mocks/datafilecollector-testharness/simulator-group/README.md index 4d448f8d2..5981c7978 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/README.md +++ b/test/mocks/datafilecollector-testharness/simulator-group/README.md @@ -4,15 +4,17 @@ Mainly this is needed for CSIT tests but can be used also for local testing. ###Preparation +Do the manual steps to prepare the simulator images + Build the mr-sim image. cd ../mr-sim -Run the docker build command to build the image for the MR simulator: 'docker build -t mrsim:latest . +Run the docker build command to build the image for the MR simulator: 'docker build -t mrsim:latest .' cd ../dr-sim -Run the docker build command to build the image for the DR simulators: `docker build -t drsim_common:latest . +Run the docker build command to build the image for the DR simulators: `docker build -t drsim_common:latest .' cd ../simulator-group @@ -21,21 +23,53 @@ Copy the 'configuration' and 'tls' catalogues from the ftps-sftp-server dir. Check the README.md in ftps-sftp-server dir in case the cert need to be updated. -cp -r ./ftps-sftp-server/configuration . +cp -r ../ftps-sftp-server/configuration . cp -r ../ftps-sftp-server/tls . ###Execution -Edit the `docker-compose-setup.sh` to setup the env variables to the desired test behavior for each simulators. +Edit the `docker-compose-setup.sh` (or create a copy) to setup the env variables to the desired test behavior for each simulators. See each simulator to find a description of the available settings. +The following env variables shall be set (example values). +Note that NUM_FTPFILES and NUM_PNFS controls the number of ftp files created in the ftp servers. +A total of NUM_FTPFILES * NUM_PNFS ftp files will be created in each dtp server (4 files in the below example). +Large settings will be time consuming at start of the servers. + +DR_TC="--tc normal" #Normal behavior of the DR sim + +DR_REDIR_TC="--tc normal" #Normal behavior of the DR redirect sim + +MR_TC="--tc100" #One 1 MB file in one event, once. + +BC_TC="" #Not in use yet + +NUM_FTPFILES="2" #Two file for each PNF -Run the script `docker-compose-setup.sh`to create a docker-compose with the desired settings. All simulators -will be started with the generated docker-compose.yml file +NUM_PNFS="2" #Two PNFs + +Run the script `docker-compose-setup.sh`to create a docker-compose with the desired settings. The desired setting +in the script need to be manually adapted to for each specific simulator behavior according to the above. Check each simulator for available +parameters. +All simulators will be started with the generated docker-compose.yml file + +To generate ftp url with IP different from localhost, set SFTP_SIM_IP and/or FTPS_SIM_IP env variables to the addreses of the ftp servers before starting. +So farm, this only works when the simulator python script is started from the command line. Kill all the containers with `simulators-kill.se` `simulators_start.sh` is for CSIT test and requires the env variables for test setting to be present in the shell. `setup-ftp-files.for-image.sh` is for CSIT and executed when the ftp servers are started from the docker-compose-setup.sh`. +To make DFC to be able to connect to the simulator containers, DFC need to run in host mode. +Start DFC by the following cmd: `docker run -d --network="host" --name dfc_app <dfc-image> ` + +`<dfc-image>` could be either the locally built image `onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server` +or the one in nexus `nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server`. + + + +###Simulator monitor +Start the simulator monitor server with `sim-monitor-start.sh` and the open a browser with the url `localhost:9999/mon` +to see the statisics page with data from MR sim, DR sim and DR redir sim. diff --git a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh index d0171a5e3..af36d0562 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh @@ -2,10 +2,13 @@ #Script for manually starting all simulators with test setting below -export DR_TC="" -export DR_REDIR_TC="" +export DR_TC="--tc normal" +export DR_REDIR_TC="--tc normal" export MR_TC="--tc100" -export BC_TC="" -export NUM_FTPFILES="250" +export BC_TC="" #Not in use yet +export NUM_FTPFILES="10" +export NUM_PNFS="700" +export FILE_SIZE="1MB" +export FTP_TYPE="SFTP" source ./simulators-start.sh
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml index e7c700759..89a45a476 100644 --- a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml +++ b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml @@ -1,8 +1,17 @@ version: '2' + +networks: + dfcnet: + ipam: + config: + - subnet: 192.168.100.0/16 + services: drsim: - network_mode: "host" + networks: + dfcnet: + ipv4_address: 192.168.100.2 image: drsim_common:latest ports: - "3906:3906" @@ -11,6 +20,9 @@ services: command: node dmaapDR.js ${DR_TC} drsim_redir: + networks: + dfcnet: + ipv4_address: 192.168.100.3 image: drsim_common:latest ports: - "3908:3908" @@ -19,6 +31,9 @@ services: command: node dmaapDR_redir.js ${DR_REDIR_TC} mrsim: + networks: + dfcnet: + ipv4_address: 192.168.100.1 image: mrsim:latest ports: - "2222:2222" @@ -26,17 +41,17 @@ services: command: python mr-sim.py ${MR_TC} sftp-server: + network_mode: bridge container_name: dfc_sftp-server image: atmoz/sftp:alpine ports: - "1022:22" -# volumes: -# - ./files/onap/sftp/:/home/onap/ restart: on-failure command: onap:pano:1001 ftpes-server-vsftpd: + network_mode: bridge container_name: dfc_ftpes-server-vsftpd image: docker.io/panubo/vsftpd ports: @@ -54,6 +69,5 @@ services: - ./tls/dfc.crt:/etc/ssl/private/dfc.crt:ro - ./configuration/vsftpd_ssl.conf:/etc/vsftpd_ssl.conf:ro -# - ./files/onap/ftps/:/srv/ restart: on-failure
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh b/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh index d8e57a3d8..6c2b4f29f 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh @@ -1,28 +1,47 @@ -#!/bin/bash +#!/usr/bin/env bash # Script to create files for the FTP server to return upon request. # The file names matches the files names in the events polled from the MR simulator. # Intended for execution in the running ftp containers in the ftp-root dir. -echo "Running ftp file creations" - NUM=200 #Default number of files +PNFS=1 #Default number of PNFs +FSIZE="ALL" if [ $# -eq 1 ]; then NUM=$1 +elif [ $# -eq 2 ]; then + NUM=$1 + PNFS=$2 +elif [ $# -eq 3 ]; then + NUM=$1 + PNFS=$2 + FSIZE=$3 + if [ $3 != "1KB" ] && [ $3 != "1MB" ] && [ $3 != "5MB" ] && [ $3 != "50MB" ] && [ $3 != "ALL" ]; then + echo "File size shall be 1KB|1MB|5MB|50MB|ALL" + exit + fi +else + echo "Wrong args, usage: setup-ftp-files-for-image.sh [ <num-files> [ <num-PNFs> [ 1KB|1MB|5MB|50MB ] ] ]" + exit fi +echo "Running ftp file creations. " $PNFS " PNFs and " $NUM " files for each PNF with file size(s) "$FSIZE + +truncate -s 1KB 1KB.tar.gz truncate -s 1MB 1MB.tar.gz truncate -s 5MB 5MB.tar.gz truncate -s 50MB 50MB.tar.gz - -i=0 -while [ $i -lt $NUM ]; do #Problem with for loop and var substituion in curly bracket....so used good old style loop - ln -s 1MB.tar.gz 1MB_$i.tar.gz - ln -s 5MB.tar.gz 5MB_$i.tar.gz - let i=i+1 +p=0 +while [ $p -lt $PNFS ]; do + i=0 + while [ $i -lt $NUM ]; do #Problem with for loop and var substituion in curly bracket....so used good old style loop + if [ $FSIZE = "ALL" ] || [ $FSIZE = "1KB" ]; then ln -s 1KB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-1KB.tar.gz'; fi + if [ $FSIZE = "ALL" ] || [ $FSIZE = "1MB" ]; then ln -s 1MB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-1MB.tar.gz'; fi + if [ $FSIZE = "ALL" ] || [ $FSIZE = "5MB" ]; then ln -s 5MB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-5MB.tar.gz'; fi + if [ $FSIZE = "ALL" ] || [ $FSIZE = "50MB" ]; then ln -s 50MB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-50MB.tar.gz'; fi + let i=i+1 + done + let p=p+1 done - - -ln -s 50MB.tar.gz 50MB_0.tar.gz #Large file, only for single file test diff --git a/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor-start.sh b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor-start.sh new file mode 100755 index 000000000..52c8c1cac --- /dev/null +++ b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor-start.sh @@ -0,0 +1,7 @@ +#/bin/bash + +#Script to start the sim-monitor + +#Re-using modules for dr-sim +cp -r ../dr-sim/node_modules . +node sim-monitor.js
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor.js b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor.js new file mode 100644 index 000000000..e4a19c21e --- /dev/null +++ b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor.js @@ -0,0 +1,165 @@ +var http = require('http'); + +var express = require('express'); +var app = express(); + +//I am alive +app.get("/",function(req, res){ + res.send("ok"); +}) + +//Get parameter valuye from other server +function getSimCtr(url, cb) { + var data = ''; + http.get(url, (resp) => { + // A chunk of data has been recieved. + resp.on('data', (chunk) => { + data += chunk; + }); + + // The whole response has been received. + resp.on('end', () => { + //Pad data to fixed length + var i = 20-data.length; + while(i>0) { + data = data+" "; + i--; + } + cb(data); + }); + + }).on("error", (err) => { + console.log("Error: " + err.message); + cb("no response from simulator"); + }); +}; + +//Status variables, for parameters values fetched from other simulators +var mr1, mr2, mr3, mr4, mr5, mr6, mr7; + +var dr1, dr2, dr3, dr4, dr5, dr6, dr7, dr8, dr9; + +var drr1, drr2, drr3, drr4, drr5, drr6; + +app.get("/mon",function(req, res){ + + //MR + getSimCtr("http://127.0.0.1:2222/ctr_requests", function(data) { + mr1 = data; + }); + getSimCtr("http://127.0.0.1:2222/ctr_responses", function(data) { + mr2 = data; + }); + getSimCtr("http://127.0.0.1:2222/ctr_unique_files", function(data) { + mr3 = data; + }); + getSimCtr("http://127.0.0.1:2222/tc_info", function(data) { + mr4 = data; + }); + getSimCtr("http://127.0.0.1:2222/ctr_events", function(data) { + mr5 = data; + }); + getSimCtr("http://127.0.0.1:2222/execution_time", function(data) { + mr6 = data; + }); + getSimCtr("http://127.0.0.1:2222/ctr_unique_PNFs", function(data) { + mr7 = data; + }); + + //DR + getSimCtr("http://127.0.0.1:3906/ctr_publish_query", function(data) { + dr1 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_query_published", function(data) { + dr2 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_query_not_published", function(data) { + dr3 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_req", function(data) { + dr4 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_req_redirect", function(data) { + dr5 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_publish_req_published", function(data) { + dr6 = data; + }); + getSimCtr("http://127.0.0.1:3906/ctr_published_files", function(data) { + dr7 = data; + }); + getSimCtr("http://127.0.0.1:3906/tc_info", function(data) { + dr8 = data; + }); + getSimCtr("http://127.0.0.1:3906/execution_time", function(data) { + dr9 = data; + }); + + //DR REDIR + getSimCtr("http://127.0.0.1:3908/ctr_publish_requests", function(data) { + drr1 = data; + }); + getSimCtr("http://127.0.0.1:3908/ctr_publish_responses", function(data) { + drr2 = data; + }); + getSimCtr("http://127.0.0.1:3908/tc_info", function(data) { + drr3 = data; + }); + getSimCtr("http://127.0.0.1:3908/execution_time", function(data) { + drr4 = data; + }); + getSimCtr("http://127.0.0.1:3908/time_lastpublish", function(data) { + drr5 = data; + }); + getSimCtr("http://127.0.0.1:3908/dwl_volume", function(data) { + drr6 = data; + }); + + //Build web page + var str = "<!DOCTYPE html>" + + "<html>" + + "<head>" + + "<meta http-equiv=\"refresh\" content=\"5\">"+ //5 sec auto reefresh + "<title>Simulator monitor</title>"+ + "</head>" + + "<body>" + + "<h3>MR Simulator</h3>" + + "<font face=\"courier\">"+ + "MR TC:........................................." + mr4 + "<br>" + + "Execution time (mm.ss):........................" + mr6 + "<br>" + + "Number of requests (polls):...................." + mr1 + "<br>" + + "Number of responses (polls):..................." + mr2 + "<br>" + + "Number of unique files in all responses:......." + mr3 + "<br>" + + "Number of events..............................." + mr5 + "<br>" + + "Number of unique PNFs.........................." + mr7 + "<br>" + + "</font>"+ + "<h3>DR Simulator</h3>" + + "<font face=\"courier\">"+ + "DR TC:........................................." + dr8 + "<br>" + + "Execution time (mm.ss):........................" + dr9 + "<br>" + + "Number of queries:............................." + dr1 + "<br>" + + "Number of query responses, file published:....." + dr2 + "<br>" + + "Number of query responses, file not published:." + dr3 + "<br>" + + "Number of requests:............................" + dr4 + "<br>" + + "Number of responses with redirect:............." + dr5 + "<br>" + + "Number of responses without redirect:.........." + dr6 + "<br>" + + "Number of published files:....................." + dr7 + "<br>" + + "</font>"+ + "<h3>DR Redirect Simulator</h3>" + + "<font face=\"courier\">"+ + "DR REDIR TC:..................................." + drr3 + "<br>" + + "Execution time (mm.ss):........................" + drr4 + "<br>" + + "Number of requests:............................" + drr1 + "<br>" + + "Number of responses:..........................." + drr2 + "<br>" + + "Downloaded volume (bytes):....................." + drr6 + "<br>" + + "Last publish (mm:ss):.........................." + drr5 + "<br>" + + "</font>"+ + "</body>" + + "</html>"; + res.send(str); +}) + +var httpServer = http.createServer(app); +var httpPort=9999; +httpServer.listen(httpPort); +console.log("Simulator monitor listening (http) at "+httpPort);
\ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh b/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh index ef1b90a38..cc5ce32ec 100755 --- a/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh +++ b/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh @@ -35,11 +35,27 @@ if [ -z "$NUM_FTPFILES" ] then NUM_FTPFILES=200 fi +if [ -z "$NUM_PNFS" ] + then + NUM_PNFS=1 +fi +if [ -z "$FILE_SIZE" ] + then + FILE_SIZE="ALL" +fi +if [ -z "$FTP_TYPE" ] + then + FTP_TYPE="ALL" +fi - -docker cp setup-ftp-files-for-image.sh $SFTP_SIM:/tmp/ -docker exec -w /home/onap/ $SFTP_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES >/dev/null 2>&1 - -docker cp setup-ftp-files-for-image.sh $FTPS_SIM:/tmp/setup-ftp-files-for-image.sh -docker exec -w /srv $FTPS_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES >/dev/null 2>&1 - +if [ $FTP_TYPE = "ALL" ] || [ $FTP_TYPE = "SFTP" ]; then + echo "Creating files for SFTP server, may take time...." + docker cp setup-ftp-files-for-image.sh $SFTP_SIM:/tmp/ + docker exec -w /home/onap/ $SFTP_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES $NUM_PNFS $FILE_SIZE #>/dev/null 2>&1 +fi +if [ $FTP_TYPE = "ALL" ] || [ $FTP_TYPE = "FTPS" ]; then + echo "Creating files for FTPS server, may take time...." + docker cp setup-ftp-files-for-image.sh $FTPS_SIM:/tmp/setup-ftp-files-for-image.sh + docker exec -w /srv $FTPS_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES $NUM_PNFS $FILE_SIZE #>/dev/null 2>&1 +fi +echo "Done: All simulators started and configured" |