From f4e1836eab2cff83bc2542383ed7f97d98a7b68c Mon Sep 17 00:00:00 2001 From: BjornMagnussonXA Date: Wed, 10 Apr 2019 13:04:08 +0000 Subject: Enhancements of simulators for DFC New test cases for MR and DR simulators Added simulator monitor server for simulator supervision Update - Adapted simulators to changed naming of ftp files Update2 - Added some more test behaviour to simulators. Issue-ID: DCAEGEN2-1313 Change-Id: I90a346d44ab4e7c9f5a65f599b64f1907525dd51 Signed-off-by: BjornMagnussonXA --- .../dr-sim/Dockerfile | 11 +- .../datafilecollector-testharness/dr-sim/README.md | 50 +++- .../dr-sim/dmaapDR.js | 268 +++++++++++++++++---- .../dr-sim/dmaapDR_redir.js | 204 +++++++++++++++- 4 files changed, 474 insertions(+), 59 deletions(-) (limited to 'test/mocks/datafilecollector-testharness/dr-sim') diff --git a/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile b/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile index fc903d7e7..3d95492a8 100644 --- a/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile +++ b/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile @@ -12,7 +12,10 @@ COPY cert/ cert/ RUN npm install express RUN npm install argparse -EXPOSE 3906 -EXPOSE 3907 -EXPOSE 3908 -EXPOSE 3909 \ No newline at end of file +#Ports for DR +#EXPOSE 3906 +#EXPOSE 3907 + +#Ports for DR redir +#EXPOSE 3908 +#EXPOSE 3909 \ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/dr-sim/README.md b/test/mocks/datafilecollector-testharness/dr-sim/README.md index f0cdf589b..8761d0cef 100644 --- a/test/mocks/datafilecollector-testharness/dr-sim/README.md +++ b/test/mocks/datafilecollector-testharness/dr-sim/README.md @@ -1,6 +1,9 @@ -#Alternative to running python (as described below) on your machine, use the docker files. +###Alternative to running python (as described below) on your machine, use the docker files. 1. Build docker container with ```docker build -t drsim_common:latest .``` 2. Run the container ```docker-compose up``` +3. For specific behavior of of the simulators, add arguments to the `command` entries in the `docker-compose.yml`. +For example `command: node dmaapDR.js --tc no_publish` . (No argument will assume '--tc normal'). Run `node dmaapDR.js --printtc` +and `node dmaapDR-redir.js --printtc` for details. @@ -9,5 +12,46 @@ Make sure that you run these commands in the application directory "dr-sim" 3. `npm install express` 4. `npm install argparse` -5. `node dmaapDR.js` #keep it in the foreground -6. `node dmaapDR_redir.js` #keep it in the foreground +5. `node dmaapDR.js` #keep it in the foreground, see item 3 in the above list for arg to the simulator +6. `node dmaapDR_redir.js` #keep it in the foreground, see item 3 in the above list for arg to the simulator + + +The dmaapDR_redir server send callback to dmaapDR server to update the list of successfully published files. +As default, the ip for dmaapDR is set to work when running as container (using an ip address from the dfc_net docker network) . When running the servers from command line, set the env variable DR_SIM_IP=localhost + +The simulator can be queried for statistics (use curl from cmd line or open in browser, curl used below): + +DR + +`curl localhost:3906/ctr_publish_query` - returns the number of publish queries + +`curl localhost:3906/ctr_publish_query_published` - returns the number of responses where the files was published + +`curl localhost:3906/ctr_publish_query_not_published` - returns the number of responses where the file was not published + +`curl localhost:3906/ctr_publish_req` - returns the number of publish requests + +`curl localhost:3906/ctr_publish_req_redirect` - returns the number of publish responses with redirect + +`curl localhost:3906/ctr_publish_req_published` - returns the number of publish responses without redirect + +`curl localhost:3906/ctr_published_files` - returns the number of unique published files + +`curl localhost:3906/tc_info` - returns the tc name (argument on the command line) + +`curl localhost:3906/execution_time` - returns the execution times in mm:ss + + +DR REDIR + +`curl localhost:3908/ctr_publish_requests` - returns the number of publish queries + +`curl localhost:3908/ctr_publish_responses` - returns the number of publish responses + +`curl localhost:3908/tc_info` - returns the tc name (argument on the command line) + +`curl localhost:3908/execution_time` - returns the execution times in mm:ss + +`curl localhost:3908/time_lastpublish` - returns the time (mm:ss) for the latest publish + +`curl localhost:3908/dwl_volume` - returns the total received data volume of file data diff --git a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js index 4e7317473..fffe57ce3 100644 --- a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js +++ b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js @@ -5,12 +5,33 @@ var express = require('express'); const stream = require('stream'); var app = express(); var fs = require('fs'); -var path = require('path'); var privateKey = fs.readFileSync('cert/private.key', 'utf8'); var certificate = fs.readFileSync('cert/certificate.crt', 'utf8'); var credentials = {key: privateKey, cert: certificate}; -const allPublished = "allPublished"; -const nonePublished = "nonePublished"; + +//For execution time calculation +var startTime = Date.now(); + +//Test case constants +const tc_normal = "normal"; +const tc_none_published = "none_published"; +const tc_all_published = "all_published" +const tc_10p_no_response = "10p_no_response"; +const tc_10first_no_response = "10first_no_response"; +const tc_100first_no_response = "100first_no_response"; +const tc_all_delay_10s = "all_delay_10s"; +const tc_10p_delay_10s = "10p_delay_10s"; +const tc_10p_error_response = "10p_error_response"; +const tc_10first_error_response = "10first_error_response"; +const tc_100first_error_response = "100first_error_response"; + +//Counters +var ctr_publish_query = 0; +var ctr_publish_query_published = 0; +var ctr_publish_query_not_published = 0; +var ctr_publish_req = 0; +var ctr_publish_req_redirect = 0; +var ctr_publish_req_published = 0; var parser = new ArgumentParser({ version: '0.0.1', @@ -28,20 +49,59 @@ parser.addArgument('--printtc' , var args = parser.parseArgs(); -if (args.tc=="nonePublished") { - console.log("TC: nonePublished") -} -if (args.tc=="allPublished") { - console.log("TC: allPublished") - //preparations +if (args.tc==tc_normal) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_none_published) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_all_published) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10first_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_100first_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_all_delay_10s) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_delay_10s) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_error_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10first_error_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_100first_error_response) { + console.log("TC: " + args.tc) +} else { + console.log("No TC specified, use: --tc "); + process.exit(0); } if (args.printtc) { - console.log("TC nonePublished: no file has already been published."); - console.log("TC allPublished: whatever is the request, this file is considered as published."); - console.log("No argument passed: normal behaviour, that is publish if not already published"); + console.log("TC " + tc_normal + ": Normal case, query respone based on published files. Publish responde with ok/redirect depending on if file is published or not."); + console.log("TC " + tc_none_published + ": Query responde 'ok'. Publish respond with redirect."); + console.log("TC " + tc_all_published + ": Query respond with filename. Publish respond with 'ok'."); + console.log("TC " + tc_10p_no_response + ": 10% % no response for query and publish. Otherwise normal case."); + console.log("TC " + tc_10first_no_response + ": 10 first queries and requests gives no response for query and publish. Otherwise normal case."); + console.log("TC " + tc_100first_no_response + ": 100 first queries and requests gives no response for query and publish. Otherwise normal case."); + console.log("TC " + tc_all_delay_10s + ": All responses delayed 10s (both query and publish)."); + console.log("TC " + tc_10p_delay_10s + ": 10% of responses delayed 10s, (both query and publish)."); + console.log("TC " + tc_10p_error_response + ": 10% error response for query and publish. Otherwise normal case."); + console.log("TC " + tc_10first_error_response + ": 10 first queries and requests gives no response for query and publish. Otherwise normal case."); + console.log("TC " + tc_100first_error_response + ": 100 first queries and requests gives no response for query and publish. Otherwise normal case."); + process.exit(0); -} + } + var bodyParser = require('body-parser') app.use(bodyParser.urlencoded({ extended: false })) @@ -52,55 +112,179 @@ app.use(bodyParser.json()) // parse application/vnd.api+json as json app.use(bodyParser.json({ type: 'application/vnd.api+json' })) -// parse some custom thing into a Buffer -app.use(bodyParser.raw({limit:1024*1024*20, type: 'application/octet-stream' })) +// parse some custom thing into a Buffer (to cater for 60MB files) +app.use(bodyParser.raw({limit:1024*1024*60, type: 'application/octet-stream' })) // parse an HTML body into a string app.use(bodyParser.text({ type: 'text/html' })) + + + +//Is alive function app.get("/",function(req, res){ res.send("ok"); }) +//Counter readout +app.get("/ctr_publish_query",function(req, res){ + res.send(""+ctr_publish_query); +}) +app.get("/ctr_publish_query_published",function(req, res){ + res.send(""+ctr_publish_query_published); +}) +app.get("/ctr_publish_query_not_published",function(req, res){ + res.send(""+ctr_publish_query_not_published); +}) +app.get("/ctr_publish_req",function(req, res){ + res.send(""+ctr_publish_req); +}) +app.get("/ctr_publish_req_redirect",function(req, res){ + res.send(""+ctr_publish_req_redirect); +}) +app.get("/ctr_publish_req_published",function(req, res){ + res.send(""+ctr_publish_req_published); +}) +app.get("/ctr_published_files",function(req, res){ + res.send(""+published.length); +}) +app.get("/tc_info",function(req, res){ + res.send(args.tc); +}) +function fmtMSS(s){ + return(s-(s%=60))/60+(9console.log("sleeping done")); + } else if (args.tc==tc_10p_delay_10s && (ctr_publish_query%10) == 0) { + console.log("sleep begin"); + timer(10000).then(_=>console.log("sleeping done")); + } else if (args.tc==tc_10p_error_response && (ctr_publish_query%10) == 0) { + res.send(400); + return; + } else if (args.tc==tc_10first_error_response && ctr_publish_query<11) { + res.send(400); + return; + } else if (args.tc==tc_100first_error_response & ctr_publish_query<101) { + res.send(400); + return; + } + + if (published.includes(filename)) { + ctr_publish_query_published++; + res.send("[" + filename + "]"); } else { - if (published.includes(filename)) { - res.send("[" + filename + "]"); - } else { - res.send("[]"); - } + ctr_publish_query_not_published++; + res.send("[]"); } }) +app.put('/publish/1/:filename', function (req, res) { + console.log("url:"+req.url); + console.log("body (first 25 bytes):"+req.body.slice(0,25)); + console.log("headers:"+req.headers); + ctr_publish_req++; -app.put('/publish/1/', function (req, res) { - var filename = req.query.filename; - var type = req.query.type; - if(typeof(filename) == 'undefined'){ - res.status(400).send({error: 'No filename provided.'}); - } else if(typeof(type) == 'undefined'){ - res.status(400).send({error: 'No type provided.'}); + var filename = req.params.filename; + console.log(filename); + + if (args.tc==tc_normal) { + //continue + } else if (args.tc==tc_none_published) { + ctr_publish_req_redirect++; + res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename); + return; + } else if (args.tc==tc_all_published) { + ctr_publish_req_published++; + res.send("ok"); + return; + }else if (args.tc==tc_10p_no_response && (ctr_publish_req%10) == 0) { + return; + } else if (args.tc==tc_10first_no_response && ctr_publish_req<11) { + return; + } else if (args.tc==tc_100first_no_response && ctr_publish_req<101) { + return; + } else if (args.tc==tc_all_delay_10s) { + console.log("sleep begin"); + timer(10000).then(_=>console.log("sleeping done")); + } else if (args.tc==tc_10p_delay_10s && (ctr_publish_req%10) == 0) { + console.log("sleep begin"); + timer(10000).then(_=>console.log("sleeping done")); + } else if (args.tc==tc_10p_error_response && (ctr_publish_req%10) == 0) { + res.send(400); + return; + } else if (args.tc==tc_10first_error_response && ctr_publish_req<11) { + res.send(400); + return; + } else if (args.tc==tc_100first_error_response & ctr_publish_req<101) { + res.send(400); + return; + } + + if (!published.includes(filename)) { + ctr_publish_req_redirect++; + res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename); } else { - if(args.tc == allPublished){ - res.send("[" + filename + "]"); - } else if(args.tc == nonePublished){ - res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename); - } else { - if (!published.includes(filename)) { - published.push(filename); - res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename); - } else { - res.send("ok"); - } - } + ctr_publish_req_published++; + res.send("ok"); } }) +//Callback from DR REDIR server, when file is published ok this PUT request update the list of published files. +app.put('/dr_redir_publish/:filename', function (req, res) { + console.log("url:"+req.url); + var filename = req.params.filename; + console.log(filename); + + if (!published.includes(filename)) { + console.log("File marked as published by callback from DR redir SIM. url: " + req.url); + published.push(filename); + } else { + console.log("File already marked as published. Callback from DR redir SIM. url: " + req.url); + } + + res.send("ok"); +}) var httpServer = http.createServer(app); var httpsServer = https.createServer(credentials, app); diff --git a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js index 5be1f689e..4494e899e 100644 --- a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js +++ b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js @@ -6,11 +6,97 @@ const stream = require('stream'); var app = express(); var fs = require("fs"); var path = require('path'); +var ArgumentParser = require('argparse').ArgumentParser; var privateKey = fs.readFileSync('cert/private.key', 'utf8'); var certificate = fs.readFileSync('cert/certificate.crt', 'utf8'); var credentials = {key: privateKey, cert: certificate}; + var bodyParser = require('body-parser') +var startTime = Date.now(); + +var dr_callback_ip = '192.168.100.2'; //IP for DR when running as container. Can be changed by env DR_SIM_IP + +//Counters +var ctr_publish_requests = 0; +var ctr_publish_responses = 0; +var lastPublish = ""; +var dwl_volume = 0; + +var parser = new ArgumentParser({ + version: '0.0.1', + addHelp:true, + description: 'Datarouter redirect simulator' + }); + +parser.addArgument('--tc' , { help: 'TC $NoOfTc' } ); +parser.addArgument('--printtc' , + { + help: 'Print complete usage help', + action: 'storeTrue' + } + ); + +var args = parser.parseArgs(); +const tc_normal = "normal"; +const tc_no_publish ="no_publish" +const tc_10p_no_response = "10p_no_response"; +const tc_10first_no_response = "10first_no_response"; +const tc_100first_no_response = "100first_no_response"; +const tc_all_delay_10s = "all_delay_10s"; +const tc_10p_delay_10s = "10p_delay_10s"; +const tc_10p_error_response = "10p_error_response"; +const tc_10first_error_response = "10first_error_response"; +const tc_100first_error_response = "100first_error_response"; + +if (args.tc==tc_normal) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_no_publish) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10first_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_100first_no_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_all_delay_10s) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_delay_10s) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10p_error_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_10first_error_response) { + console.log("TC: " + args.tc) + +} else if (args.tc==tc_100first_error_response) { + console.log("TC: " + args.tc) +} else { + console.log("No TC specified, use: --tc "); + process.exit(0); +} + +if (args.printtc) { + console.log("TC " + tc_normal + ": Normal case, all files publish and DR updated"); + console.log("TC " + tc_no_publish + ": Ok response but no files published"); + console.log("TC " + tc_10p_no_response + ": 10% % no response (file not published)"); + console.log("TC " + tc_10first_no_response + ": 10 first requests give no response (files not published)"); + console.log("TC " + tc_100first_no_response + ": 100 first requests give no response (files not published)"); + console.log("TC " + tc_all_delay_10s + ": All responses delayed 10s, normal publish"); + console.log("TC " + tc_10p_delay_10s + ": 10% of responses delayed 10s, normal publish"); + console.log("TC " + tc_10p_error_response + ": 10% error response (file not published)"); + console.log("TC " + tc_10first_error_response + ": 10 first requests give error response (file not published)"); + console.log("TC " + tc_100first_error_response + ": 100 first requests give error responses (file not published)"); + + process.exit(0); +} // parse application/x-www-form-urlencoded app.use(bodyParser.urlencoded({ extended: false })) @@ -22,26 +108,120 @@ app.use(bodyParser.json()) app.use(bodyParser.json({ type: 'application/vnd.api+json' })) // parse some custom thing into a Buffer -app.use(bodyParser.raw({limit:1024*1024*20, type: 'application/octet-stream' })) +app.use(bodyParser.raw({limit:1024*1024*60, type: 'application/octet-stream' })) // parse an HTML body into a string app.use(bodyParser.text({ type: 'text/html' })) + +//Formatting +function fmtMSS(s){ + return(s-(s%=60))/60+(9console.log("sleeping done")); + } else if (args.tc==tc_all_delay_10s) { + //var sleep = require('sleep'); + console.log("sleep begin"); + //sleep.sleep(10); + timer(10000).then(_=>console.log("sleeping done")); + } + + //Remaining part if normal file publish + + var filename = req.params.filename; + console.log(filename); + //Create filename (appending file size to name) to store + var storedFilename = path.resolve(__dirname, filename+"-"+req.body.length); + fs.writeFile(storedFilename, "", function (error) { //Store file with zero size + if (error) { console.error(error); } }); - res.send("ok") -}) + + //Make callback to update list of publish files in DR sim + //Note the hard code ip-adress, DR sim get this ip if simulators started from the + //script in the 'simulatorgroup' dir. + //Work around: Could not get a normal http put to work from nodejs, using curl instead + var util = require('util'); + var exec = require('child_process').exec; + + var command = 'curl -s -X PUT http://' + dr_callback_ip + ':3906/dr_redir_publish/' +req.params.filename; + + console.log("Callback to DR sim to report file published, cmd: " + command); + child = exec(command, function(error, stdout, stderr){ + console.log('stdout: ' + stdout); + console.log('stderr: ' + stderr); + if(error !== null) { + console.log('exec error: ' + error); + } + + }); + + //Update status variables + ctr_publish_responses++; + lastPublish = fmtMSS(Math.floor((Date.now()-startTime)/1000)); + dwl_volume = dwl_volume + req.body.length; + + res.send("ok") +}); + + var httpServer = http.createServer(app); var httpsServer = https.createServer(credentials, app); @@ -52,3 +232,7 @@ console.log("DR-simulator listening (http) at "+httpPort) httpsServer.listen(httpsPort); console.log("DR-simulator listening (https) at "+httpsPort) +if (process.env.DR_SIM_IP) { + dr_callback_ip=process.env.DR_SIM_IP; +} +console.log("Using IP " + dr_callback_ip + " for callback to DR sim"); -- cgit 1.2.3-korg