Enhancements of simulators for DFC

New test cases for MR and DR simulators
Added simulator monitor server for simulator supervision
Update - Adapted simulators to changed naming of ftp files
Update2 - Added some more test behaviour to simulators.

Issue-ID: DCAEGEN2-1313

Change-Id: I90a346d44ab4e7c9f5a65f599b64f1907525dd51
Signed-off-by: BjornMagnussonXA <bjorn.magnusson@est.tech>

13 files changed, 1070 insertions, 159 deletions

diff --git a/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile b/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile
index fc903d7e7..3d95492a8 100644
--- a/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile
+++ b/test/mocks/datafilecollector-testharness/dr-sim/Dockerfile
@@ -12,7 +12,10 @@ COPY cert/ cert/
 RUN npm install express
 RUN npm install argparse
 
-EXPOSE 3906
-EXPOSE 3907
-EXPOSE 3908
-EXPOSE 3909
\ No newline at end of file
+#Ports for DR
+#EXPOSE 3906
+#EXPOSE 3907
+
+#Ports for DR redir
+#EXPOSE 3908
+#EXPOSE 3909
\ No newline at end of file
diff --git a/test/mocks/datafilecollector-testharness/dr-sim/README.md b/test/mocks/datafilecollector-testharness/dr-sim/README.md
index f0cdf589b..8761d0cef 100644
--- a/test/mocks/datafilecollector-testharness/dr-sim/README.md
+++ b/test/mocks/datafilecollector-testharness/dr-sim/README.md
@@ -1,6 +1,9 @@
-#Alternative to running python (as described below) on your machine, use the docker files.
+###Alternative to running python (as described below) on your machine, use the docker files.
 1. Build docker container with ```docker build -t drsim_common:latest .```
 2. Run the container ```docker-compose up```
+3. For specific behavior of of the simulators, add arguments to the `command` entries in the `docker-compose.yml`.
+For example `command: node dmaapDR.js --tc no_publish` . (No argument will assume '--tc normal'). Run `node dmaapDR.js --printtc`
+and `node dmaapDR-redir.js --printtc` for details. 
 
 
 
@@ -9,5 +12,46 @@
 Make sure that you run these commands in the application directory "dr-sim"
 3. `npm install express`
 4. `npm install argparse`
-5. `node dmaapDR.js`   #keep it in the foreground
-6. `node dmaapDR_redir.js`  #keep it in the foreground
+5. `node dmaapDR.js`   #keep it in the foreground, see item 3 in the above list for arg to the simulator
+6. `node dmaapDR_redir.js`  #keep it in the foreground, see item 3 in the above list for arg to the simulator
+
+
+The dmaapDR_redir server send callback to dmaapDR server to update the list of successfully published files.
+As default, the ip for dmaapDR is set to work when running as container (using an ip address from the dfc_net docker network) . When running the servers from command line, set the env variable DR_SIM_IP=localhost
+
+The simulator can be queried for statistics (use curl from cmd line or open in browser, curl used below):
+
+DR
+
+`curl localhost:3906/ctr_publish_query` - returns the number of publish queries
+
+`curl localhost:3906/ctr_publish_query_published` - returns the number of responses where the files was published
+
+`curl localhost:3906/ctr_publish_query_not_published` - returns the number of responses where the file was not published
+
+`curl localhost:3906/ctr_publish_req` - returns the number of publish requests
+
+`curl localhost:3906/ctr_publish_req_redirect` - returns the number of publish responses with redirect
+
+`curl localhost:3906/ctr_publish_req_published` - returns the number of publish responses without redirect
+
+`curl localhost:3906/ctr_published_files` - returns the number of unique published files
+
+`curl localhost:3906/tc_info` - returns the tc name (argument on the command line)
+
+`curl localhost:3906/execution_time` - returns the execution times in mm:ss
+
+
+DR REDIR
+
+`curl localhost:3908/ctr_publish_requests` - returns the number of publish queries
+
+`curl localhost:3908/ctr_publish_responses` - returns the number of publish responses
+
+`curl localhost:3908/tc_info` - returns the tc name (argument on the command line)
+
+`curl localhost:3908/execution_time` - returns the execution times in mm:ss
+
+`curl localhost:3908/time_lastpublish` - returns the time (mm:ss) for the latest publish
+
+`curl localhost:3908/dwl_volume` - returns the total received data volume of file data
diff --git a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js
index 4e7317473..fffe57ce3 100644
--- a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js
+++ b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js
@@ -5,12 +5,33 @@ var express = require('express');
 const stream = require('stream');
 var app = express();
 var fs = require('fs');
-var path = require('path');
 var privateKey  = fs.readFileSync('cert/private.key', 'utf8');
 var certificate = fs.readFileSync('cert/certificate.crt', 'utf8');
 var credentials = {key: privateKey, cert: certificate};
-const allPublished = "allPublished";
-const nonePublished = "nonePublished";
+
+//For execution time calculation
+var startTime = Date.now();
+
+//Test case constants
+const tc_normal = "normal";
+const tc_none_published = "none_published";
+const tc_all_published = "all_published"
+const tc_10p_no_response = "10p_no_response";
+const tc_10first_no_response = "10first_no_response";
+const tc_100first_no_response = "100first_no_response";
+const tc_all_delay_10s = "all_delay_10s";
+const tc_10p_delay_10s = "10p_delay_10s";
+const tc_10p_error_response = "10p_error_response";
+const tc_10first_error_response = "10first_error_response";
+const tc_100first_error_response = "100first_error_response";
+
+//Counters
+var ctr_publish_query = 0;
+var ctr_publish_query_published = 0;
+var ctr_publish_query_not_published = 0;
+var ctr_publish_req = 0;
+var ctr_publish_req_redirect = 0;
+var ctr_publish_req_published = 0;
 
 var parser = new ArgumentParser({
 	  version: '0.0.1',
@@ -28,20 +49,59 @@ parser.addArgument('--printtc' ,
 
 var args = parser.parseArgs();
 
-if (args.tc=="nonePublished") {
-	console.log("TC: nonePublished")
-}
-if (args.tc=="allPublished") {
-	console.log("TC: allPublished")
-	//preparations
+if (args.tc==tc_normal) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_none_published) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_all_published) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_10p_no_response) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_10first_no_response) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_100first_no_response) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_all_delay_10s) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_10p_delay_10s) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_10p_error_response) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_10first_error_response) {
+	console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_100first_error_response) {
+	console.log("TC: " + args.tc)
+} else {
+	console.log("No TC specified, use: --tc <tc-id>");
+	process.exit(0);
 }
 
 if (args.printtc) {
-	console.log("TC nonePublished: no file has already been published.");
-	console.log("TC allPublished: whatever is the request, this file is considered as published.");
-	console.log("No argument passed: normal behaviour, that is publish if not already published");
+	console.log("TC " + tc_normal + ": Normal case, query respone based on published files. Publish responde with ok/redirect depending on if file is published or not.");
+	console.log("TC " + tc_none_published + ": Query responde 'ok'. Publish respond with redirect.");
+	console.log("TC " + tc_all_published + ": Query respond with filename. Publish respond with 'ok'.");
+	console.log("TC " + tc_10p_no_response + ": 10% % no response for query and publish. Otherwise normal case.");
+	console.log("TC " + tc_10first_no_response + ": 10 first queries and requests gives no response for query and publish. Otherwise normal case.");
+	console.log("TC " + tc_100first_no_response + ": 100 first queries and requests gives no response for query and publish. Otherwise normal case.");
+	console.log("TC " + tc_all_delay_10s + ": All responses delayed 10s (both query and publish).");
+	console.log("TC " + tc_10p_delay_10s + ": 10% of responses delayed 10s, (both query and publish).");
+	console.log("TC " + tc_10p_error_response + ": 10% error response for query and publish. Otherwise normal case.");
+	console.log("TC " + tc_10first_error_response + ": 10 first queries and requests gives no response for query and publish. Otherwise normal case.");
+	console.log("TC " + tc_100first_error_response + ": 100 first queries and requests gives no response for query and publish. Otherwise normal case.");
+
 	process.exit(0);
-}
+  }
+
 
 var bodyParser = require('body-parser')
 app.use(bodyParser.urlencoded({ extended: false }))
@@ -52,55 +112,179 @@ app.use(bodyParser.json())
 // parse application/vnd.api+json as json
 app.use(bodyParser.json({ type: 'application/vnd.api+json' }))
 
-// parse some custom thing into a Buffer
-app.use(bodyParser.raw({limit:1024*1024*20, type: 'application/octet-stream' }))
+// parse some custom thing into a Buffer (to cater for 60MB files)
+app.use(bodyParser.raw({limit:1024*1024*60, type: 'application/octet-stream' }))
 // parse an HTML body into a string
 app.use(bodyParser.text({ type: 'text/html' }))
+
+
+
+//Is alive function
 app.get("/",function(req, res){
 	res.send("ok");
 })
 
+//Counter readout
+app.get("/ctr_publish_query",function(req, res){
+	res.send(""+ctr_publish_query);
+})
+app.get("/ctr_publish_query_published",function(req, res){
+	res.send(""+ctr_publish_query_published);
+})
+app.get("/ctr_publish_query_not_published",function(req, res){
+	res.send(""+ctr_publish_query_not_published);
+})
+app.get("/ctr_publish_req",function(req, res){
+	res.send(""+ctr_publish_req);
+})
+app.get("/ctr_publish_req_redirect",function(req, res){
+	res.send(""+ctr_publish_req_redirect);
+})
+app.get("/ctr_publish_req_published",function(req, res){
+	res.send(""+ctr_publish_req_published);
+})
+app.get("/ctr_published_files",function(req, res){
+	res.send(""+published.length);
+})
+app.get("/tc_info",function(req, res){
+	res.send(args.tc);
+})
+function fmtMSS(s){
+	return(s-(s%=60))/60+(9<s?':':':0')+s    //Format time diff in mm:ss
+}
+app.get("/execution_time",function(req, res){
+	diff = fmtMSS(Math.floor((Date.now()-startTime)/1000));
+	res.send(""+diff);
+})
 
+//db of published files
 var published = [];
+
 app.get('/feedlog/1/',function(req, res){
+	console.log("url:"+req.url);
+	ctr_publish_query++;
 	var filename = req.query.filename;
-	if(args.tc == allPublished){
-		res.send("[" + filename + "]");
-	} else if(args.tc == nonePublished){
+	console.log(filename);
+	var qtype = req.query.type;
+	if(typeof(filename) == 'undefined'){
+		res.status(400).send({error: 'No filename provided.'});
+		return;
+	} else if(typeof(qtype) == 'undefined'){
+		res.status(400).send({error: 'No type provided.'});
+		return;
+	}
+	
+	//Ugly fix, plus signs replaces with spaces in query params....need to put them back
+	filename = filename.replace(/ /g,"+");
+	
+	if (args.tc==tc_normal) {
+	  //continue
+	}  else if (args.tc==tc_none_published) {
+		ctr_publish_query_not_published++;
 		res.send("[]");
+		return;
+	} else if (args.tc==tc_all_published) {
+		ctr_publish_query_published++;
+		res.send("[" + filename + "]");
+		return;
+	} else if (args.tc==tc_10p_no_response && (ctr_publish_query%10) == 0) {
+		return;
+	} else if (args.tc==tc_10first_no_response && ctr_publish_query<11) {
+		return;
+	} else if (args.tc==tc_100first_no_response && ctr_publish_query<101) {
+		return;
+	} else if (args.tc==tc_all_delay_10s) {
+		console.log("sleep begin");
+		timer(10000).then(_=>console.log("sleeping done")); 
+	} else if (args.tc==tc_10p_delay_10s && (ctr_publish_query%10) == 0) {
+		console.log("sleep begin");
+		timer(10000).then(_=>console.log("sleeping done")); 
+	} else if (args.tc==tc_10p_error_response && (ctr_publish_query%10) == 0) {
+		res.send(400);
+		return;
+	} else if (args.tc==tc_10first_error_response && ctr_publish_query<11) {
+		res.send(400);
+		return;
+	} else if (args.tc==tc_100first_error_response & ctr_publish_query<101) {
+		res.send(400);
+		return;
+	}
+
+	if (published.includes(filename)) {
+		ctr_publish_query_published++;
+		res.send("[" + filename + "]");
 	} else {
-		if (published.includes(filename)) {
-			res.send("[" + filename + "]");
-		} else {
-			res.send("[]");
-		}
+		ctr_publish_query_not_published++;
+		res.send("[]");
 	}
 })
 
+app.put('/publish/1/:filename', function (req, res) {
+	console.log("url:"+req.url);
+	console.log("body (first 25 bytes):"+req.body.slice(0,25));
+	console.log("headers:"+req.headers);
+	ctr_publish_req++;
 
-app.put('/publish/1/', function (req, res) {
-	var filename = req.query.filename;
-	var type = req.query.type;
-	if(typeof(filename) == 'undefined'){
-		res.status(400).send({error: 'No filename provided.'});
-	} else if(typeof(type) == 'undefined'){
-		res.status(400).send({error: 'No type provided.'});
+	var filename = req.params.filename;
+	console.log(filename);
+
+	if (args.tc==tc_normal) {
+	    //continue
+	} else if (args.tc==tc_none_published) {
+		ctr_publish_req_redirect++;
+		res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename);
+		return;
+	} else if (args.tc==tc_all_published) {
+		ctr_publish_req_published++;
+		res.send("ok");
+		return;
+	}else if (args.tc==tc_10p_no_response && (ctr_publish_req%10) == 0) {
+		return;
+	} else if (args.tc==tc_10first_no_response && ctr_publish_req<11) {
+		return;
+	} else if (args.tc==tc_100first_no_response && ctr_publish_req<101) {
+		return;
+	} else if (args.tc==tc_all_delay_10s) {
+		console.log("sleep begin");
+		timer(10000).then(_=>console.log("sleeping done")); 
+	} else if (args.tc==tc_10p_delay_10s && (ctr_publish_req%10) == 0) {
+		console.log("sleep begin");
+		timer(10000).then(_=>console.log("sleeping done")); 
+	} else if (args.tc==tc_10p_error_response && (ctr_publish_req%10) == 0) {
+		res.send(400);
+		return;
+	} else if (args.tc==tc_10first_error_response && ctr_publish_req<11) {
+		res.send(400);
+		return;
+	} else if (args.tc==tc_100first_error_response & ctr_publish_req<101) {
+		res.send(400);
+		return;
+	}
+
+	if (!published.includes(filename)) {
+		ctr_publish_req_redirect++;
+		res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename);
 	} else {
-		if(args.tc == allPublished){
-			res.send("[" + filename + "]");
-		} else if(args.tc == nonePublished){
-			res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename);
-		} else {
-			if (!published.includes(filename)) {
-				published.push(filename);
-				res.redirect(301, 'http://127.0.0.1:3908/publish/1/'+filename);
-			} else {
-				res.send("ok");
-			}
-		}
+		ctr_publish_req_published++;
+		res.send("ok");
 	}
 })
 
+//Callback from DR REDIR server, when file is published ok this PUT request update the list of published files.
+app.put('/dr_redir_publish/:filename', function (req, res) {
+	console.log("url:"+req.url);
+	var filename = req.params.filename;
+	console.log(filename);
+
+	if (!published.includes(filename)) {
+		console.log("File marked as published by callback from DR redir SIM. url: " + req.url);
+		published.push(filename);
+	} else {
+		console.log("File already marked as published. Callback from DR redir SIM. url: " + req.url);
+	}
+
+	res.send("ok");
+})
 
 var httpServer = http.createServer(app);
 var httpsServer = https.createServer(credentials, app);
diff --git a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js
index 5be1f689e..4494e899e 100644
--- a/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js
+++ b/test/mocks/datafilecollector-testharness/dr-sim/dmaapDR_redir.js
@@ -6,11 +6,97 @@ const stream = require('stream');
 var app = express();
 var fs = require("fs");
 var path = require('path');
+var ArgumentParser = require('argparse').ArgumentParser;
 var privateKey  = fs.readFileSync('cert/private.key', 'utf8');
 var certificate = fs.readFileSync('cert/certificate.crt', 'utf8');
 var credentials = {key: privateKey, cert: certificate};
 
+
 var bodyParser = require('body-parser')
+var startTime = Date.now();
+
+var dr_callback_ip = '192.168.100.2'; //IP for DR when running as container. Can be changed by env DR_SIM_IP
+
+//Counters
+var ctr_publish_requests = 0;
+var ctr_publish_responses = 0;
+var lastPublish = "";
+var dwl_volume = 0;
+
+var parser = new ArgumentParser({
+	version: '0.0.1',
+	addHelp:true,
+	description: 'Datarouter redirect simulator'
+  });
+
+parser.addArgument('--tc' , { help: 'TC $NoOfTc' } );
+parser.addArgument('--printtc' ,
+	  {
+		  help: 'Print complete usage help',
+		  action: 'storeTrue'
+	  }
+  );
+
+var args = parser.parseArgs();
+const tc_normal = "normal";
+const tc_no_publish ="no_publish"
+const tc_10p_no_response = "10p_no_response";
+const tc_10first_no_response = "10first_no_response";
+const tc_100first_no_response = "100first_no_response";
+const tc_all_delay_10s = "all_delay_10s";
+const tc_10p_delay_10s = "10p_delay_10s";
+const tc_10p_error_response = "10p_error_response";
+const tc_10first_error_response = "10first_error_response";
+const tc_100first_error_response = "100first_error_response";
+
+if (args.tc==tc_normal) {
+  console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_no_publish) {
+	console.log("TC: " + args.tc)
+  
+} else if (args.tc==tc_10p_no_response) {
+  console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_10first_no_response) {
+  console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_100first_no_response) {
+  console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_all_delay_10s) {
+  console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_10p_delay_10s) {
+	console.log("TC: " + args.tc)
+  
+} else if (args.tc==tc_10p_error_response) {
+  console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_10first_error_response) {
+  console.log("TC: " + args.tc)
+
+} else if (args.tc==tc_100first_error_response) {
+  console.log("TC: " + args.tc)
+} else {
+	console.log("No TC specified, use: --tc <tc-id>");
+	process.exit(0);
+}
+
+if (args.printtc) {
+  console.log("TC " + tc_normal + ": Normal case, all files publish and DR updated");
+  console.log("TC " + tc_no_publish + ": Ok response but no files published");
+  console.log("TC " + tc_10p_no_response + ": 10% % no response (file not published)");
+  console.log("TC " + tc_10first_no_response + ": 10 first requests give no response (files not published)");
+  console.log("TC " + tc_100first_no_response + ": 100 first requests give no response (files not published)");
+  console.log("TC " + tc_all_delay_10s + ": All responses delayed 10s, normal publish");
+  console.log("TC " + tc_10p_delay_10s + ": 10% of responses delayed 10s, normal publish");
+  console.log("TC " + tc_10p_error_response + ": 10% error response (file not published)");
+  console.log("TC " + tc_10first_error_response + ": 10 first requests give error response (file not published)");
+  console.log("TC " + tc_100first_error_response + ": 100 first requests give error responses (file not published)");
+
+  process.exit(0);
+}
 
 // parse application/x-www-form-urlencoded
 app.use(bodyParser.urlencoded({ extended: false }))
@@ -22,26 +108,120 @@ app.use(bodyParser.json())
 app.use(bodyParser.json({ type: 'application/vnd.api+json' }))
 
 // parse some custom thing into a Buffer
-app.use(bodyParser.raw({limit:1024*1024*20, type: 'application/octet-stream' }))
+app.use(bodyParser.raw({limit:1024*1024*60, type: 'application/octet-stream' }))
 
 // parse an HTML body into a string
 app.use(bodyParser.text({ type: 'text/html' }))
+
+//Formatting
+function fmtMSS(s){
+	return(s-(s%=60))/60+(9<s?':':':0')+s  //Format time diff to mm:ss
+}
+function fmtLargeNumber(x) {
+	return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, " "); //Format large with space, eg: 1 000 000
+}
+
+//I'm alive function
 app.get("/",function(req, res){
 	res.send("ok");
 })
 
+//Counter readout
+app.get("/ctr_publish_requests",function(req, res){
+	res.send(""+ctr_publish_requests);
+})
+app.get("/ctr_publish_responses",function(req, res){
+	res.send(""+ctr_publish_responses);
+})
+app.get("/execution_time",function(req, res){
+	diff = fmtMSS(Math.floor((Date.now()-startTime)/1000));
+	res.send(""+diff);
+})
+app.get("/time_lastpublish",function(req, res){
+	res.send(""+lastPublish);
+})
+app.get("/dwl_volume",function(req, res){
+	res.send(""+fmtLargeNumber(dwl_volume));
+})
+app.get("/tc_info",function(req, res){
+	res.send(args.tc);
+})
+
 app.put('/publish/1/:filename', function (req, res) {
-	console.log(req.files);
-	console.log(req.body)
+	console.log(req.url);
+	console.log("First 25 bytes of body: " + req.body.slice(0,25))
 	console.log(req.headers)
-	var filename = path.basename(req.params.filename);
-  filename = path.resolve(__dirname, filename);
-	console.log(req.params.filename);
-  fs.writeFile(filename, req.body, function (error) {
-  	if (error) { console.error(error); }
+	ctr_publish_requests++;
+	if (args.tc == tc_no_publish) {
+		tr_publish_responses++;
+		res.send("ok")
+		return;
+	} else if (args.tc==tc_10p_no_response && (ctr_publish_requests%10)==0) {
+		return;
+	} else if (args.tc==tc_10first_no_response && ctr_publish_requests<11) {
+		return;
+	} else if (args.tc==tc_100first_no_response && ctr_publish_requests<101) {
+		return;
+	} else if (args.tc==tc_10p_error_response && (ctr_publish_requests%10)==0) {
+		tr_publish_responses++;
+		res.send(400, "");
+		return;
+	} else if (args.tc==tc_10first_error_response && ctr_publish_requests<11) {
+		tr_publish_responses++;
+		res.send(400, "");
+		return;
+	} else if (args.tc==tc_100first_error_response && ctr_publish_requests<101) {
+		tr_publish_responses++;
+		res.send(400, "");
+		return;
+	} else if (args.tc==tc_10p_delay_10s && (ctr_publish_requests%10)==0) {
+		console.log("sleep begin");
+		timer(10000).then(_=>console.log("sleeping done")); 
+	} else if (args.tc==tc_all_delay_10s) {
+		//var sleep = require('sleep');
+		console.log("sleep begin");
+		//sleep.sleep(10); 
+		timer(10000).then(_=>console.log("sleeping done")); 
+	}
+
+	//Remaining part if normal file publish
+
+	var filename = req.params.filename;
+	console.log(filename);
+	//Create filename (appending file size to name) to store
+  	var storedFilename = path.resolve(__dirname, filename+"-"+req.body.length); 
+  	fs.writeFile(storedFilename, "", function (error) {  //Store file with zero size
+  		if (error) { console.error(error); }
 	});
-	 res.send("ok")
-})
+	
+	//Make callback to update list of publish files in DR sim
+	//Note the hard code ip-adress, DR sim get this ip if simulators started from the
+	//script in the 'simulatorgroup' dir.
+	//Work around: Could not get a normal http put to work from nodejs, using curl instead
+	var util = require('util');
+	var exec = require('child_process').exec;
+
+	var command = 'curl -s -X PUT http://' + dr_callback_ip + ':3906/dr_redir_publish/' +req.params.filename;
+
+	console.log("Callback to DR sim to report file published, cmd: " + command);
+	child = exec(command, function(error, stdout, stderr){
+		console.log('stdout: ' + stdout);
+		console.log('stderr: ' + stderr);
+		if(error !== null) {
+			console.log('exec error: ' + error);
+		}
+		
+	});
+
+	//Update status variables
+	ctr_publish_responses++;
+	lastPublish = fmtMSS(Math.floor((Date.now()-startTime)/1000));
+	dwl_volume = dwl_volume + req.body.length;
+
+	res.send("ok")
+});
+
+
 var httpServer = http.createServer(app);
 var httpsServer = https.createServer(credentials, app);
 
@@ -52,3 +232,7 @@ console.log("DR-simulator listening (http) at "+httpPort)
 httpsServer.listen(httpsPort);
 console.log("DR-simulator listening (https) at "+httpsPort)
 
+if (process.env.DR_SIM_IP) {
+	dr_callback_ip=process.env.DR_SIM_IP;
+} 
+console.log("Using IP " + dr_callback_ip + " for callback to DR sim");
diff --git a/test/mocks/datafilecollector-testharness/mr-sim/README.md b/test/mocks/datafilecollector-testharness/mr-sim/README.md
index 57463453f..8fafdfe1f 100644
--- a/test/mocks/datafilecollector-testharness/mr-sim/README.md
+++ b/test/mocks/datafilecollector-testharness/mr-sim/README.md
@@ -4,11 +4,21 @@
 2. Run the container ```docker-compose up```
 The behavior can be changed by argument to the python script in the docker-compose.yml
 
-The simulator can be queried for statistics
-localhost:2222/ctr_requests   - return an integer of the number of get request to the event poll path
-localhost:2222/ctr_responses  - return an integer of the number of get responses to the event poll path
-localhost:2222/ctr_unique_files - returns an integer or the number of unique files. A unique file is the combination of node+file_sequence_number 
+The simulator can be queried for statistics (use curl from cmd line or open in browser, curl used below):
 
+`curl localhost:2222/ctr_requests`   - return an integer of the number of get request to the event poll path
+
+`curl localhost:2222/ctr_responses`  - return an integer of the number of get responses to the event poll path
+
+`curl localhost:2222/ctr_unique_files` - returns an integer or the number of unique files. A unique file is the combination of node+file_sequence_number 
+
+`curl localhost:2222/tc_info` - returns the tc string (as given on the cmd line)
+
+`curl localhost:2222/ctr_events` - returns the total number of events
+
+`curl localhost:2222/execution_time` - returns the execution time in mm:ss
+
+`curl localhost:2222/ctr_unique_PNFs` - return the number of unique PNFS in alla events.
 
 ##Common TC info
 File names for 1MB, 5MB and 50MB files
@@ -20,33 +30,44 @@ Limited event streams
 When the number of events are exhausted, empty replies are returned '[]'
 
 TC100 - One ME, SFTP, 1 1MB file, 1 event
+
 TC101 - One ME, SFTP, 1 5MB file, 1 event
+
 TC102 - One ME, SFTP, 1 50MB file, 1 event
 
 TC110 - One ME, SFTP, 1MB files, 1 file per event, 100 events, 1 event per poll.
+
 TC111 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per poll.
+
 TC112 - One ME, SFTP, 5MB files, 100 files per event, 100 events, 1 event per poll.
+
 TC113 - One ME, SFTP, 1MB files, 100 files per event, 100 events. All events in one poll.
 
 
 TC120 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per poll. 10% of replies each: no response, empty message, slow response, 404-error, malformed json
-TC121 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per poll. 10% missing files
-TC122 - One ME, SFTP, 1MB files, 100 files per event, 100 events. 1 event per poll. All files with identical name. 
-
-Endless event streams
-TC1000 - One ME, SFTP, 1MB files, 100 files per event, endless number of events, 1 event per poll
-TC1001 - One ME, SFTP, 5MB files, 100 files per event, endless number of events, 1 event per poll
 
+TC121 - One ME, SFTP, 1MB files, 100 files per event, 100 events, 1 event per poll. 10% missing files
 
-TC510 - 5 ME, SFTP, 1MB files, 1 file per event, 100 events, 1 event per poll.
+TC122 - One ME, SFTP, 1MB files, 100 files per event, 100 events. 1 event per poll. All files with identical name. 
 
+TC510 - 700 MEs, SFTP, 1MB files, 1 file per event, 3500 events, 700 event per poll.
 
 TC200-TC202 same as TC100-TC102 but with FTPS
+
 TC210-TC213 same as TC110-TC113 but with FTPS
+
 TC2000-TC2001 same as TC1000-TC1001 but with FTPS
+
 TC610 same as TC510 but with FTPS
 
 
+Endless event streams
+
+TC1000 - One ME, SFTP, 1MB files, 100 files per event, endless number of events, 1 event per poll
+
+TC1001 - One ME, SFTP, 5MB files, 100 files per event, endless number of events, 1 event per poll
+
+
 ## Developer workflow
 
 1. ```sudo apt install python3-venv```
diff --git a/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py b/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py
index ef46535f5..c1bed8f63 100644
--- a/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py
+++ b/test/mocks/datafilecollector-testharness/mr-sim/mr-sim.py
@@ -3,15 +3,20 @@ import os
 from werkzeug import secure_filename
 from flask import Flask, render_template, request
 from time import sleep
+import time
 import sys
 import json
 from flask import Flask
+
 app = Flask(__name__)
 
 #Server info
 HOST_IP = "0.0.0.0"
 HOST_PORT = 2222
 
+SFTP_PORT = 1022
+FTPS_PORT = 21
+
 #Test function to check server running
 @app.route('/',
     methods=['GET'])
@@ -46,6 +51,30 @@ def testcase_info():
     global tc_num
     return tc_num
 
+#Returns number of events
+@app.route('/ctr_events',
+    methods=['GET'])
+def counter_events():
+    global ctr_events
+    return str(ctr_events)
+
+#Returns number of events
+@app.route('/execution_time',
+    methods=['GET'])
+def exe_time():
+    global startTime
+    
+    stopTime = time.time()
+    minutes, seconds = divmod(stopTime-startTime, 60)
+    return "{:0>2}:{:0>2}".format(int(minutes),int(seconds))
+
+#Returns number of unique PNFs
+@app.route('/ctr_unique_PNFs',
+    methods=['GET'])
+def counter_uniquePNFs():
+    global pnfMap
+    return str(len(pnfMap))
+
 #Messages polling function
 @app.route(
     "/events/unauthenticated.VES_NOTIFICATION_OUTPUT/OpenDcae-c12/C12",
@@ -86,7 +115,12 @@ def MR_reply():
       return tc1001("sftp")
 
     elif args.tc510:
-      return tc510("sftp")      
+      return tc510("sftp")  
+    elif args.tc511:
+      return tc511("sftp")   
+  
+    elif args.tc710:
+      return tc710("sftp")     
 
 
     elif args.tc200:
@@ -118,7 +152,12 @@ def MR_reply():
       return tc2001("ftps")
 
     elif args.tc610:
-      return tc510("ftps")     
+      return tc510("ftps")  
+    elif args.tc611:
+      return tc511("ftps") 
+  
+    elif args.tc810:
+      return tc710("ftps")   
 
 
 #### Test case functions
@@ -127,6 +166,7 @@ def MR_reply():
 def tc100(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
@@ -134,28 +174,35 @@ def tc100(ftptype):
     return buildOkResponse("[]")
 
   seqNr = (ctr_responses-1)
-  msg = getEventHead() + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + getEventEnd()
+  nodeName = createNodeName(0)
+  fileName = createFileName(nodeName, seqNr, "1MB")
+  msg = getEventHead(nodeName) + getEventName(fileName,ftptype,"onap","pano") + getEventEnd()
   fileMap[seqNr] = seqNr
+  ctr_events = ctr_events+1
   return buildOkResponse("["+msg+"]")
 
 def tc101(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
   if (ctr_responses > 1):
     return buildOkResponse("[]")  
- 
+
   seqNr = (ctr_responses-1)
-  msg = getEventHead() + getEventName("5MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + getEventEnd()
+  nodeName = createNodeName(0)
+  fileName = createFileName(nodeName, seqNr, "5MB")
+  msg = getEventHead(nodeName) + getEventName(fileName,ftptype,"onap","pano") + getEventEnd()
   fileMap[seqNr] = seqNr
-
+  ctr_events = ctr_events+1
   return buildOkResponse("["+msg+"]")
 
 def tc102(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
@@ -163,14 +210,17 @@ def tc102(ftptype):
     return buildOkResponse("[]")  
 
   seqNr = (ctr_responses-1)
-  msg = getEventHead() + getEventName("50MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + getEventEnd()
+  nodeName = createNodeName(0)
+  fileName = createFileName(nodeName, seqNr, "50MB")
+  msg = getEventHead(nodeName) + getEventName(fileName,ftptype,"onap","pano") + getEventEnd()
   fileMap[seqNr] = seqNr
-
+  ctr_events = ctr_events+1
   return buildOkResponse("["+msg+"]")
 
 def tc110(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
@@ -178,75 +228,89 @@ def tc110(ftptype):
     return buildOkResponse("[]")  
   
   seqNr = (ctr_responses-1)
-  msg = getEventHead() + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022) + getEventEnd()
+  nodeName = createNodeName(0)
+  fileName = createFileName(nodeName, seqNr, "1MB")
+  msg = getEventHead(nodeName) + getEventName(fileName,ftptype,"onap","pano") + getEventEnd()
   fileMap[seqNr] = seqNr
-
+  ctr_events = ctr_events+1
   return buildOkResponse("["+msg+"]")
 
 def tc111(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
   if (ctr_responses > 100):
     return buildOkResponse("[]")  
-  
-  msg = getEventHead()
+
+  nodeName = createNodeName(0)
+  msg = getEventHead(nodeName)
 
   for i in range(100):
     seqNr = i+(ctr_responses-1)
     if i != 0: msg = msg + ","
-    msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022)
+    fileName = createFileName(nodeName, seqNr, "1MB")
+    msg = msg + getEventName(fileName,ftptype,"onap","pano")
     fileMap[seqNr] = seqNr
 
   msg = msg + getEventEnd()
+  ctr_events = ctr_events+1
 
   return buildOkResponse("["+msg+"]")
 
 def tc112(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
   if (ctr_responses > 100):
     return buildOkResponse("[]")  
-  
-  msg = getEventHead()
+
+  nodeName = createNodeName(0)
+  msg = getEventHead(nodeName)
 
   for i in range(100):
     seqNr = i+(ctr_responses-1)
     if i != 0: msg = msg + ","
-    msg = msg + getEventName("5MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022)
+    fileName = createFileName(nodeName, seqNr, "5MB")
+    msg = msg + getEventName(fileName,ftptype,"onap","pano")
     fileMap[seqNr] = seqNr
 
   msg = msg + getEventEnd()
+  ctr_events = ctr_events+1
 
   return buildOkResponse("["+msg+"]")
 
 def tc113(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
   if (ctr_responses > 1):
     return buildOkResponse("[]")  
-  
+
+  nodeName = createNodeName(0)
   msg = ""
 
   for evts in range(100):  # build 100 evts
     if (evts > 0):
       msg = msg + ","
-    msg = msg + getEventHead()
+    msg = msg + getEventHead(nodeName)
     for i in range(100):   # build 100 files
       seqNr = i+evts+100*(ctr_responses-1)
       if i != 0: msg = msg + ","
-      msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022)
+      fileName = createFileName(nodeName, seqNr, "1MB")
+      msg = msg + getEventName(fileName,ftptype,"onap","pano")
       fileMap[seqNr] = seqNr
 
     msg = msg + getEventEnd()
+    ctr_events = ctr_events+1
 
   return buildOkResponse("["+msg+"]")
 
@@ -254,20 +318,23 @@ def tc113(ftptype):
 def tc120(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
+  nodeName = createNodeName(0)
+
   if (ctr_responses > 100):
     return buildOkResponse("[]")  
 
   if (ctr_responses % 10 == 2):
     return  # Return nothing
-  
+
   if (ctr_responses % 10 == 3):
     return buildOkResponse("") # Return empty message
 
   if (ctr_responses % 10 == 4):
-    return buildOkResponse(getEventHead()) # Return part of a json event
+    return buildOkResponse(getEventHead(nodeName)) # Return part of a json event
 
   if (ctr_responses % 10 == 5):
     return buildEmptyResponse(404) # Return empty message with status code
@@ -275,64 +342,72 @@ def tc120(ftptype):
   if (ctr_responses % 10 == 6):
     sleep(60)
 
-  
-  msg = getEventHead()
+
+  msg = getEventHead(nodeName)
 
   for i in range(100):
     seqNr = i+(ctr_responses-1)
     if i != 0: msg = msg + ","
-    msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022)
+    fileName = createFileName(nodeName, seqNr, "1MB")
+    msg = msg + getEventName(fileName,ftptype,"onap","pano")
     fileMap[seqNr] = seqNr
 
   msg = msg + getEventEnd()
+  ctr_events = ctr_events+1
 
   return buildOkResponse("["+msg+"]")
 
 def tc121(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
   if (ctr_responses > 100):
-    return buildOkResponse("[]")  
-  
-  msg = getEventHead()
+    return buildOkResponse("[]")
+
+  nodeName = createNodeName(0)
+  msg = getEventHead(nodeName)
 
+  fileName = ""
   for i in range(100):
     seqNr = i+(ctr_responses-1)
     if (seqNr%10 == 0):     # Every 10th file is "missing"
-      fn = "MissingFile_" + str(seqNr) + ".tar.gz"
+      fileName = createMissingFileName(nodeName, seqNr, "1MB")
     else:
-      fn = "1MB_" + str(seqNr) + ".tar.gz"
+      fileName = createFileName(nodeName, seqNr, "1MB")
       fileMap[seqNr] = seqNr
 
     if i != 0: msg = msg + ","
-    msg = msg + getEventName(fn,ftptype,"onap","pano","localhost",1022)
-    
+    msg = msg + getEventName(fileName,ftptype,"onap","pano")
 
   msg = msg + getEventEnd()
+  ctr_events = ctr_events+1
 
   return buildOkResponse("["+msg+"]")
 
 def tc122(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
   if (ctr_responses > 100):
     return buildOkResponse("[]")  
-  
-  msg = getEventHead()
+
+  nodeName = createNodeName(0)
+  msg = getEventHead(nodeName)
 
   for i in range(100):
-    fn = "1MB_0.tar.gz"  # All files identical names
+    fileName = createFileName(nodeName, 0, "1MB")  # All files identical names
     if i != 0: msg = msg + ","
-    msg = msg + getEventName(fn,ftptype,"onap","pano","localhost",1022)
+    msg = msg + getEventName(fileName,ftptype,"onap","pano")
 
   fileMap[0] = 0
   msg = msg + getEventEnd()
+  ctr_events = ctr_events+1
 
   return buildOkResponse("["+msg+"]")
 
@@ -340,42 +415,51 @@ def tc122(ftptype):
 def tc1000(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
-  msg = getEventHead()
+  nodeName = createNodeName(0)
+  msg = getEventHead(nodeName)
 
   for i in range(100):
     seqNr = i+(ctr_responses-1)
     if i != 0: msg = msg + ","
-    msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022)
+    fileName = createFileName(nodeName, seqNr, "1MB")
+    msg = msg + getEventName(fileName,ftptype,"onap","pano")
     fileMap[seqNr] = seqNr
 
   msg = msg + getEventEnd()
+  ctr_events = ctr_events+1
 
   return buildOkResponse("["+msg+"]")
 
 def tc1001(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
-  msg = getEventHead()
+  nodeName = createNodeName(0)
+  msg = getEventHead(nodeName)
 
   for i in range(100):
     seqNr = i+(ctr_responses-1)
     if i != 0: msg = msg + ","
-    msg = msg + getEventName("5MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022)
+    fileName = createFileName(nodeName, seqNr, "5MB")
+    msg = msg + getEventName(fileName,ftptype,"onap","pano")
     fileMap[seqNr] = seqNr
 
   msg = msg + getEventEnd()
+  ctr_events = ctr_events+1
 
   return buildOkResponse("["+msg+"]")
 
 def tc510(ftptype):
   global ctr_responses
   global ctr_unique_files
+  global ctr_events
 
   ctr_responses = ctr_responses + 1
 
@@ -384,18 +468,82 @@ def tc510(ftptype):
 
   msg = ""
 
-  for evts in range(700):  # build events for 5 MEs
-    if (evts > 0):
+  for pnfs in range(700):  # build events for 700 MEs
+    if (pnfs > 0):
+      msg = msg + ","
+    nodeName = createNodeName(pnfs)
+    msg = msg + getEventHead(nodeName)
+    seqNr = (ctr_responses-1)
+    fileName = createFileName(nodeName, seqNr, "1MB")
+    msg = msg + getEventName(fileName,ftptype,"onap","pano")
+    seqNr = seqNr + pnfs*1000000 #Create unique id for this node and file
+    fileMap[seqNr] = seqNr
+    msg = msg + getEventEnd()
+    ctr_events = ctr_events+1
+
+  return buildOkResponse("["+msg+"]")
+
+def tc511(ftptype):
+  global ctr_responses
+  global ctr_unique_files
+  global ctr_events
+
+  ctr_responses = ctr_responses + 1
+
+  if (ctr_responses > 5):
+    return buildOkResponse("[]")  
+
+  msg = ""
+
+  for pnfs in range(700):  # build events for 700 MEs
+    if (pnfs > 0):
       msg = msg + ","
-    msg = msg + getEventHeadNodeName("PNF"+str(evts))
+    nodeName = createNodeName(pnfs)
+    msg = msg + getEventHead(nodeName)
     seqNr = (ctr_responses-1)
-    msg = msg + getEventName("1MB_" + str(seqNr) + ".tar.gz",ftptype,"onap","pano","localhost",1022)
-    seqNr = seqNr + evts*1000000 #Create unique id for this node and file
+    fileName = createFileName(nodeName, seqNr, "1KB")
+    msg = msg + getEventName(fileName,ftptype,"onap","pano")
+    seqNr = seqNr + pnfs*1000000 #Create unique id for this node and file
     fileMap[seqNr] = seqNr
     msg = msg + getEventEnd()
+    ctr_events = ctr_events+1
 
   return buildOkResponse("["+msg+"]")
 
+def tc710(ftptype):
+  global ctr_responses
+  global ctr_unique_files
+  global ctr_events
+
+  ctr_responses = ctr_responses + 1
+  
+  if (ctr_responses > 100):
+    return buildOkResponse("[]")
+
+  msg = ""
+  
+  batch = (ctr_responses-1)%20;  
+
+  for pnfs in range(35):  # build events for 35 PNFs at a time. 20 batches -> 700
+    if (pnfs > 0):
+      msg = msg + ","
+    nodeName = createNodeName(pnfs + batch*35)
+    msg = msg + getEventHead(nodeName)
+
+    for i in range(100):  # 100 files per event
+      seqNr = i + int((ctr_responses-1)/20);
+      if i != 0: msg = msg + ","
+      fileName = createFileName(nodeName, seqNr, "1MB")
+      msg = msg + getEventName(fileName,ftptype,"onap","pano")
+      seqNr = seqNr + (pnfs+batch*35)*1000000 #Create unique id for this node and file
+      fileMap[seqNr] = seqNr
+
+    msg = msg + getEventEnd()
+    ctr_events = ctr_events+1
+
+  return buildOkResponse("["+msg+"]")
+
+
 #Mapping FTPS TCs
 def tc200(ftptype):
   return tc100(ftptype)
@@ -419,6 +567,14 @@ def tc221(ftptype):
   return tc121(ftptype)
 def tc222(ftptype):
   return tc122(ftptype)
+  
+def tc610(ftptype):
+  return tc510(ftptype)
+def tc611(ftptype):
+  return tc511(ftptype)
+  
+def tc810(ftptype):
+  return tc710(ftptype)
 
 def tc2000(ftptype):
   return tc1000(ftptype)
@@ -427,11 +583,21 @@ def tc2001(ftptype):
 
 #### Functions to build json messages and respones ####
 
+def createNodeName(index):
+    return "PNF"+str(index);
+
+def createFileName(nodeName, index, size):
+    return "A20000626.2315+0200-2330+0200_" + nodeName + "-" + str(index) + "-" +size + ".tar.gz";
+
+def createMissingFileName(nodeName, index, size):
+    return "AMissingFile_" + nodeName + "-" + str(index) + "-" +size + ".tar.gz";
+
+
 # Function to build fixed beginning of an event
-def getEventHead():
-  return getEventHeadNodeName("oteNB5309")
 
-def getEventHeadNodeName(nodename):
+def getEventHead(nodename):
+  global pnfMap
+  pnfMap.add(nodename) 
   headStr = """
         {
           "event": {
@@ -461,7 +627,13 @@ def getEventHeadNodeName(nodename):
   return headStr
 
 # Function to build the variable part of an event
-def getEventName(fn,type,user,passwd,ip,port):
+def getEventName(fn,type,user,passwd):
+    port = SFTP_PORT
+    ip = sftp_ip
+    if (type == "ftps"):
+        port = FTPS_PORT
+        ip = ftps_ip
+        
     nameStr =        """{
                   "name": \"""" + fn + """",
                   "hashMap": {
@@ -501,15 +673,26 @@ def buildEmptyResponse(status_code):
 
 
 if __name__ == "__main__":
-  
+
+    # IP addresses to use for ftp servers, using localhost if not env var is set
+    sftp_ip = os.environ.get('SFTP_SIM_IP', 'localhost')
+    ftps_ip = os.environ.get('FTPS_SIM_IP', 'localhost')
+    
+    
+
     #Counters
     ctr_responses = 0
     ctr_requests = 0
     ctr_unique_files = 0
+    ctr_events = 0
+    startTime = time.time()
 
-    #Keeps all reponded file names
+    #Keeps all responded file names
     fileMap = {}
 
+    #Keeps all responded PNF names
+    pnfMap = set()
+
     tc_num = "Not set"
     tc_help = "Not set"
 
@@ -572,9 +755,17 @@ if __name__ == "__main__":
     parser.add_argument(
         '--tc510',
         action='store_true',
-        help='TC510 - 5 MEs, SFTP, 1MB files, 1 file per event, 100 events, 1 event per poll.')
-
+        help='TC510 - 700 MEs, SFTP, 1MB files, 1 file per event, 3500 events, 700 event per poll.')
+    
+    parser.add_argument(
+        '--tc511',
+        action='store_true',
+        help='TC511 - 700 MEs, SFTP, 1KB files, 1 file per event, 3500 events, 700 event per poll.')
 
+    parser.add_argument(
+        '--tc710',
+        action='store_true',
+        help='TC710 - 700 MEs, SFTP, 1MB files, 100 files per event, 3500 events, 35 event per poll.')
 
 # FTPS TCs with single ME
     parser.add_argument(
@@ -632,11 +823,21 @@ if __name__ == "__main__":
     parser.add_argument(
         '--tc610',
         action='store_true',
-        help='TC510 - 5 MEs, FTPS, 1MB files, 1 file per event, 100 events, 1 event per poll.')
+        help='TC610 - 700 MEs, FTPS, 1MB files, 1 file per event, 3500 events, 700 event per poll.')
+
+    parser.add_argument(
+        '--tc611',
+        action='store_true',
+        help='TC611 - 700 MEs, FTPS, 1KB files, 1 file per event, 3500 events, 700 event per poll.')
+
+    parser.add_argument(
+        '--tc810',
+        action='store_true',
+        help='TC810 - 700 MEs, FTPS, 1MB files, 100 files per event, 3500 events, 35 event per poll.')
 
     args = parser.parse_args()
 
-    
+
 
     if args.tc100:
         tc_num = "TC# 100"
@@ -668,6 +869,11 @@ if __name__ == "__main__":
 
     elif args.tc510:
         tc_num = "TC# 510"
+    elif args.tc511:
+        tc_num = "TC# 511"
+        
+    elif args.tc710:
+        tc_num = "TC# 710"
 
     elif args.tc200:
         tc_num = "TC# 200"
@@ -697,14 +903,25 @@ if __name__ == "__main__":
     elif args.tc2001:
         tc_num = "TC# 2001"
 
+
     elif args.tc610:
         tc_num = "TC# 610"
+    elif args.tc611:
+        tc_num = "TC# 611"
+           
+    elif args.tc810:
+        tc_num = "TC# 810"
 
     else:
         print("No TC was defined")
         print("use --help for usage info")
         sys.exit()
 
-    print(tc_num)
- 
+    print("TC num: " + tc_num)
+    
+        
+    print("Using " + sftp_ip + " for sftp server address in file urls.")
+    print("Using " + ftps_ip + " for ftps server address in file urls.")
+
     app.run(port=HOST_PORT, host=HOST_IP)
+
diff --git a/test/mocks/datafilecollector-testharness/simulator-group/README.md b/test/mocks/datafilecollector-testharness/simulator-group/README.md
index 4d448f8d2..5981c7978 100644
--- a/test/mocks/datafilecollector-testharness/simulator-group/README.md
+++ b/test/mocks/datafilecollector-testharness/simulator-group/README.md
@@ -4,15 +4,17 @@ Mainly this is needed for CSIT tests but can be used also for local testing.
 
 
 ###Preparation 
+Do the manual steps to prepare the simulator images
+
 Build the mr-sim image.
 
 cd ../mr-sim
 
-Run the docker build command to build the image for the MR simulator: 'docker build -t mrsim:latest .
+Run the docker build command to build the image for the MR simulator: 'docker build -t mrsim:latest .'
 
 cd ../dr-sim
 
-Run the docker build command to build the image for the DR simulators: `docker build -t drsim_common:latest . 
+Run the docker build command to build the image for the DR simulators: `docker build -t drsim_common:latest .'
 
 
 cd ../simulator-group
@@ -21,21 +23,53 @@ Copy the 'configuration' and 'tls' catalogues from the ftps-sftp-server dir.
 
 Check the README.md in ftps-sftp-server dir in case the cert need to be updated.
 
-cp -r ./ftps-sftp-server/configuration .
+cp -r ../ftps-sftp-server/configuration .
 
 cp -r ../ftps-sftp-server/tls .
 
 
 ###Execution
 
-Edit the `docker-compose-setup.sh` to setup the env variables to the desired test behavior for each simulators.
+Edit the `docker-compose-setup.sh` (or create a copy) to setup the env variables to the desired test behavior for each simulators.
 See each simulator to find a description of the available settings.
+The following env variables shall be set (example values).
+Note that NUM_FTPFILES and NUM_PNFS controls the number of ftp files created in the ftp servers. 
+A total of NUM_FTPFILES * NUM_PNFS ftp files will be created in each dtp server (4 files in the below example). 
+Large settings will be time consuming at start of the servers.
+
+DR_TC="--tc normal"           #Normal behavior of the DR sim
+
+DR_REDIR_TC="--tc normal"     #Normal behavior of the DR redirect sim
+
+MR_TC="--tc100"               #One 1 MB file in one event, once. 
+
+BC_TC=""                      #Not in use yet
+
+NUM_FTPFILES="2"              #Two file for each PNF
 
-Run the script `docker-compose-setup.sh`to create a docker-compose with the desired settings. All simulators
-will be started with the generated docker-compose.yml file
+NUM_PNFS="2"                  #Two PNFs
+
+Run the script `docker-compose-setup.sh`to create a docker-compose with the desired settings. The desired setting
+in the script need to be manually adapted to for each specific simulator behavior according to the above. Check each simulator for available
+parameters.
+All simulators will be started with the generated docker-compose.yml file
+
+To generate ftp url with IP different from localhost, set SFTP_SIM_IP and/or FTPS_SIM_IP env variables to the addreses of the ftp servers before starting. 
+So farm, this only works when the simulator python script is started from the command line.
 
 Kill all the containers with `simulators-kill.se`
 
 `simulators_start.sh` is for CSIT test and requires the env variables for test setting to be present in the shell.
 `setup-ftp-files.for-image.sh` is for CSIT and executed when the ftp servers are started from the docker-compose-setup.sh`.
 
+To make DFC to be able to connect to the simulator containers, DFC need to run in host mode.
+Start DFC by the following cmd: `docker run -d --network="host" --name dfc_app <dfc-image> `
+
+`<dfc-image>` could be either the locally built image `onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server`
+or the one in nexus `nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server`. 
+
+
+
+###Simulator monitor
+Start the simulator monitor server with `sim-monitor-start.sh` and the open a browser with the url `localhost:9999/mon`
+to see the statisics page with data from MR sim, DR sim and DR redir sim.
diff --git a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh
index d0171a5e3..af36d0562 100755
--- a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh
+++ b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-setup.sh
@@ -2,10 +2,13 @@
 
 #Script for manually starting all simulators with test setting below
 
-export DR_TC=""
-export DR_REDIR_TC=""
+export DR_TC="--tc normal"
+export DR_REDIR_TC="--tc normal"
 export MR_TC="--tc100"
-export BC_TC=""
-export NUM_FTPFILES="250"
+export BC_TC=""  #Not in use yet
+export NUM_FTPFILES="10"
+export NUM_PNFS="700"
+export FILE_SIZE="1MB"
+export FTP_TYPE="SFTP"
 
 source ./simulators-start.sh
\ No newline at end of file
diff --git a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml
index e7c700759..89a45a476 100644
--- a/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml
+++ b/test/mocks/datafilecollector-testharness/simulator-group/docker-compose-template.yml
@@ -1,8 +1,17 @@
 version: '2'
+
+networks:
+  dfcnet:
+    ipam:
+      config:
+        - subnet: 192.168.100.0/16
+          
 services:
       
   drsim:
-    network_mode: "host"
+    networks:
+      dfcnet:
+        ipv4_address: 192.168.100.2
     image: drsim_common:latest
     ports:
      - "3906:3906"
@@ -11,6 +20,9 @@ services:
     command: node dmaapDR.js ${DR_TC}
 
   drsim_redir:
+    networks:
+      dfcnet:
+        ipv4_address: 192.168.100.3
     image: drsim_common:latest
     ports:
      - "3908:3908"
@@ -19,6 +31,9 @@ services:
     command: node dmaapDR_redir.js ${DR_REDIR_TC}
 
   mrsim:
+    networks:
+      dfcnet:
+        ipv4_address: 192.168.100.1
     image: mrsim:latest
     ports:
      - "2222:2222"
@@ -26,17 +41,17 @@ services:
     command: python mr-sim.py ${MR_TC}
 
   sftp-server:
+    network_mode: bridge
     container_name: dfc_sftp-server
     image: atmoz/sftp:alpine
     ports:
       - "1022:22"
-#    volumes:
-#      - ./files/onap/sftp/:/home/onap/
     restart: on-failure
     command: onap:pano:1001
 
 
   ftpes-server-vsftpd:
+    network_mode: bridge
     container_name: dfc_ftpes-server-vsftpd
     image: docker.io/panubo/vsftpd
     ports:
@@ -54,6 +69,5 @@ services:
       - ./tls/dfc.crt:/etc/ssl/private/dfc.crt:ro
       - ./configuration/vsftpd_ssl.conf:/etc/vsftpd_ssl.conf:ro
 
-#      - ./files/onap/ftps/:/srv/
     restart: on-failure
   
\ No newline at end of file
diff --git a/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh b/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh
index d8e57a3d8..6c2b4f29f 100755
--- a/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh
+++ b/test/mocks/datafilecollector-testharness/simulator-group/setup-ftp-files-for-image.sh
@@ -1,28 +1,47 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Script to create files for the FTP server to return upon request.
 # The file names matches the files names in the events polled from the MR simulator.
 # Intended for execution in the running ftp containers in the ftp-root dir.
 
-echo "Running ftp file creations"
-
 NUM=200 #Default number of files 
+PNFS=1 #Default number of PNFs
+FSIZE="ALL"
 
 if [ $# -eq 1 ]; then 
     NUM=$1
+elif [ $# -eq 2 ]; then
+    NUM=$1
+    PNFS=$2
+elif [ $# -eq 3 ]; then
+	NUM=$1
+    PNFS=$2
+    FSIZE=$3
+    if [ $3 != "1KB" ] && [ $3 != "1MB" ] && [ $3 != "5MB" ]  && [ $3 != "50MB" ]  && [ $3 != "ALL" ]; then
+    	echo "File size shall be 1KB|1MB|5MB|50MB|ALL"
+    	exit
+    fi
+else
+    echo "Wrong args, usage: setup-ftp-files-for-image.sh [ <num-files> [ <num-PNFs> [ 1KB|1MB|5MB|50MB ] ] ]"
+    exit
 fi
 
+echo "Running ftp file creations. " $PNFS " PNFs and " $NUM " files for each PNF with file size(s) "$FSIZE
+
+truncate -s 1KB 1KB.tar.gz
 truncate -s 1MB 1MB.tar.gz
 truncate -s 5MB 5MB.tar.gz
 truncate -s 50MB 50MB.tar.gz
 
-
-i=0
-while [ $i -lt $NUM ]; do  #Problem with for loop and var substituion in curly bracket....so used good old style loop
-   ln -s 1MB.tar.gz 1MB_$i.tar.gz
-   ln -s 5MB.tar.gz 5MB_$i.tar.gz
-   let i=i+1
+p=0
+while [ $p -lt $PNFS ]; do 
+    i=0
+    while [ $i -lt $NUM ]; do  #Problem with for loop and var substituion in curly bracket....so used good old style loop
+    	if [ $FSIZE = "ALL" ] || [ $FSIZE = "1KB" ]; then ln -s 1KB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-1KB.tar.gz'; fi
+        if [ $FSIZE = "ALL" ] || [ $FSIZE = "1MB" ]; then ln -s 1MB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-1MB.tar.gz'; fi
+        if [ $FSIZE = "ALL" ] || [ $FSIZE = "5MB" ]; then ln -s 5MB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-5MB.tar.gz'; fi
+        if [ $FSIZE = "ALL" ] || [ $FSIZE = "50MB" ]; then ln -s 50MB.tar.gz 'A20000626.2315+0200-2330+0200_PNF'$p'-'$i'-50MB.tar.gz'; fi
+    let i=i+1
+    done
+    let p=p+1
 done
-
-
-ln -s 50MB.tar.gz 50MB_0.tar.gz   #Large file, only for single file test
diff --git a/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor-start.sh b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor-start.sh
new file mode 100755
index 000000000..52c8c1cac
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor-start.sh
@@ -0,0 +1,7 @@
+#/bin/bash
+
+#Script to start the sim-monitor
+
+#Re-using modules for dr-sim
+cp -r ../dr-sim/node_modules .
+node sim-monitor.js
\ No newline at end of file
diff --git a/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor.js b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor.js
new file mode 100644
index 000000000..e4a19c21e
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/simulator-group/sim-monitor.js
@@ -0,0 +1,165 @@
+var http = require('http');
+
+var express = require('express');
+var app = express();
+
+//I am alive
+app.get("/",function(req, res){
+	res.send("ok");
+})
+
+//Get parameter valuye from other server
+function getSimCtr(url, cb) {
+    var data = '';
+	http.get(url, (resp) => {
+  		// A chunk of data has been recieved.
+  		resp.on('data', (chunk) => {
+    		data += chunk;
+  		});
+
+  		// The whole response has been received.
+  		resp.on('end', () => {
+  			//Pad data to fixed length
+  			var i = 20-data.length;
+  			while(i>0) {
+  				data = data+"&nbsp;";
+  				i--;
+  			}
+    		cb(data);
+  		});
+
+	}).on("error", (err) => {
+  		console.log("Error: " + err.message);
+  		cb("no response from simulator");
+	});
+};
+
+//Status variables, for parameters values fetched from other simulators
+var mr1, mr2, mr3, mr4, mr5, mr6, mr7;
+
+var dr1, dr2, dr3, dr4, dr5, dr6, dr7, dr8, dr9;
+
+var drr1, drr2, drr3, drr4, drr5, drr6;
+
+app.get("/mon",function(req, res){
+
+	//MR
+    getSimCtr("http://127.0.0.1:2222/ctr_requests", function(data) {
+    	mr1 = data;
+    });
+    getSimCtr("http://127.0.0.1:2222/ctr_responses", function(data) {
+    	mr2 = data;
+    });
+    getSimCtr("http://127.0.0.1:2222/ctr_unique_files", function(data) {
+    	mr3 = data;
+    });
+    getSimCtr("http://127.0.0.1:2222/tc_info", function(data) {
+    	mr4 = data;
+    });
+    getSimCtr("http://127.0.0.1:2222/ctr_events", function(data) {
+    	mr5 = data;
+    });
+    getSimCtr("http://127.0.0.1:2222/execution_time", function(data) {
+    	mr6 = data;
+    });
+    getSimCtr("http://127.0.0.1:2222/ctr_unique_PNFs", function(data) {
+    	mr7 = data;
+    });
+
+    //DR
+    getSimCtr("http://127.0.0.1:3906/ctr_publish_query", function(data) {
+    	dr1 = data;
+    });
+    getSimCtr("http://127.0.0.1:3906/ctr_publish_query_published", function(data) {
+    	dr2 = data;
+    });    
+    getSimCtr("http://127.0.0.1:3906/ctr_publish_query_not_published", function(data) {
+    	dr3 = data;
+    });
+    getSimCtr("http://127.0.0.1:3906/ctr_publish_req", function(data) {
+    	dr4 = data;
+    });
+    getSimCtr("http://127.0.0.1:3906/ctr_publish_req_redirect", function(data) {
+    	dr5 = data;
+    });
+    getSimCtr("http://127.0.0.1:3906/ctr_publish_req_published", function(data) {
+    	dr6 = data;
+    });
+    getSimCtr("http://127.0.0.1:3906/ctr_published_files", function(data) {
+    	dr7 = data;
+    });
+    getSimCtr("http://127.0.0.1:3906/tc_info", function(data) {
+    	dr8 = data;
+    });
+    getSimCtr("http://127.0.0.1:3906/execution_time", function(data) {
+    	dr9 = data;
+    });
+ 
+    //DR REDIR
+    getSimCtr("http://127.0.0.1:3908/ctr_publish_requests", function(data) {
+    	drr1 = data;
+    });
+    getSimCtr("http://127.0.0.1:3908/ctr_publish_responses", function(data) {
+    	drr2 = data;
+    });
+    getSimCtr("http://127.0.0.1:3908/tc_info", function(data) {
+    	drr3 = data;
+    });
+    getSimCtr("http://127.0.0.1:3908/execution_time", function(data) {
+    	drr4 = data;
+    });
+    getSimCtr("http://127.0.0.1:3908/time_lastpublish", function(data) {
+    	drr5 = data;
+    });
+    getSimCtr("http://127.0.0.1:3908/dwl_volume", function(data) {
+    	drr6 = data;
+    });
+
+  //Build web page
+	var str = "<!DOCTYPE html>" +
+          "<html>" +
+          "<head>" +
+            "<meta http-equiv=\"refresh\" content=\"5\">"+  //5 sec auto reefresh
+            "<title>Simulator monitor</title>"+
+          "</head>" +
+          "<body>" +
+            "<h3>MR Simulator</h3>" +
+            "<font face=\"courier\">"+
+            "MR TC:........................................." + mr4 + "<br>" +
+            "Execution time (mm.ss):........................" + mr6 + "<br>" +
+            "Number of requests (polls):...................." + mr1 + "<br>" +
+            "Number of responses (polls):..................." + mr2 + "<br>" +
+            "Number of unique files in all responses:......." + mr3 + "<br>" +
+            "Number of events..............................." + mr5 + "<br>" +
+            "Number of unique PNFs.........................." + mr7 + "<br>" +
+            "</font>"+
+            "<h3>DR Simulator</h3>" +
+            "<font face=\"courier\">"+
+            "DR TC:........................................." + dr8 + "<br>" +
+            "Execution time (mm.ss):........................" + dr9 + "<br>" +
+            "Number of queries:............................." + dr1 + "<br>" +
+            "Number of query responses, file published:....." + dr2 + "<br>" +
+            "Number of query responses, file not published:." + dr3 + "<br>" +
+            "Number of requests:............................" + dr4 + "<br>" +
+            "Number of responses with redirect:............." + dr5 + "<br>" +
+            "Number of responses without redirect:.........." + dr6 + "<br>" +
+            "Number of published files:....................." + dr7 + "<br>" +
+            "</font>"+
+            "<h3>DR Redirect Simulator</h3>" +
+            "<font face=\"courier\">"+
+            "DR REDIR TC:..................................." + drr3 + "<br>" +
+            "Execution time (mm.ss):........................" + drr4 + "<br>" +
+            "Number of requests:............................" + drr1 + "<br>" +
+            "Number of responses:..........................." + drr2 + "<br>" +
+            "Downloaded volume (bytes):....................." + drr6 + "<br>" +
+            "Last publish (mm:ss):.........................." + drr5 + "<br>" +
+            "</font>"+
+           "</body>" +
+          "</html>";
+	res.send(str);
+})
+
+var httpServer = http.createServer(app);
+var httpPort=9999;
+httpServer.listen(httpPort);
+console.log("Simulator monitor listening (http) at "+httpPort);
\ No newline at end of file
diff --git a/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh b/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh
index ef1b90a38..cc5ce32ec 100755
--- a/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh
+++ b/test/mocks/datafilecollector-testharness/simulator-group/simulators-start.sh
@@ -35,11 +35,27 @@ if [ -z "$NUM_FTPFILES" ]
  then
  NUM_FTPFILES=200
 fi
+if [ -z "$NUM_PNFS" ]
+ then
+ NUM_PNFS=1
+fi
+if [ -z "$FILE_SIZE" ]
+ then
+ FILE_SIZE="ALL"
+fi
+if [ -z "$FTP_TYPE" ]
+ then
+ FTP_TYPE="ALL"
+fi
 
-
-docker cp setup-ftp-files-for-image.sh $SFTP_SIM:/tmp/
-docker exec -w /home/onap/ $SFTP_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES >/dev/null 2>&1
-
-docker cp setup-ftp-files-for-image.sh $FTPS_SIM:/tmp/setup-ftp-files-for-image.sh
-docker exec -w /srv $FTPS_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES >/dev/null 2>&1
-
+if [ $FTP_TYPE = "ALL" ] || [ $FTP_TYPE = "SFTP" ]; then
+	echo "Creating files for SFTP server, may take time...."
+	docker cp setup-ftp-files-for-image.sh $SFTP_SIM:/tmp/
+	docker exec -w /home/onap/ $SFTP_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES $NUM_PNFS $FILE_SIZE #>/dev/null 2>&1
+fi
+if [ $FTP_TYPE = "ALL" ] || [ $FTP_TYPE = "FTPS" ]; then
+	echo "Creating files for FTPS server, may take time...."
+	docker cp setup-ftp-files-for-image.sh $FTPS_SIM:/tmp/setup-ftp-files-for-image.sh
+	docker exec -w /srv $FTPS_SIM /tmp/setup-ftp-files-for-image.sh $NUM_FTPFILES $NUM_PNFS $FILE_SIZE #>/dev/null 2>&1
+fi
+echo "Done: All simulators started and configured"