Adjust DFC test files to use FTPES instead of FTPS

DFC doesn't support FTPS protocol. FTPS is treated exactly like
FTPES and soon DFC won't accept file  locations starting with
"ftps://".

Fix mr-sim.py lint issues

Issue-ID: DCAEGEN2-2367
Signed-off-by: Remigiusz Janeczek <remigiusz.janeczek@nokia.com>
Change-Id: Idab56aab3bdb6734a04bfc9c9212f2ea800d0d74

11 files changed, 270 insertions, 0 deletions

diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/.gitignore b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/.gitignore
new file mode 100644
index 000000000..bd6c5bed8
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/.gitignore
@@ -0,0 +1 @@
+files/onap/*
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/Dockerfile-ftpes b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/Dockerfile-ftpes
new file mode 100644
index 000000000..a0d6cfafc
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/Dockerfile-ftpes
@@ -0,0 +1,5 @@
+FROM docker.io/panubo/vsftpd
+
+COPY configuration/vsftpd_ssl.conf /etc/vsftpd_ssl.conf
+RUN chmod 644 /etc/vsftpd_ssl.conf
+RUN chown root:root /etc/vsftpd_ssl.conf
\ No newline at end of file
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/README.md b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/README.md
new file mode 100644
index 000000000..44d329e76
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/README.md
@@ -0,0 +1,37 @@
+# Deployment of certificates: (in case of update)
+
+This folder is prepared with a set of keys matching DfC for test purposes.
+
+Copy from datafile-app-server/config/keys to the ./tls/ the following files:
+
+- dfc.crt
+- ftp.crt
+- ftp.key
+
+# Docker preparations
+
+Source: <https://docs.docker.com/install/linux/linux-postinstall/>
+
+`sudo usermod -aG docker $USER`
+
+then logout-login to activate it.
+
+# Prepare files for the simulator
+
+Run `prepare.sh` with an argument found in `test_cases.yml` (or add a new tc in that file) to create files (1MB, 5MB and 50MB files) and a large number of 
+symbolic links to these files to simulate PM files. The files names maches the files in
+the events produced by the MR simulator. The dirs with the files will be mounted
+by the ftp containers, defined in the docker-compse file, when started
+
+# Starting/stopping the FTPES/SFTP server(s)
+
+Start: `docker-compose up`
+
+Stop: Ctrl +C, then `docker-compose down`  or `docker-compose down --remove-orphans`
+
+If you experience issues (or port collision), check the currently running other containers
+by using 'docker ps' and stop them if necessary.
+
+# Cleaning docker structure
+
+Deep cleaning: `docker system prune`
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/configuration/vsftpd_ssl.conf b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/configuration/vsftpd_ssl.conf
new file mode 100644
index 000000000..0a24e38a8
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/configuration/vsftpd_ssl.conf
@@ -0,0 +1,57 @@
+# Server Config
+anonymous_enable=NO
+local_enable=YES
+write_enable=YES
+local_umask=022
+dirmessage_enable=YES
+
+# Security and User auth
+chroot_local_user=YES
+pam_service_name=vsftpd_virtual
+virtual_use_local_privs=YES
+chmod_enable=NO
+user_config_dir=/etc/vsftpd/user_conf
+user_sub_token=$USER
+#local_root=/srv/$USER
+local_root=/srv/
+userlist_enable=NO
+allow_writeable_chroot=YES
+
+# Logging
+log_ftp_protocol=YES
+xferlog_enable=YES
+xferlog_std_format=YES
+#xferlog_file=/dev/stdout
+syslog_enable=NO
+dual_log_enable=YES
+
+# Remap all login users to this username
+guest_enable=YES
+guest_username=ftp
+hide_ids=YES
+
+# Networking
+connect_from_port_20=NO
+listen=YES
+tcp_wrappers=YES
+#pasv_min_port=8001
+#pasv_max_port=8010
+
+# SSL
+ssl_enable=Yes
+require_ssl_reuse=NO
+force_local_data_ssl=YES
+force_local_logins_ssl=YES
+ssl_ciphers=HIGH
+allow_anon_ssl=NO
+
+ssl_tlsv1=YES
+ssl_sslv2=YES
+ssl_sslv3=YES
+rsa_cert_file=/etc/ssl/private/cert.pem
+rsa_private_key_file=/etc/ssl/private/key.pem
+
+require_cert=YES
+ssl_request_cert=YES
+ca_certs_file=/etc/ssl/private/cacert.pem
+
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/docker-compose.yml b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/docker-compose.yml
new file mode 100644
index 000000000..5bc91efa1
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/docker-compose.yml
@@ -0,0 +1,35 @@
+version: '3'
+
+services:
+
+  sftp-server1:
+    container_name: sftp-server1
+    image: atmoz/sftp:alpine
+    ports:
+      - "1022:22"
+    volumes:
+      - ./files/onap/sftp/:/home/onap/
+    restart: on-failure
+    command: onap:pano:1001
+
+  ftpes-server-vsftpd:
+    container_name: ftpes-server-vsftpd
+    image: docker.io/panubo/vsftpd
+    ports:
+      - "21:21"
+      - "8001-8010:8001-8010"
+    environment:
+      FTP_USER: onap
+      FTP_PASSWORD: pano
+      PASV_ADDRESS: localhost
+      PASV_MIN_PORT: 8001
+      PASV_MAX_PORT: 8010
+    volumes:
+      - ./tls/ftp.crt:/etc/ssl/private/ftp.crt:ro
+      - ./tls/ftp.key:/etc/ssl/private/ftp.key:ro
+      - ./tls/dfc.crt:/etc/ssl/private/dfc.crt:ro
+      - ./configuration/vsftpd_ssl.conf:/etc/vsftpd_ssl.conf:ro
+
+      - ./files/onap/ftpes/:/srv/
+    restart: on-failure
+    command: vsftpd /etc/vsftpd_ssl.conf
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/prepare.sh b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/prepare.sh
new file mode 100755
index 000000000..086d43a49
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/prepare.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+# EXAMPLE: Run test case TC2 using the command "./prepare.sh TC2"
+MAIN_DIRECTORY=./files/onap
+TEST_FILE=./test_cases.yml
+TEST=$1
+echo "Generating files for test case:" "$TEST"
+
+sf=$(sed -n '/'$TEST'/,$p' $TEST_FILE | grep -m 1 'size_files')
+sf=${sf//*size_files: /}
+sf_array=($sf)
+echo "size_files=""$sf"
+
+nf=$(sed -n '/'$TEST'/,$p' $TEST_FILE | grep -m 1 'number_files')
+nf=${nf//*number_files: /}
+nf_array=($nf)
+echo "number_files=""$nf"
+
+df=$(sed -n '/'$TEST'/,$p' $TEST_FILE | grep -m 1 'directory_files')
+df=${df//*directory_files: /}
+df_array=($df)
+echo "directory_files=""$df"
+
+rm -rf $MAIN_DIRECTORY/*
+if [ "${#sf_array[@]}" = "${#nf_array[@]}" ] && [ "${#nf_array[@]}" = "${#df_array[@]}" ];
+then
+    N_ELEMENTS=${#df_array[@]}
+	for ((n=0;n<$N_ELEMENTS;n++))
+	do
+		# Create directory
+		DIRECTORY=$MAIN_DIRECTORY/${df_array[$n]}
+		mkdir -p "$DIRECTORY"
+
+		# Create original file
+		FILE_SIZE=${sf_array[$n]}
+		FILE_NAME=$FILE_SIZE"MB.tar.gz"
+		dd if=/dev/urandom of=$DIRECTORY/$FILE_NAME bs=1k count=$(echo $FILE_SIZE*1000/1 | bc)
+
+		# Create symlinks
+		N_SYMLINKS=${nf_array[$n]}-1
+		for ((l=0;l<=$N_SYMLINKS;l++))
+		do
+			SYMLINK_NAME=$FILE_SIZE"MB_"$l".tar.gz"
+			ln -s ./$FILE_NAME $DIRECTORY/$SYMLINK_NAME
+		done
+	done
+else
+echo "ERROR: The number of parameters in size_files, number_files, and directory_files must be equal!"
+fi
+
+sudo chown root:root ./configuration/vsftpd_ssl.conf
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/test_cases.yml b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/test_cases.yml
new file mode 100644
index 000000000..e07e3a0c6
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/test_cases.yml
@@ -0,0 +1,16 @@
+# EXAMPLE: TC1 generates (i) 10 files of 1 MB in ftpes directory,
+# (ii) 30 files of 5 MB in sftp directory, and (iii) 10 files of 10 MB in sftp directory
+TC1:
+  size_files: 1 5 10
+  number_files: 10 30 10
+  directory_files: ftpes sftp sftp
+
+TC2:
+  size_files: 0.5 1 5
+  number_files: 2 3 1
+  directory_files: ftpes ftpes sftp
+
+TC_10000:
+  size_files: 1 1 5 5 50 50
+  number_files: 10000 10000 10000 10000 1 1
+  directory_files: ftpes sftp ftpes sftp ftpes sftp
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/README.md b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/README.md
new file mode 100644
index 000000000..5edfeddec
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/README.md
@@ -0,0 +1,4 @@
+# To verify the certificate expiration dates:
+
+openssl x509 -enddate -noout -in dfc.crt
+openssl x509 -enddate -noout -in ftp.crt
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/dfc.crt b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/dfc.crt
new file mode 100644
index 000000000..f747f20bb
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/dfc.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDFjCCAf4CCQCqH10CLXcbUDANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJT
+RTEKMAgGA1UECAwBMDEKMAgGA1UEBwwBMDEKMAgGA1UECgwBMDEMMAoGA1UECwwD
+RVNUMQwwCgYDVQQDDANFU1QwHhcNMTkwNDA0MDgwMjQwWhcNMjAwNDAzMDgwMjQw
+WjBNMQswCQYDVQQGEwJTRTEKMAgGA1UECAwBMDEKMAgGA1UEBwwBMDEKMAgGA1UE
+CgwBMDEMMAoGA1UECwwDRVNUMQwwCgYDVQQDDANFU1QwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDAY7wM9khd7OIaZsfZv8EzWUnLTepzSx6hlAvjSbU/
+ZBpq94QOJWt22hQQSHA+Vdl8184PoyOKX+qRfbKJCtyArbv0DoWjlv16WNs938A/
+0TGFh+6xA464/GQmP/AXEuI0tSa2GEgXkhZ0uy6Pmdq+8sD6YcRyVCeqLTq8bqSq
+YbpPrKnbZsd3l7PzpYCZgZLfWoENQ7nuT+C7j4pGGnPKpGn4ubiscV3nTI6nwU19
+ANexToikIL1v18z+gJdva8QtKih5dt9+2QJuJ6lPcwTa7xVkMmuVyr5FXwL11yII
+gKRtknWObU7BjcKkQOG3xnWXZTNzLV/th8GCCWJuRFPbAgMBAAEwDQYJKoZIhvcN
+AQELBQADggEBAGZa23j04vghZY+/81FyRsoeJs8o9aDeK5KLT9JUpRaZlu4AEXSN
+I3oEhencocc07DTndfxw7eSALnTD9ibGUxnRyEvgtW595ajb7A3TtEKGsFzXFvar
+y9RdVm6RRA4Bty8UyLW75ZpAlfc3twCmsRwKjShXeN90Yn3sjpcEU83q8fZBU35C
+xrgrW34gphPtuzZ7kvCJGjhYIfIsTcWObvDGUROtlRhb8w2v1K4G3UcfwDTBixK/
+e61mMUbhoqVyoMYgSuU4idY+n4Evjs5IwalXM1daUZmCGGZoreD/9aSpDEeUWnQy
+vqYC0YY0VJkOe+WI6JZ2r49BtRtl/jxWDUI=
+-----END CERTIFICATE-----
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.crt b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.crt
new file mode 100644
index 000000000..f412d013c
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDFjCCAf4CCQDaiGveWOXqNjANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJT
+RTEKMAgGA1UECAwBMDEKMAgGA1UEBwwBMDEKMAgGA1UECgwBMDEMMAoGA1UECwwD
+RVNUMQwwCgYDVQQDDANFU1QwHhcNMTkwNDA0MDgwMjQwWhcNMjAwNDAzMDgwMjQw
+WjBNMQswCQYDVQQGEwJTRTEKMAgGA1UECAwBMDEKMAgGA1UEBwwBMDEKMAgGA1UE
+CgwBMDEMMAoGA1UECwwDRVNUMQwwCgYDVQQDDANFU1QwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDSsF7lN/gXpevQpGj8W/3g3h9AVOE83Z49yJAdyIHF
+PQz6PI+bKutYdORCUZkzsl2fegLzkXl4CmoVIkJRBL1SZkzQXKe+fjfuRr9PQKCC
+lp/LA161Qak+9pz2Oc1lfLbgEdv22RLji3akCQso3G9vlx+rLHPRgbew0iiTViJP
+v3CHwiY89t1ai149OEywhjsJBJjBoj6fvxfvv46QmK7FuV5Tz0vTL/eB/Z9P7jm+
+twHRz9Ae4s97c6UhbFKafHLrwdMK+yz7qe55tpgthCgoedeSB0gXWIiS7RY18dEN
+JUB/FAt64LfOiKBl0aUbcQOgUinorhCN8gcNTn7Hrn+1AgMBAAEwDQYJKoZIhvcN
+AQELBQADggEBAKVGHkTLe5R/fG/C7prxiknD+QXo9WACcZNVKMuKhpJwQh1iwc4h
+4tq9lj//giyRrt+yPVQF8pRTiIdeewLVyf5O1ugxzb68UtHzVJWD6ooFqGmyPFkm
+WOdLvtgbasGPZvO6y8HZA3kxKgONbYcL0sdtRWpp5b+KTEyvN/50jAtvkB/wlaee
+emgdRdsSVZqg1p8dUfF6j3Alzsuff7YzEZEZPoJKYdb1vikvj21+LdzTDSj5WRno
+PWXQhdTTqN5/TNMZRHJp/UZY6hVmQL+ILqVYGiOPotfxGNUyo+WsKJwZfZnq4adh
+BzdSIIsDCZB34Njz/qjIXh307/seNWWhNFw=
+-----END CERTIFICATE-----
diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.key b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.key
new file mode 100644
index 000000000..f90c781d3
--- /dev/null
+++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA0rBe5Tf4F6Xr0KRo/Fv94N4fQFThPN2ePciQHciBxT0M+jyP
+myrrWHTkQlGZM7Jdn3oC85F5eApqFSJCUQS9UmZM0Fynvn437ka/T0CggpafywNe
+tUGpPvac9jnNZXy24BHb9tkS44t2pAkLKNxvb5cfqyxz0YG3sNIok1YiT79wh8Im
+PPbdWotePThMsIY7CQSYwaI+n78X77+OkJiuxbleU89L0y/3gf2fT+45vrcB0c/Q
+HuLPe3OlIWxSmnxy68HTCvss+6nuebaYLYQoKHnXkgdIF1iIku0WNfHRDSVAfxQL
+euC3zoigZdGlG3EDoFIp6K4QjfIHDU5+x65/tQIDAQABAoIBAEs+G5XG6D4hzlbD
+8I53l/JvwT9rUMk46GNuNjG8wsOa6wCPEkY7DLOZg08/7vOsdo0WuOkdggDhz0Le
+6koe5DICQNqEzI9WakkZUQdPsEMS4dxRxsf6vCO1VRcGS5k78d+R4TmGCp3i16r7
+Y9Xi65UxpmiuRmqC5gQq+bysnTJXKUhK/3NCPa7Bwo7hgASJcI55Nk58KHokIv84
+7dweKuIIxeKAR4/094q243lTu1n273J+ckjVMWWZROIIn2E+CrjAVh59DaC7QX6d
+kWdPwvjYiwH2LBqzJ3dKvLmk6XZZ5bcjmBYXSiS1ahjQ8zhDdLoHBt/gDXcmtOVw
+gZPjgn0CgYEA6nJGMig2SRB25CqtA/gwLbQV54th0Vxj1SapoO+W4jIUEPBRJN1t
+0JkM9mTMWWBiEzZi4ICpJpgT/+iGXx6q6WZZqOvbWWS4yjuC+wLCttt2yriFkmlA
+eylz0rYTGm5gQ3wpAUuf0wOfqVICfQ2KnASY0p8g2fSjOI0/pULpX18CgYEA5g72
+UVspmBLqaA/PGYip/7neh00sGs95b9Wh1UqvcrstxkgR4LyMOReTBNwL+39kyZPv
+LNsfMtaiezIfSF+o77RiXnBsYQ/ZnKBNZ7cTRr76f5fKStzRSzHoQrf+98YkSfg3
+sI1vYH/hnfbd/6ti8Wiloc7O28IHCwG4vGXWPWsCgYEA4Oqaj1nmonfKJ6ENdSZw
+Shhff1BHmovxNrctuzi/Xue+OnXi0uQfiySZu/P926zMyjO97jVgkacKYNMZvj10
+qA/J6nXDbOJlKZaoVNlUJgsrztXxRwH0m3OsKzUD5LKJZZTC3fxIKy9pyA3mV0Rb
+eswqNL32zUKWKBXSPmCP9S8CgYEAoJIk3dfRCjF5pjQSinHWpYRniuwUMopI6gOj
+tqkwF9YJNvGqeL9g56BKnwOu4VSjVgaI0zgFaIhiU7ZCG1OZJ+UxG1VAb+aOLkG8
+hP1E2QYG9PNO4D2LXv5wa9DchrenMHGs/O9ao3HeWVCq1f4bSv+NS4h63Otp2wbS
+l7xEcg0CgYBcrapVzHfGn73Z9tgs7GOVEmoXKoT6Q8yrdwwhVrcWdDHHtLFPso21
+eA9dhbbawaEmIReWtNKvo+Q2kxn+WcLIL6jWaASsJH4cnnRl58IYcUv8cb6o6G34
+nTkWuhXm5XBUupacr8lqz5bXtoRBU7BcSnqXd20dTR5QEHU5Rrg/uw==
+-----END RSA PRIVATE KEY-----