From 7b4d5778c13818b871a5fe5aaf249480981ac15d Mon Sep 17 00:00:00 2001 From: Remigiusz Janeczek Date: Mon, 27 Jul 2020 13:42:09 +0200 Subject: Adjust DFC test files to use FTPES instead of FTPS DFC doesn't support FTPS protocol. FTPS is treated exactly like FTPES and soon DFC won't accept file locations starting with "ftps://". Fix mr-sim.py lint issues Issue-ID: DCAEGEN2-2367 Signed-off-by: Remigiusz Janeczek Change-Id: Idab56aab3bdb6734a04bfc9c9212f2ea800d0d74 --- .../ftpes-sftp-server/.gitignore | 1 + .../ftpes-sftp-server/Dockerfile-ftpes | 5 ++ .../ftpes-sftp-server/README.md | 37 ++++++++++++++ .../configuration/vsftpd_ssl.conf | 57 ++++++++++++++++++++++ .../ftpes-sftp-server/docker-compose.yml | 35 +++++++++++++ .../ftpes-sftp-server/prepare.sh | 50 +++++++++++++++++++ .../ftpes-sftp-server/test_cases.yml | 16 ++++++ .../ftpes-sftp-server/tls/README.md | 4 ++ .../ftpes-sftp-server/tls/dfc.crt | 19 ++++++++ .../ftpes-sftp-server/tls/ftp.crt | 19 ++++++++ .../ftpes-sftp-server/tls/ftp.key | 27 ++++++++++ 11 files changed, 270 insertions(+) create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/.gitignore create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/Dockerfile-ftpes create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/README.md create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/configuration/vsftpd_ssl.conf create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/docker-compose.yml create mode 100755 test/mocks/datafilecollector-testharness/ftpes-sftp-server/prepare.sh create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/test_cases.yml create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/README.md create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/dfc.crt create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.crt create mode 100644 test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.key (limited to 'test/mocks/datafilecollector-testharness/ftpes-sftp-server') diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/.gitignore b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/.gitignore new file mode 100644 index 000000000..bd6c5bed8 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/.gitignore @@ -0,0 +1 @@ +files/onap/* diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/Dockerfile-ftpes b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/Dockerfile-ftpes new file mode 100644 index 000000000..a0d6cfafc --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/Dockerfile-ftpes @@ -0,0 +1,5 @@ +FROM docker.io/panubo/vsftpd + +COPY configuration/vsftpd_ssl.conf /etc/vsftpd_ssl.conf +RUN chmod 644 /etc/vsftpd_ssl.conf +RUN chown root:root /etc/vsftpd_ssl.conf \ No newline at end of file diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/README.md b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/README.md new file mode 100644 index 000000000..44d329e76 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/README.md @@ -0,0 +1,37 @@ +# Deployment of certificates: (in case of update) + +This folder is prepared with a set of keys matching DfC for test purposes. + +Copy from datafile-app-server/config/keys to the ./tls/ the following files: + +- dfc.crt +- ftp.crt +- ftp.key + +# Docker preparations + +Source: + +`sudo usermod -aG docker $USER` + +then logout-login to activate it. + +# Prepare files for the simulator + +Run `prepare.sh` with an argument found in `test_cases.yml` (or add a new tc in that file) to create files (1MB, 5MB and 50MB files) and a large number of +symbolic links to these files to simulate PM files. The files names maches the files in +the events produced by the MR simulator. The dirs with the files will be mounted +by the ftp containers, defined in the docker-compse file, when started + +# Starting/stopping the FTPES/SFTP server(s) + +Start: `docker-compose up` + +Stop: Ctrl +C, then `docker-compose down` or `docker-compose down --remove-orphans` + +If you experience issues (or port collision), check the currently running other containers +by using 'docker ps' and stop them if necessary. + +# Cleaning docker structure + +Deep cleaning: `docker system prune` diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/configuration/vsftpd_ssl.conf b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/configuration/vsftpd_ssl.conf new file mode 100644 index 000000000..0a24e38a8 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/configuration/vsftpd_ssl.conf @@ -0,0 +1,57 @@ +# Server Config +anonymous_enable=NO +local_enable=YES +write_enable=YES +local_umask=022 +dirmessage_enable=YES + +# Security and User auth +chroot_local_user=YES +pam_service_name=vsftpd_virtual +virtual_use_local_privs=YES +chmod_enable=NO +user_config_dir=/etc/vsftpd/user_conf +user_sub_token=$USER +#local_root=/srv/$USER +local_root=/srv/ +userlist_enable=NO +allow_writeable_chroot=YES + +# Logging +log_ftp_protocol=YES +xferlog_enable=YES +xferlog_std_format=YES +#xferlog_file=/dev/stdout +syslog_enable=NO +dual_log_enable=YES + +# Remap all login users to this username +guest_enable=YES +guest_username=ftp +hide_ids=YES + +# Networking +connect_from_port_20=NO +listen=YES +tcp_wrappers=YES +#pasv_min_port=8001 +#pasv_max_port=8010 + +# SSL +ssl_enable=Yes +require_ssl_reuse=NO +force_local_data_ssl=YES +force_local_logins_ssl=YES +ssl_ciphers=HIGH +allow_anon_ssl=NO + +ssl_tlsv1=YES +ssl_sslv2=YES +ssl_sslv3=YES +rsa_cert_file=/etc/ssl/private/cert.pem +rsa_private_key_file=/etc/ssl/private/key.pem + +require_cert=YES +ssl_request_cert=YES +ca_certs_file=/etc/ssl/private/cacert.pem + diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/docker-compose.yml b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/docker-compose.yml new file mode 100644 index 000000000..5bc91efa1 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/docker-compose.yml @@ -0,0 +1,35 @@ +version: '3' + +services: + + sftp-server1: + container_name: sftp-server1 + image: atmoz/sftp:alpine + ports: + - "1022:22" + volumes: + - ./files/onap/sftp/:/home/onap/ + restart: on-failure + command: onap:pano:1001 + + ftpes-server-vsftpd: + container_name: ftpes-server-vsftpd + image: docker.io/panubo/vsftpd + ports: + - "21:21" + - "8001-8010:8001-8010" + environment: + FTP_USER: onap + FTP_PASSWORD: pano + PASV_ADDRESS: localhost + PASV_MIN_PORT: 8001 + PASV_MAX_PORT: 8010 + volumes: + - ./tls/ftp.crt:/etc/ssl/private/ftp.crt:ro + - ./tls/ftp.key:/etc/ssl/private/ftp.key:ro + - ./tls/dfc.crt:/etc/ssl/private/dfc.crt:ro + - ./configuration/vsftpd_ssl.conf:/etc/vsftpd_ssl.conf:ro + + - ./files/onap/ftpes/:/srv/ + restart: on-failure + command: vsftpd /etc/vsftpd_ssl.conf diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/prepare.sh b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/prepare.sh new file mode 100755 index 000000000..086d43a49 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/prepare.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# EXAMPLE: Run test case TC2 using the command "./prepare.sh TC2" +MAIN_DIRECTORY=./files/onap +TEST_FILE=./test_cases.yml +TEST=$1 +echo "Generating files for test case:" "$TEST" + +sf=$(sed -n '/'$TEST'/,$p' $TEST_FILE | grep -m 1 'size_files') +sf=${sf//*size_files: /} +sf_array=($sf) +echo "size_files=""$sf" + +nf=$(sed -n '/'$TEST'/,$p' $TEST_FILE | grep -m 1 'number_files') +nf=${nf//*number_files: /} +nf_array=($nf) +echo "number_files=""$nf" + +df=$(sed -n '/'$TEST'/,$p' $TEST_FILE | grep -m 1 'directory_files') +df=${df//*directory_files: /} +df_array=($df) +echo "directory_files=""$df" + +rm -rf $MAIN_DIRECTORY/* +if [ "${#sf_array[@]}" = "${#nf_array[@]}" ] && [ "${#nf_array[@]}" = "${#df_array[@]}" ]; +then + N_ELEMENTS=${#df_array[@]} + for ((n=0;n<$N_ELEMENTS;n++)) + do + # Create directory + DIRECTORY=$MAIN_DIRECTORY/${df_array[$n]} + mkdir -p "$DIRECTORY" + + # Create original file + FILE_SIZE=${sf_array[$n]} + FILE_NAME=$FILE_SIZE"MB.tar.gz" + dd if=/dev/urandom of=$DIRECTORY/$FILE_NAME bs=1k count=$(echo $FILE_SIZE*1000/1 | bc) + + # Create symlinks + N_SYMLINKS=${nf_array[$n]}-1 + for ((l=0;l<=$N_SYMLINKS;l++)) + do + SYMLINK_NAME=$FILE_SIZE"MB_"$l".tar.gz" + ln -s ./$FILE_NAME $DIRECTORY/$SYMLINK_NAME + done + done +else +echo "ERROR: The number of parameters in size_files, number_files, and directory_files must be equal!" +fi + +sudo chown root:root ./configuration/vsftpd_ssl.conf diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/test_cases.yml b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/test_cases.yml new file mode 100644 index 000000000..e07e3a0c6 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/test_cases.yml @@ -0,0 +1,16 @@ +# EXAMPLE: TC1 generates (i) 10 files of 1 MB in ftpes directory, +# (ii) 30 files of 5 MB in sftp directory, and (iii) 10 files of 10 MB in sftp directory +TC1: + size_files: 1 5 10 + number_files: 10 30 10 + directory_files: ftpes sftp sftp + +TC2: + size_files: 0.5 1 5 + number_files: 2 3 1 + directory_files: ftpes ftpes sftp + +TC_10000: + size_files: 1 1 5 5 50 50 + number_files: 10000 10000 10000 10000 1 1 + directory_files: ftpes sftp ftpes sftp ftpes sftp diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/README.md b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/README.md new file mode 100644 index 000000000..5edfeddec --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/README.md @@ -0,0 +1,4 @@ +# To verify the certificate expiration dates: + +openssl x509 -enddate -noout -in dfc.crt +openssl x509 -enddate -noout -in ftp.crt diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/dfc.crt b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/dfc.crt new file mode 100644 index 000000000..f747f20bb --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/dfc.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDFjCCAf4CCQCqH10CLXcbUDANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJT +RTEKMAgGA1UECAwBMDEKMAgGA1UEBwwBMDEKMAgGA1UECgwBMDEMMAoGA1UECwwD +RVNUMQwwCgYDVQQDDANFU1QwHhcNMTkwNDA0MDgwMjQwWhcNMjAwNDAzMDgwMjQw +WjBNMQswCQYDVQQGEwJTRTEKMAgGA1UECAwBMDEKMAgGA1UEBwwBMDEKMAgGA1UE +CgwBMDEMMAoGA1UECwwDRVNUMQwwCgYDVQQDDANFU1QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDAY7wM9khd7OIaZsfZv8EzWUnLTepzSx6hlAvjSbU/ +ZBpq94QOJWt22hQQSHA+Vdl8184PoyOKX+qRfbKJCtyArbv0DoWjlv16WNs938A/ +0TGFh+6xA464/GQmP/AXEuI0tSa2GEgXkhZ0uy6Pmdq+8sD6YcRyVCeqLTq8bqSq +YbpPrKnbZsd3l7PzpYCZgZLfWoENQ7nuT+C7j4pGGnPKpGn4ubiscV3nTI6nwU19 +ANexToikIL1v18z+gJdva8QtKih5dt9+2QJuJ6lPcwTa7xVkMmuVyr5FXwL11yII +gKRtknWObU7BjcKkQOG3xnWXZTNzLV/th8GCCWJuRFPbAgMBAAEwDQYJKoZIhvcN +AQELBQADggEBAGZa23j04vghZY+/81FyRsoeJs8o9aDeK5KLT9JUpRaZlu4AEXSN +I3oEhencocc07DTndfxw7eSALnTD9ibGUxnRyEvgtW595ajb7A3TtEKGsFzXFvar +y9RdVm6RRA4Bty8UyLW75ZpAlfc3twCmsRwKjShXeN90Yn3sjpcEU83q8fZBU35C +xrgrW34gphPtuzZ7kvCJGjhYIfIsTcWObvDGUROtlRhb8w2v1K4G3UcfwDTBixK/ +e61mMUbhoqVyoMYgSuU4idY+n4Evjs5IwalXM1daUZmCGGZoreD/9aSpDEeUWnQy +vqYC0YY0VJkOe+WI6JZ2r49BtRtl/jxWDUI= +-----END CERTIFICATE----- diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.crt b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.crt new file mode 100644 index 000000000..f412d013c --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDFjCCAf4CCQDaiGveWOXqNjANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJT +RTEKMAgGA1UECAwBMDEKMAgGA1UEBwwBMDEKMAgGA1UECgwBMDEMMAoGA1UECwwD +RVNUMQwwCgYDVQQDDANFU1QwHhcNMTkwNDA0MDgwMjQwWhcNMjAwNDAzMDgwMjQw +WjBNMQswCQYDVQQGEwJTRTEKMAgGA1UECAwBMDEKMAgGA1UEBwwBMDEKMAgGA1UE +CgwBMDEMMAoGA1UECwwDRVNUMQwwCgYDVQQDDANFU1QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDSsF7lN/gXpevQpGj8W/3g3h9AVOE83Z49yJAdyIHF +PQz6PI+bKutYdORCUZkzsl2fegLzkXl4CmoVIkJRBL1SZkzQXKe+fjfuRr9PQKCC +lp/LA161Qak+9pz2Oc1lfLbgEdv22RLji3akCQso3G9vlx+rLHPRgbew0iiTViJP +v3CHwiY89t1ai149OEywhjsJBJjBoj6fvxfvv46QmK7FuV5Tz0vTL/eB/Z9P7jm+ +twHRz9Ae4s97c6UhbFKafHLrwdMK+yz7qe55tpgthCgoedeSB0gXWIiS7RY18dEN +JUB/FAt64LfOiKBl0aUbcQOgUinorhCN8gcNTn7Hrn+1AgMBAAEwDQYJKoZIhvcN +AQELBQADggEBAKVGHkTLe5R/fG/C7prxiknD+QXo9WACcZNVKMuKhpJwQh1iwc4h +4tq9lj//giyRrt+yPVQF8pRTiIdeewLVyf5O1ugxzb68UtHzVJWD6ooFqGmyPFkm +WOdLvtgbasGPZvO6y8HZA3kxKgONbYcL0sdtRWpp5b+KTEyvN/50jAtvkB/wlaee +emgdRdsSVZqg1p8dUfF6j3Alzsuff7YzEZEZPoJKYdb1vikvj21+LdzTDSj5WRno +PWXQhdTTqN5/TNMZRHJp/UZY6hVmQL+ILqVYGiOPotfxGNUyo+WsKJwZfZnq4adh +BzdSIIsDCZB34Njz/qjIXh307/seNWWhNFw= +-----END CERTIFICATE----- diff --git a/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.key b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.key new file mode 100644 index 000000000..f90c781d3 --- /dev/null +++ b/test/mocks/datafilecollector-testharness/ftpes-sftp-server/tls/ftp.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA0rBe5Tf4F6Xr0KRo/Fv94N4fQFThPN2ePciQHciBxT0M+jyP +myrrWHTkQlGZM7Jdn3oC85F5eApqFSJCUQS9UmZM0Fynvn437ka/T0CggpafywNe +tUGpPvac9jnNZXy24BHb9tkS44t2pAkLKNxvb5cfqyxz0YG3sNIok1YiT79wh8Im +PPbdWotePThMsIY7CQSYwaI+n78X77+OkJiuxbleU89L0y/3gf2fT+45vrcB0c/Q +HuLPe3OlIWxSmnxy68HTCvss+6nuebaYLYQoKHnXkgdIF1iIku0WNfHRDSVAfxQL +euC3zoigZdGlG3EDoFIp6K4QjfIHDU5+x65/tQIDAQABAoIBAEs+G5XG6D4hzlbD +8I53l/JvwT9rUMk46GNuNjG8wsOa6wCPEkY7DLOZg08/7vOsdo0WuOkdggDhz0Le +6koe5DICQNqEzI9WakkZUQdPsEMS4dxRxsf6vCO1VRcGS5k78d+R4TmGCp3i16r7 +Y9Xi65UxpmiuRmqC5gQq+bysnTJXKUhK/3NCPa7Bwo7hgASJcI55Nk58KHokIv84 +7dweKuIIxeKAR4/094q243lTu1n273J+ckjVMWWZROIIn2E+CrjAVh59DaC7QX6d +kWdPwvjYiwH2LBqzJ3dKvLmk6XZZ5bcjmBYXSiS1ahjQ8zhDdLoHBt/gDXcmtOVw +gZPjgn0CgYEA6nJGMig2SRB25CqtA/gwLbQV54th0Vxj1SapoO+W4jIUEPBRJN1t +0JkM9mTMWWBiEzZi4ICpJpgT/+iGXx6q6WZZqOvbWWS4yjuC+wLCttt2yriFkmlA +eylz0rYTGm5gQ3wpAUuf0wOfqVICfQ2KnASY0p8g2fSjOI0/pULpX18CgYEA5g72 +UVspmBLqaA/PGYip/7neh00sGs95b9Wh1UqvcrstxkgR4LyMOReTBNwL+39kyZPv +LNsfMtaiezIfSF+o77RiXnBsYQ/ZnKBNZ7cTRr76f5fKStzRSzHoQrf+98YkSfg3 +sI1vYH/hnfbd/6ti8Wiloc7O28IHCwG4vGXWPWsCgYEA4Oqaj1nmonfKJ6ENdSZw +Shhff1BHmovxNrctuzi/Xue+OnXi0uQfiySZu/P926zMyjO97jVgkacKYNMZvj10 +qA/J6nXDbOJlKZaoVNlUJgsrztXxRwH0m3OsKzUD5LKJZZTC3fxIKy9pyA3mV0Rb +eswqNL32zUKWKBXSPmCP9S8CgYEAoJIk3dfRCjF5pjQSinHWpYRniuwUMopI6gOj +tqkwF9YJNvGqeL9g56BKnwOu4VSjVgaI0zgFaIhiU7ZCG1OZJ+UxG1VAb+aOLkG8 +hP1E2QYG9PNO4D2LXv5wa9DchrenMHGs/O9ao3HeWVCq1f4bSv+NS4h63Otp2wbS +l7xEcg0CgYBcrapVzHfGn73Z9tgs7GOVEmoXKoT6Q8yrdwwhVrcWdDHHtLFPso21 +eA9dhbbawaEmIReWtNKvo+Q2kxn+WcLIL6jWaASsJH4cnnRl58IYcUv8cb6o6G34 +nTkWuhXm5XBUupacr8lqz5bXtoRBU7BcSnqXd20dTR5QEHU5Rrg/uw== +-----END RSA PRIVATE KEY----- -- cgit 1.2.3-korg