Add sanitycheck for ves client

Issue-ID: INT-1869
Signed-off-by: Bogumil Zebek <bogumil.zebek@nokia.com>
Change-Id: Ib297493ad9386594b4451f05378e070747187428

18 files changed, 741 insertions, 0 deletions

diff --git a/sanitycheck/vesclient-secured/README.md b/sanitycheck/vesclient-secured/README.md
new file mode 100644
index 0000000..f791afb
--- /dev/null
+++ b/sanitycheck/vesclient-secured/README.md
@@ -0,0 +1,11 @@
+Standalone VES client configuration for HTTPS communication with VES
+------------------------
+
+This directory contains files for secured VES client deployments, which will use certificates for HTTPS communication with VES.  
+
+Currently, there are two ways for VES client to fetch certificates:
+* Using AAF Certman 
+* Using OOM CertService (CMPv2) 
+    
+Both ways are described in `certman` and `certservice` directories respectively
+
diff --git a/sanitycheck/vesclient-secured/certman/Makefile b/sanitycheck/vesclient-secured/certman/Makefile
new file mode 100644
index 0000000..d75b5d0
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certman/Makefile
@@ -0,0 +1,8 @@
+default:
+	@echo "There is no default target. Use: make <specific_target>"
+
+start-ves-client:
+	docker-compose -f docker-compose.yml up
+
+clean-ves-client:
+	docker-compose -f docker-compose.yml down
diff --git a/sanitycheck/vesclient-secured/certman/README.md b/sanitycheck/vesclient-secured/certman/README.md
new file mode 100644
index 0000000..92985f8
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certman/README.md
@@ -0,0 +1,91 @@
+## Fetching from AAF Certman
+This readme describes how to run VES client with certificates fetched using AAF Certman
+
+### Description
+
+docker-compose.yml prepares VES client container for HTTPS communication with VES.
+
+When docker-compose starts certs-init container fills connected volume with certificates, truststores, keystores, 
+passwords etc. Next ves-client container starts and connects to the same volume. On startup it should read password
+values from proper files and set them in system environment variables. With these variables and files in volume 
+application is ready to work on HTTPS.
+
+### Prerequisites
+
+certs-init container works with external AAF on cloud. Due to that fact it must have set correct IPs to workers that
+has access to AAF. In docker-compose.yml fields with mentioned IPs are:
+    
+    * aaf-locate.onap
+    * aaf-cm.onap
+    * aaf-service.onap
+
+### Start
+
+Run VES client:
+
+```
+make start-ves-client
+```
+
+### Send event
+
+**ATTENTION**
+
+``sanitycheck/events/eventToVes.json`` file which is request for sending event to VES must have correct ``vesServerURL`` 
+field before sending event. 
+IP of ``vesServerURL`` should be the same as given in docker-compose-certman.yml in ``aaf-locate.onap`` field.
+To use secured connection remember about setting protocol to https:// and port to proper secured port of VES.
+
+To send event from VES client to VES use this command from ``ne-simulator/sanitycheck`` directory:
+
+````
+make generate-event
+````
+
+Sample ``sanitycheck/events/eventToVes.json`` file content is:
+
+```json
+{
+  "vesServerUrl": "https://10.183.35.177:30417/eventListener/v7",
+  "event": {
+    "event": {
+      "commonEventHeader": {
+        "version": "4.0.1",
+        "vesEventListenerVersion": "7.0.1",
+        "domain": "fault",
+        "eventName": "Fault_Vscf:Acs-Ericcson_PilotNumberPoolExhaustion",
+        "eventId": "fault0000245",
+        "sequence": 1,
+        "priority": "High",
+        "reportingEntityId": "cc305d54-75b4-431b-adb2-eb6b9e541234",
+        "reportingEntityName": "ibcx0001vm002oam001",
+        "sourceId": "de305d54-75b4-431b-adb2-eb6b9e546014",
+        "sourceName": "scfx0001vm002cap001",
+        "nfVendorName": "Ericsson",
+        "nfNamingCode": "scfx",
+        "nfcNamingCode": "ssc",
+        "startEpochMicrosec": 1413378172000000,
+        "lastEpochMicrosec": 1413378172000000,
+        "timeZoneOffset": "UTC-05:30"
+      },
+      "faultFields": {
+        "faultFieldsVersion": "4.0",
+        "alarmCondition": "PilotNumberPoolExhaustion",
+        "eventSourceType": "other",
+        "specificProblem": "Calls cannot complete - pilot numbers are unavailable",
+        "eventSeverity": "CRITICAL",
+        "vfStatus": "Active",
+        "alarmAdditionalInformation": {
+          "PilotNumberPoolSize": "1000"
+        }
+      }
+    }
+  }
+}
+```
+
+### Stop
+To remove VES client containers use:
+```
+make clean-ves-client
+```
diff --git a/sanitycheck/vesclient-secured/certman/docker-compose.yml b/sanitycheck/vesclient-secured/certman/docker-compose.yml
new file mode 100644
index 0000000..2714751
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certman/docker-compose.yml
@@ -0,0 +1,69 @@
+version: '3'
+
+networks:
+  tls-init-network:
+
+volumes:
+  certs-volume:
+
+services:
+  certs-init:
+    image: nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.tls-init-container:2.1.0
+    extra_hosts:
+      #set worker IP with access to AAF
+      aaf-locate.onap: <WORKER_IP> #for example 10.183.35.177
+      aaf-cm.onap: <WORKER_IP> #for example 10.183.35.177
+      aaf-service.onap: <WORKER_IP> #for example 10.183.35.177
+    environment:
+      - aaf_locate_url=https://aaf-locate.onap:31111
+      - aaf_url_cm=https://aaf-cm.onap:31114
+      - aaf_url=https://aaf-service.onap:31110
+    networks:
+      - tls-init-network
+    volumes:
+      - certs-volume:/opt/app/osaaf
+  mongo:
+    image: mongo
+    restart: always
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: zXcVbN123!
+      MONGO_INITDB_DATABASE: pnf_simulator
+    networks:
+      - tls-init-network
+    volumes:
+      - ../../../../ves-client/db:/docker-entrypoint-initdb.d
+    ports:
+      - "27017:27017"
+
+  mongo-express:
+    image: mongo-express
+    restart: always
+    ports:
+      - 8081:8081
+    networks:
+      - tls-init-network
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: root
+      ME_CONFIG_MONGODB_ADMINPASSWORD: zXcVbN123!
+
+  ves-client:
+    image: onap/org.onap.integration.nfsimulator.vesclient
+    ports:
+      - "5000:5000"
+    command: bash -c "
+      while [[ $$(ls -1 /app/store | wc -l) != '10' ]]; do echo 'Waiting for certs...'; sleep 3; done
+      && java -Dspring.config.location=file:/app/application.properties -cp /app/libs/*:/app/vesclient.jar org.onap.integration.simulators.nfsimulator.vesclient.Main
+      "
+    volumes:
+      - ../../../../ves-client/logs:/var/log
+      - ../../../../ves-client/templates:/app/templates
+      - ../../../../ves-client/src/main/resources/application.properties:/app/application.properties
+      - certs-volume:/app/store
+    networks:
+      - tls-init-network
+    restart: on-failure
+    depends_on:
+      - certs-init
+      - mongo
+      - mongo-express
diff --git a/sanitycheck/vesclient-secured/certservice/Makefile b/sanitycheck/vesclient-secured/certservice/Makefile
new file mode 100644
index 0000000..0f41b0e
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/Makefile
@@ -0,0 +1,59 @@
+default:
+	@echo "There is no default target. Use: make <specific_target>"
+
+setup-env: --start-certservice-and-ejbca --run-certservice-clients --start-local-secured-ves
+
+start-ves-client:
+	docker-compose -f docker-compose-vesclient.yml up
+
+restart-ves-client: --clean-ves-client start-ves-client
+
+clean-all: --clean-ves-client --clean-env
+
+
+--start-certservice-and-ejbca: --create-certservice-internal-certs --start-certservice-ejbca-containers --configure-ejbca
+
+--start-certservice-ejbca-containers:
+	docker-compose -f docker-compose-certservice-ejbca.yml up -d
+
+--create-certservice-internal-certs:
+	make -C resources/certs all
+
+--configure-ejbca: --wait-for-ejbca --run-ejbca-script
+
+--wait-for-ejbca:
+	@echo 'Waiting for EJBCA... It may take a minute or two'
+	until docker container inspect oomcert-ejbca | grep '"Status": "healthy"'; do sleep 3; done
+
+--run-ejbca-script:
+	docker exec oomcert-ejbca /opt/primekey/scripts/ejbca-configuration.sh
+
+--run-certservice-clients: --create-client-volumes
+	docker-compose -f docker-compose-certservice-clients.yml up -d
+	@echo 'Waiting for client certifiactes...'
+	@until ls -1 ./resources/certservice-client/client-volume-for-vesclient | grep "store" 1>/dev/null; do sleep 3; done
+	@until ls -1 ./resources/certservice-client/client-volume-for-ves | grep "store" 1>/dev/null; do sleep 3; done
+	@until ls -1 ./resources/certservice-client/client-volume-for-httpserver | grep "store" 1>/dev/null; do sleep 3; done
+
+--create-client-volumes:
+	mkdir -p ./resources/certservice-client/client-volume-for-vesclient  -m 777
+	mkdir -p ./resources/certservice-client/client-volume-for-ves -m 777
+	mkdir -p ./resources/certservice-client/client-volume-for-httpserver -m 777
+
+--start-local-secured-ves:
+	docker-compose -f docker-compose-ves-dmaap.yml up
+
+--clean-ves-client:
+	docker-compose -f docker-compose-vesclient.yml down
+	rm -rf ./resources/certservice-client/client-volume-for-vesclient || true
+	rm -rf ./resources/certservice-client/client-volume-for-httpserver || true
+
+
+--clean-env:
+	docker-compose -f docker-compose-ves-dmaap.yml down
+	docker-compose -f docker-compose-certservice-clients.yml down
+	rm -rf ./resources/certservice-client/client-volume-for-vesclient || true
+	rm -rf ./resources/certservice-client/client-volume-for-ves || true
+	rm -rf ./resources/certservice-client/client-volume-for-httpserver || true
+	docker-compose -f docker-compose-certservice-ejbca.yml down
+	make -C resources/certs clear
diff --git a/sanitycheck/vesclient-secured/certservice/README.md b/sanitycheck/vesclient-secured/certservice/README.md
new file mode 100644
index 0000000..27f68ef
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/README.md
@@ -0,0 +1,85 @@
+## Fetching certificates from OOM CertService (CMPv2)
+This readme describes how to run VES client with certificates fetched using OOM CertService (CMPv2) 
+
+### Description
+
+Using Makefile in this directory following can be achieved:
+
+* Setup environment for VES client, i.e.:
+    * Create certificates that will be used for internal communication between CertService and CertService Clients.
+      Generated internal certificates should be present in `resources/certs` directory.
+    * Start and configure EJBCA
+    * Start and configure AAF Cert Service.
+    * Run Cert Service Clients to fetch certificates for VES and VES client. Certificates will be stored for the
+      components in `resources/certservice-client/client-volume-for-ves`
+      and `resources/certservice-client/client-volume-for-vesclient` accordingly.
+    * Start VES and DMaaP Simulator. Fetched certificates will be mounted to VES.
+
+* Start VES client. Fetched certificates will be mounted to VES client.
+* Clean up.
+
+### Prerequisites
+##### VES collector local deployment prerequisites
+
+By default, the image of VES from Nexus supports only HTTP communication. A local image with enabled HTTPS must be build
+to use local VES as VES client destination.
+
+1. Pull VES repository
+2. In `<VES_PROJECT_ROOT>/etc/collector.properties` file set field `auth.method=certBasicAuth`
+3. Build a local image: `mvn clean install docker:build` from VES project root directory.
+
+Local VES deployment uses also DMaaP simulator. Its image should be built locally as well.
+1. Go to `sanitycheck/dmaap-simulator` directory
+2. Run: `make build`
+
+### Setup environment
+To set up whole environment for VES client, i.e.:
+- deploy and configure EJBCA
+- deploy Cert Service
+- fetch certificates for VES and VES client using Cert Service clients
+- run DMaaP Simulator
+- run VES with fetched certificates
+
+execute:
+````
+make setup-env
+````
+Note that this command setups whole environment besides VES client itself. 
+
+## Run VES client
+To run VES client execute:
+````
+make start-ves-client
+````
+VES client starts together with the http server.
+This command starts VES client with certificates fetched using CertService (certificates are fetched in the previous
+step)
+
+### Send event
+
+
+Configure VES client to use proper VES URL by executing this command from ``nf-simulator/sanitycheck`` directory:
+
+  TIP: edit vesAddressConfigure.json and set "vesServerUrl": "https://172.17.0.1:8443/eventListener/v7"
+
+```
+make reconfigure-ves-url
+```
+
+Send an event from VES client to VES by executing this command from ``nf-simulator/sanitycheck`` directory:
+```
+make generate-event
+```
+
+### Restart VES client
+
+To restart only VES client execute:
+```
+make restart-ves-client
+```
+
+### Clean up
+To clean all generated certificates, remove VES client, CertService, EJBCA, VES and DMaaP Simulator containers:
+```
+make clean-all
+```
diff --git a/sanitycheck/vesclient-secured/certservice/docker-compose-certservice-clients.yml b/sanitycheck/vesclient-secured/certservice/docker-compose-certservice-clients.yml
new file mode 100644
index 0000000..d721561
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/docker-compose-certservice-clients.yml
@@ -0,0 +1,39 @@
+version: "2.1"
+
+networks:
+  onap:
+    external: true
+
+services:
+  oom-cert-client-ves:
+    image: nexus3.onap.org:10001/onap/org.onap.oom.platform.cert-service.oom-certservice-client:2.1.1
+    container_name: oomcert-client-for-ves
+    env_file: ./resources/certservice-client/client-configuration-for-ves.env
+    networks:
+      - onap
+    volumes:
+      - ./resources/certservice-client/client-volume-for-ves:/var/certs:rw
+      - ./resources/certs/truststore.jks:/etc/onap/oom/certservice/certs/truststore.jks
+      - ./resources/certs/certServiceClient-keystore.jks:/etc/onap/oom/certservice/certs/certServiceClient-keystore.jks
+
+  oom-cert-client-vesclient:
+    image: nexus3.onap.org:10001/onap/org.onap.oom.platform.cert-service.oom-certservice-client:2.1.1
+    container_name: oomcert-client
+    env_file: ./resources/certservice-client/client-configuration-for-vesclient.env
+    networks:
+      - onap
+    volumes:
+      - ./resources/certservice-client/client-volume-for-vesclient:/var/certs:rw
+      - ./resources/certs/truststore.jks:/etc/onap/oom/certservice/certs/truststore.jks
+      - ./resources/certs/certServiceClient-keystore.jks:/etc/onap/oom/certservice/certs/certServiceClient-keystore.jks
+
+  oom-cert-client-httpserver:
+    image: nexus3.onap.org:10001/onap/org.onap.oom.platform.cert-service.oom-certservice-client:2.1.1
+    container_name: oomcert-client-for-httpserver
+    env_file: ./resources/certservice-client/client-configuration-for-httpserver.env
+    networks:
+      - onap
+    volumes:
+      - ./resources/certservice-client/client-volume-for-httpserver:/var/certs:rw
+      - ./resources/certs/truststore.jks:/etc/onap/oom/certservice/certs/truststore.jks
+      - ./resources/certs/certServiceClient-keystore.jks:/etc/onap/oom/certservice/certs/certServiceClient-keystore.jks
diff --git a/sanitycheck/vesclient-secured/certservice/docker-compose-certservice-ejbca.yml b/sanitycheck/vesclient-secured/certservice/docker-compose-certservice-ejbca.yml
new file mode 100644
index 0000000..a400eb9
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/docker-compose-certservice-ejbca.yml
@@ -0,0 +1,47 @@
+version: "2.1"
+
+networks:
+  onap:
+    driver: bridge
+    name: onap
+  public:
+    driver: bridge
+    name: public
+
+services:
+  ejbca:
+    image: primekey/ejbca-ce:6.15.2.5
+    hostname: cahostname
+    container_name: oomcert-ejbca
+    ports:
+      - "80:8080"
+      - "443:8443"
+    volumes:
+      - ./resources/ejbca/ejbca-configuration.sh:/opt/primekey/scripts/ejbca-configuration.sh
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -kI https://localhost:8443/ejbca/publicweb/healthcheck/ejbcahealth" ]
+      interval: 10s
+      timeout: 3s
+      retries: 15
+    networks:
+      - onap
+
+  oom-cert-service:
+    image: nexus3.onap.org:10001/onap/org.onap.oom.platform.cert-service.oom-certservice-api:2.1.1
+    volumes:
+      - ./resources/certservice/cmpServers.json:/etc/onap/oom/certservice/cmpServers.json
+      - ./resources/certs/truststore.jks:/etc/onap/oom/certservice/certs/truststore.jks
+      - ./resources/certs/root.crt:/etc/onap/oom/certservice/certs/root.crt
+      - ./resources/certs/certServiceServer-keystore.jks:/etc/onap/oom/certservice/certs/certServiceServer-keystore.jks
+      - ./resources/certs/certServiceServer-keystore.p12:/etc/onap/oom/certservice/certs/certServiceServer-keystore.p12
+    container_name: oomcert-service
+    ports:
+      - "8443:8443"
+    healthcheck:
+      test: ["CMD-SHELL", "curl https://localhost:8443/actuator/health --cacert /etc/onap/oom/certservice/certs/root.crt --cert-type p12 --cert /etc/onap/oom/certservice/certs/certServiceServer-keystore.p12 --pass secret"]
+      interval: 10s
+      timeout: 3s
+      retries: 15
+    networks:
+      - onap
+      - public
diff --git a/sanitycheck/vesclient-secured/certservice/docker-compose-ves-dmaap.yml b/sanitycheck/vesclient-secured/certservice/docker-compose-ves-dmaap.yml
new file mode 100644
index 0000000..86f0202
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/docker-compose-ves-dmaap.yml
@@ -0,0 +1,33 @@
+version: "2.1"
+
+networks:
+  public:
+    external: true
+  onap:
+    external: true
+
+services:
+  ves:
+    container_name: ves
+    image: nexus3.onap.org:10003/onap/org.onap.dcaegen2.collectors.ves.vescollector:latest
+    ports:
+      - "8082:8080"
+      - "8444:8443"
+    networks:
+      - onap
+      - public
+    volumes:
+      - ./resources/certservice-client/client-volume-for-ves/keystore.jks:/opt/app/VESCollector/etc/keystore
+      - ./resources/certservice-client/client-volume-for-ves/keystore.pass:/opt/app/VESCollector/etc/passwordfile
+      - ./resources/certservice-client/client-volume-for-ves/truststore.jks:/opt/app/VESCollector/etc/truststore
+      - ./resources/certservice-client/client-volume-for-ves/truststore.pass:/opt/app/VESCollector/etc/trustpasswordfile
+    depends_on:
+      - onap-dmaap
+
+  onap-dmaap:
+    container_name: dmaap
+    image: dmaap-simulator
+    ports:
+      - "3904:3904"
+    networks:
+      - onap
diff --git a/sanitycheck/vesclient-secured/certservice/docker-compose-vesclient.yml b/sanitycheck/vesclient-secured/certservice/docker-compose-vesclient.yml
new file mode 100644
index 0000000..f99330b
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/docker-compose-vesclient.yml
@@ -0,0 +1,86 @@
+version: "2.1"
+
+networks:
+  ves-client:
+    driver: bridge
+    name: ves-client
+  public:
+    external: true
+  onap:
+    external: true
+
+services:
+  mongo:
+    image: mongo
+    restart: always
+    networks:
+      - ves-client
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: zXcVbN123!
+      MONGO_INITDB_DATABASE: pnf_simulator
+    volumes:
+      - ../../../../ves-client/db:/docker-entrypoint-initdb.d
+    ports:
+      - "27017:27017"
+
+  mongo-express:
+    image: mongo-express
+    restart: always
+    networks:
+      - ves-client
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: root
+      ME_CONFIG_MONGODB_ADMINPASSWORD: zXcVbN123!
+
+  http-server:
+    image: nexus3.onap.org:10003/onap/org.onap.integration.nfsimulator.pmhttpsserver
+    ports:
+      - "8080:8080"
+      - "32000:32000"
+      - "32080:80"
+      - "32100:32100"
+      - "32443:443"
+    networks:
+      - ves-client
+      - public
+    volumes:
+      - ~/httpservervolumes/:/usr/local/apache2/htdocs
+      - ../../../httpserver/logs:/var/log/apache2
+      - ./resources/certservice-client/client-volume-for-httpserver/:/etc/apache2/certs/
+    command: bash -c "
+      echo 'Http Server start';
+      while [[ $$(ls -1 /etc/apache2/certs/ | wc -l) != '3' ]]; do echo 'Waiting for certs...'; sleep 3; done;
+      chmod 777 /usr/local/apache2/htdocs;
+      cp /usr/local/apache2/conf/upload.php /usr/local/apache2/htdocs/upload.php;
+      touch /usr/local/apache2/htdocs/index.html;
+      /usr/sbin/apache2ctl -D FOREGROUND;
+      "
+    restart: on-failure
+
+  ves-client:
+    image: onap/org.onap.integration.nfsimulator.vesclient
+    ports:
+      - "5000:5000"
+    networks:
+      - ves-client
+      - public
+    command: bash -c "
+      while [[ $$(ls -1 /app/store | wc -l) != '4' ]]; do echo 'Waiting for certs...'; sleep 3; done
+      && cp /app/store/truststore.p12 /app/store/trust.jks
+      && cp /app/store/keystore.p12 /app/store/cert.p12
+      && cp /app/store/keystore.pass /app/store/p12.pass
+      && cp /app/store/truststore.pass /app/store/trust.pass
+      && java -Dspring.config.location=file:/app/application.properties  -cp /app/libs/*:/app/vesclient.jar org.onap.integration.simulators.nfsimulator.vesclient.Main
+      "
+    volumes:
+      - ../../../../ves-client/logs:/var/log
+      - ../../../../ves-client/templates:/app/templates
+      - ../../../../ves-client/src/main/resources/application.properties:/app/application.properties
+      - ./resources/certservice-client/client-volume-for-vesclient/:/app/store/
+    restart: on-failure
+    depends_on:
+      - mongo
+      - mongo-express
diff --git a/sanitycheck/vesclient-secured/certservice/resources/certs/.gitignore b/sanitycheck/vesclient-secured/certservice/resources/certs/.gitignore
new file mode 100644
index 0000000..385dcde
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/resources/certs/.gitignore
@@ -0,0 +1,3 @@
+*.jks
+*.p12
+*.crt
diff --git a/sanitycheck/vesclient-secured/certservice/resources/certs/Makefile b/sanitycheck/vesclient-secured/certservice/resources/certs/Makefile
new file mode 100644
index 0000000..507a23c
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/resources/certs/Makefile
@@ -0,0 +1,109 @@
+all: clear step_1 step_2 step_3 step_4 step_5 step_6 step_7 step_8 step_9 step_10 step_11 step_12 step_13 step_14 step_15
+.PHONY: all
+#Clear certificates
+clear:
+	@echo "Clear certificates"
+	rm -f certServiceClient-keystore.jks certServiceServer-keystore.jks root.crt truststore.jks certServiceServer-keystore.p12 root-keystore.jks
+	@echo "#####done#####"
+
+#Generate root private and public keys
+step_1:
+	@echo "Generate root private and public keys"
+	keytool -genkeypair -v -alias root -keyalg RSA -keysize 4096 -validity 3650 -keystore root-keystore.jks \
+    -dname "CN=root.com, OU=Root Org, O=Root Company, L=Wroclaw, ST=Dolny Slask, C=PL" -keypass secret \
+    -storepass secret -ext BasicConstraints:critical="ca:true"
+	@echo "#####done#####"
+
+#Export public key as certificate
+step_2:
+	@echo "(Export public key as certificate)"
+	keytool -exportcert -alias root -keystore root-keystore.jks -storepass secret -file root.crt -rfc
+	@echo "#####done#####"
+
+#Self-signed root (import root certificate into truststore)
+step_3:
+	@echo "(Self-signed root (import root certificate into truststore))"
+	keytool -importcert -alias root -keystore truststore.jks -file root.crt -storepass secret -noprompt
+	@echo "#####done#####"
+
+#Generate certService's client private and public keys
+step_4:
+	@echo "Generate certService's client private and public keys"
+	keytool -genkeypair -v -alias certServiceClient -keyalg RSA -keysize 2048 -validity 730 \
+    -keystore certServiceClient-keystore.jks -storetype JKS \
+    -dname "CN=certServiceClient.com,OU=certServiceClient company,O=certServiceClient org,L=Wroclaw,ST=Dolny Slask,C=PL" \
+    -keypass secret -storepass secret
+	@echo "####done####"
+
+#Generate certificate signing request for certService's client
+step_5:
+	@echo "Generate certificate signing request for certService's client"
+	keytool -certreq -keystore certServiceClient-keystore.jks -alias certServiceClient -storepass secret -file certServiceClient.csr
+	@echo "####done####"
+
+#Sign certService's client certificate by root CA
+step_6:
+	@echo "Sign certService's client certificate by root CA"
+	keytool -gencert -v -keystore root-keystore.jks -storepass secret -alias root -infile certServiceClient.csr \
+    -outfile certServiceClientByRoot.crt -rfc -ext bc=0  -ext ExtendedkeyUsage="serverAuth,clientAuth"
+	@echo "####done####"
+
+#Import root certificate into client
+step_7:
+	@echo "Import root certificate into intermediate"
+	cat root.crt >> certServiceClientByRoot.crt
+	@echo "####done####"
+
+#Import signed certificate into certService's client
+step_8:
+	@echo "Import signed certificate into certService's client"
+	keytool -importcert -file certServiceClientByRoot.crt -destkeystore certServiceClient-keystore.jks -alias certServiceClient -storepass secret -noprompt
+	@echo "####done####"
+
+#Generate certService private and public keys
+step_9:
+	@echo "Generate certService private and public keys"
+	keytool -genkeypair -v -alias oom-cert-service -keyalg RSA -keysize 2048 -validity 730 \
+    -keystore certServiceServer-keystore.jks -storetype JKS \
+    -dname "CN=oom-cert-service,OU=certServiceServer company,O=certServiceServer org,L=Wroclaw,ST=Dolny Slask,C=PL" \
+    -keypass secret -storepass secret -ext BasicConstraints:critical="ca:false"
+	@echo "####done####"
+
+#Generate certificate signing request for certService
+step_10:
+	@echo "Generate certificate signing request for certService"
+	keytool -certreq -keystore certServiceServer-keystore.jks -alias oom-cert-service -storepass secret -file certServiceServer.csr
+	@echo "####done####"
+
+#Sign certService certificate by root CA
+step_11:
+	@echo "Sign certService certificate by root CA"
+	keytool -gencert -v -keystore root-keystore.jks -storepass secret -alias root -infile certServiceServer.csr \
+    -outfile certServiceServerByRoot.crt -rfc -ext bc=0  -ext ExtendedkeyUsage="serverAuth,clientAuth" \
+    -ext SubjectAlternativeName:="DNS:oom-cert-service,DNS:localhost"
+	@echo "####done####"
+
+#Import root certificate into server
+step_12:
+	@echo "Import root certificate into intermediate(server)"
+	cat root.crt >> certServiceServerByRoot.crt
+	@echo "####done####"
+
+#Import signed certificate into certService
+step_13:
+	@echo "Import signed certificate into certService"
+	keytool -importcert -file certServiceServerByRoot.crt -destkeystore certServiceServer-keystore.jks -alias oom-cert-service \
+    -storepass secret -noprompt
+	@echo "####done####"
+
+#Convert certServiceServer-keystore(.jks) to PCKS12 format(.p12)
+step_14:
+	@echo "Convert certServiceServer-keystore(.jks) to PCKS12 format(.p12)"
+	keytool -importkeystore -srckeystore certServiceServer-keystore.jks -srcstorepass secret -destkeystore certServiceServer-keystore.p12 -deststoretype PKCS12 -deststorepass secret
+	@echo "#####done#####"
+
+#Clear unused certificates
+step_15:
+	@echo "Clear unused certificates"
+	rm certServiceClientByRoot.crt certServiceClient.csr root-keystore.jks certServiceServerByRoot.crt  certServiceServer.csr
+	@echo "#####done#####"
diff --git a/sanitycheck/vesclient-secured/certservice/resources/certservice-client/.gitignore b/sanitycheck/vesclient-secured/certservice/resources/certservice-client/.gitignore
new file mode 100644
index 0000000..ef10692
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/resources/certservice-client/.gitignore
@@ -0,0 +1,3 @@
+client-volume-for-httpserver
+client-volume-for-vesclient
+client-volume-for-ves
diff --git a/sanitycheck/vesclient-secured/certservice/resources/certservice-client/client-configuration-for-httpserver.env b/sanitycheck/vesclient-secured/certservice/resources/certservice-client/client-configuration-for-httpserver.env
new file mode 100644
index 0000000..8e8eb34
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/resources/certservice-client/client-configuration-for-httpserver.env
@@ -0,0 +1,18 @@
+#Client envs
+REQUEST_URL=https://oom-cert-service:8443/v1/certificate/
+REQUEST_TIMEOUT=10000
+OUTPUT_PATH=/var/certs
+CA_NAME=RA
+OUTPUT_TYPE=PEM
+#Csr config envs
+COMMON_NAME=httpserver-onap.org
+ORGANIZATION=Linux-Foundation
+ORGANIZATION_UNIT=ONAP
+LOCATION=San-Francisco
+STATE=California
+COUNTRY=US
+#Tls config envs
+KEYSTORE_PATH=/etc/onap/oom/certservice/certs/certServiceClient-keystore.jks
+KEYSTORE_PASSWORD=secret
+TRUSTSTORE_PATH=/etc/onap/oom/certservice/certs/truststore.jks
+TRUSTSTORE_PASSWORD=secret
diff --git a/sanitycheck/vesclient-secured/certservice/resources/certservice-client/client-configuration-for-ves.env b/sanitycheck/vesclient-secured/certservice/resources/certservice-client/client-configuration-for-ves.env
new file mode 100644
index 0000000..e06d147
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/resources/certservice-client/client-configuration-for-ves.env
@@ -0,0 +1,19 @@
+#Client envs
+REQUEST_URL=https://oom-cert-service:8443/v1/certificate/
+REQUEST_TIMEOUT=10000
+OUTPUT_PATH=/var/certs
+CA_NAME=RA
+OUTPUT_TYPE=JKS
+#Csr config envs
+COMMON_NAME=ves-onap.org
+ORGANIZATION=Linux-Foundation
+ORGANIZATION_UNIT=ONAP
+LOCATION=San-Francisco
+STATE=California
+COUNTRY=US
+SANS=ves
+#Tls config envs
+KEYSTORE_PATH=/etc/onap/oom/certservice/certs/certServiceClient-keystore.jks
+KEYSTORE_PASSWORD=secret
+TRUSTSTORE_PATH=/etc/onap/oom/certservice/certs/truststore.jks
+TRUSTSTORE_PASSWORD=secret
diff --git a/sanitycheck/vesclient-secured/certservice/resources/certservice-client/client-configuration-for-vesclient.env b/sanitycheck/vesclient-secured/certservice/resources/certservice-client/client-configuration-for-vesclient.env
new file mode 100644
index 0000000..c5f33b6
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/resources/certservice-client/client-configuration-for-vesclient.env
@@ -0,0 +1,18 @@
+#Client envs
+REQUEST_URL=https://oom-cert-service:8443/v1/certificate/
+REQUEST_TIMEOUT=10000
+OUTPUT_PATH=/var/certs
+CA_NAME=RA
+OUTPUT_TYPE=P12
+#Csr config envs
+COMMON_NAME=onap.org
+ORGANIZATION=Linux-Foundation
+ORGANIZATION_UNIT=ONAP
+LOCATION=San-Francisco
+STATE=California
+COUNTRY=US
+#Tls config envs
+KEYSTORE_PATH=/etc/onap/oom/certservice/certs/certServiceClient-keystore.jks
+KEYSTORE_PASSWORD=secret
+TRUSTSTORE_PATH=/etc/onap/oom/certservice/certs/truststore.jks
+TRUSTSTORE_PASSWORD=secret
diff --git a/sanitycheck/vesclient-secured/certservice/resources/certservice/cmpServers.json b/sanitycheck/vesclient-secured/certservice/resources/certservice/cmpServers.json
new file mode 100644
index 0000000..7256494
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/resources/certservice/cmpServers.json
@@ -0,0 +1,24 @@
+{
+  "cmpv2Servers": [
+    {
+      "caName": "Client",
+      "url": "http://oomcert-ejbca:8080/ejbca/publicweb/cmp/cmp",
+      "issuerDN": "CN=ManagementCA",
+      "caMode": "CLIENT",
+      "authentication": {
+        "iak": "mypassword",
+        "rv": "mypassword"
+      }
+    },
+    {
+      "caName": "RA",
+      "url": "http://oomcert-ejbca:8080/ejbca/publicweb/cmp/cmpRA",
+      "issuerDN": "CN=ManagementCA",
+      "caMode": "RA",
+      "authentication": {
+        "iak": "mypassword",
+        "rv": "mypassword"
+      }
+    }
+  ]
+}
diff --git a/sanitycheck/vesclient-secured/certservice/resources/ejbca/ejbca-configuration.sh b/sanitycheck/vesclient-secured/certservice/resources/ejbca/ejbca-configuration.sh
new file mode 100755
index 0000000..77f5c55
--- /dev/null
+++ b/sanitycheck/vesclient-secured/certservice/resources/ejbca/ejbca-configuration.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+configureEjbca() {
+    ejbca.sh config cmp addalias --alias cmpRA
+    ejbca.sh config cmp updatealias --alias cmpRA --key operationmode --value ra
+    ejbca.sh ca editca --caname ManagementCA --field cmpRaAuthSecret --value mypassword
+    ejbca.sh config cmp updatealias --alias cmpRA --key responseprotection --value pbe
+    ejbca.sh config cmp dumpalias --alias cmpRA
+    ejbca.sh config cmp addalias --alias cmp
+    ejbca.sh config cmp updatealias --alias cmp --key allowautomatickeyupdate --value true
+    ejbca.sh config cmp updatealias --alias cmp --key responseprotection --value pbe
+    ejbca.sh ra addendentity --username Node123 --dn "CN=Node123" --caname ManagementCA --password mypassword --type 1 --token USERGENERATED
+    ejbca.sh ra setclearpwd --username Node123 --password mypassword
+    ejbca.sh config cmp updatealias --alias cmp --key extractusernamecomponent --value CN
+    ejbca.sh config cmp dumpalias --alias cmp
+    ejbca.sh ca getcacert --caname ManagementCA -f /dev/stdout > cacert.pem
+}
+
+configureEjbca