E2E Integration Test for NETCONF/TLS Configuration in SDNC.

Story intended to capture needed updates to E2E Integration Test for NETCONF/TLS Configuration.
Involve updates to the PNF simulator.

Issue-ID: INT-1295
Signed-off-by: ajay_dp001 <ajay.deep.singh@est.tech>
Change-Id: Ie08fe9618a9a0522e00fe0af8d13ab48b0634a70

9 files changed, 520 insertions, 52 deletions

diff --git a/tests/sdnc/sdnc_netconf_tls_post_deploy/_init_.robot b/tests/sdnc/sdnc_netconf_tls_post_deploy/__init__.robot
index d7353060..d7353060 100644
--- a/tests/sdnc/sdnc_netconf_tls_post_deploy/_init_.robot
+++ b/tests/sdnc/sdnc_netconf_tls_post_deploy/__init__.robot
diff --git a/tests/sdnc/sdnc_netconf_tls_post_deploy/csr/netconf_pnp_simulator_csr.env b/tests/sdnc/sdnc_netconf_tls_post_deploy/csr/netconf_pnp_simulator_csr.env
new file mode 100644
index 00000000..557860de
--- /dev/null
+++ b/tests/sdnc/sdnc_netconf_tls_post_deploy/csr/netconf_pnp_simulator_csr.env
@@ -0,0 +1,16 @@
+#Client Envs
+REQUEST_TIMEOUT=30000
+OUTPUT_PATH=/var/certs
+CA_NAME=RA
+KEYSTORE_PATH=/etc/onap/aaf/certservice/certs/certServiceClient-keystore.jks
+KEYSTORE_PASSWORD=secret
+TRUSTSTORE_PATH=/etc/onap/aaf/certservice/certs/truststore.jks
+TRUSTSTORE_PASSWORD=secret
+#CSR Config Envs
+COMMON_NAME=netconf.pnp.simulator.onap.org
+ORGANIZATION=Linux-Foundation
+ORGANIZATION_UNIT=ONAP
+LOCATION=San-Francisco
+STATE=California
+COUNTRY=US
+SANS=netconf.com:netconfsimulator.com
diff --git a/tests/sdnc/sdnc_netconf_tls_post_deploy/csr/sdnc_csr.env b/tests/sdnc/sdnc_netconf_tls_post_deploy/csr/sdnc_csr.env
new file mode 100644
index 00000000..28411797
--- /dev/null
+++ b/tests/sdnc/sdnc_netconf_tls_post_deploy/csr/sdnc_csr.env
@@ -0,0 +1,16 @@
+#Client CSR
+REQUEST_TIMEOUT=30000
+OUTPUT_PATH=/var/certs
+CA_NAME=RA
+KEYSTORE_PATH=/etc/onap/aaf/certservice/certs/certServiceClient-keystore.jks
+KEYSTORE_PASSWORD=secret
+TRUSTSTORE_PATH=/etc/onap/aaf/certservice/certs/truststore.jks
+TRUSTSTORE_PASSWORD=secret
+#CSR Config Envs
+COMMON_NAME=sdnc.onap.org
+ORGANIZATION=Linux-Foundation
+ORGANIZATION_UNIT=ONAP
+LOCATION=San-Francisco
+STATE=California
+COUNTRY=US
+SANS=example.com:sample.com
\ No newline at end of file
diff --git a/tests/sdnc/sdnc_netconf_tls_post_deploy/data/mount.xml b/tests/sdnc/sdnc_netconf_tls_post_deploy/data/mount.xml
deleted file mode 100644
index 108369bc..00000000
--- a/tests/sdnc/sdnc_netconf_tls_post_deploy/data/mount.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<node xmlns="urn:TBD:params:xml:ns:yang:network-topology">
-  <node-id>netopeer2</node-id>
-    <key-based xmlns="urn:opendaylight:netconf-node-topology">
-   <key-id xmlns="urn:opendaylight:netconf-node-topology">ODL_private_key_0</key-id>
-   <username xmlns="urn:opendaylight:netconf-node-topology">netconf</username>
-   </key-based>
-   <host xmlns="urn:opendaylight:netconf-node-topology">pnfaddr</host>
-   <port xmlns="urn:opendaylight:netconf-node-topology">6513</port>
-  <tcp-only xmlns="urn:opendaylight:netconf-node-topology">false</tcp-only>
-  <protocol xmlns="urn:opendaylight:netconf-node-topology">
-  <name xmlns="urn:opendaylight:netconf-node-topology">TLS</name>
-  </protocol>
- <max-connection-attempts xmlns="urn:opendaylight:netconf-node-topology">2</max-connection-attempts>
-</node>
diff --git a/tests/sdnc/sdnc_netconf_tls_post_deploy/libraries/ClientManager.py b/tests/sdnc/sdnc_netconf_tls_post_deploy/libraries/ClientManager.py
new file mode 100644
index 00000000..ceff9742
--- /dev/null
+++ b/tests/sdnc/sdnc_netconf_tls_post_deploy/libraries/ClientManager.py
@@ -0,0 +1,179 @@
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2020 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+__author__ = "Ajay Deep Singh (ajay.deep.singh@est.tech)"
+__copyright__ = "Copyright (C) 2020 Nordix Foundation"
+__license__ = "Apache 2.0"
+
+import os
+import shutil
+import subprocess
+
+import docker
+from OpenSSL import crypto
+from docker.types import Mount
+
+DEV_NULL = open(os.devnull, 'wb')
+NETCONF_PNP_SIM_CONTAINER_NAME = 'netconf-simulator'
+ARCHIVES_PATH = os.getenv("WORKSPACE") + "/archives/"
+
+
+class ClientManager:
+
+    def __init__(self, mount_path, truststore_path):
+        self.mount_path = mount_path
+        self.truststore_path = truststore_path
+        self.caCertPem = mount_path + '/ca.pem'
+        self.serverKeyPem = mount_path + '/server_key.pem'
+        self.serverCertPem = mount_path + '/server_cert.pem'
+        self.keystoreJksPath = mount_path + '/keystore.jks'
+        self.keystorePassPath = mount_path + '/keystore.pass'
+        self.truststoreJksPath = mount_path + '/truststore.jks'
+        self.truststorePassPath = mount_path + '/truststore.pass'
+
+    # Function Create docker container.
+    def run_client_container(self, client_image, container_name, path_to_env, request_url, network):
+        self.create_mount_dir()
+        client = docker.from_env()
+        environment = self.read_env_list_from_file(path_to_env)
+        environment.append("REQUEST_URL=" + request_url)
+        container = client.containers.run(
+            image=client_image,
+            name=container_name,
+            environment=environment,
+            network=network,
+            user='root',
+            mounts=[Mount(target='/var/certs', source=self.mount_path, type='bind'),
+                    Mount(target='/etc/onap/aaf/certservice/certs/', source=self.truststore_path, type='bind')],
+            detach=True
+        )
+        exitcode = container.wait()
+        return exitcode
+
+    # Function to validate keystore.jks/truststore.jks can be opened with generated pass-phrase.
+    def can_open_keystore_and_truststore_with_pass(self):
+        can_open_keystore = self.can_open_jks_file_with_pass_file(self.keystorePassPath, self.keystoreJksPath)
+        can_open_truststore = self.can_open_jks_file_with_pass_file(self.truststorePassPath, self.truststoreJksPath)
+        return can_open_keystore & can_open_truststore
+
+    # Method for Uploading Certificate in SDNC-Container.
+    # Creating/Uploading Server-key, Server-cert, Ca-cert PEM files in Netconf-Pnp-Simulator.
+    def can_install_keystore_and_truststore_certs(self, cmd, container_name):
+        continue_exec = True
+        if container_name == NETCONF_PNP_SIM_CONTAINER_NAME:
+            print("Generating PEM files for {0} from JKS files".format(container_name))
+            continue_exec = self.create_pem(self.keystorePassPath, self.keystoreJksPath, self.truststorePassPath,
+                                            self.truststoreJksPath)
+        if continue_exec:
+            print("Initiate Configuration Push for : {0}".format(container_name))
+            resp_code = self.execute_bash_config(cmd, container_name)
+            if resp_code == 0:
+                print("Execution Successful for: {0}".format(container_name))
+                return True
+            else:
+                print("Execution Failed for: {0}".format(container_name))
+                return False
+
+    def create_pem(self, keystore_pass_file_path, keystore_jks_file_path, truststore_pass_file_path,
+                   truststore_jks_file_path):
+        # Create [server_key.pem, server_cert.pem, ca.pem] files for Netconf-Pnp-Simulation/TLS Configuration.
+        try:
+            keystore_p12 = self.get_pkcs12(keystore_pass_file_path, keystore_jks_file_path)
+            truststore_p12 = self.get_pkcs12(truststore_pass_file_path, truststore_jks_file_path)
+            with open(self.serverKeyPem, "wb+") as key_file:
+                key_file.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, keystore_p12.get_privatekey()))
+            with open(self.serverCertPem, "wb+") as server_cert_file:
+                server_cert_file.write(crypto.dump_certificate(crypto.FILETYPE_PEM, keystore_p12.get_certificate()))
+            with open(self.caCertPem, "wb+") as ca_cert_file:
+                ca_cert_file.write(
+                    crypto.dump_certificate(crypto.FILETYPE_PEM, truststore_p12.get_ca_certificates()[0]))
+            return True
+        except IOError as err:
+            print("I/O Error: {0}".format(err))
+            return False
+        except Exception as e:
+            print("UnExpected Error: {0}".format(e))
+            return False
+
+    def can_open_jks_file_with_pass_file(self, pass_file_path, jks_file_path):
+        try:
+            if jks_file_path.split('/')[-1] == 'truststore.jks':
+                pkcs12 = self.get_pkcs12(pass_file_path, jks_file_path).get_ca_certificates()[0]
+            else:
+                pkcs12 = self.get_pkcs12(pass_file_path, jks_file_path).get_certificate()
+            if pkcs12 is None:
+                return False
+            return True
+        except IOError as err:
+            print("I/O Error PKCS12 Creation failed: {0}".format(err))
+            return False
+        except Exception as e:
+            print("UnExpected Error PKCS12 Creation failed: {0}".format(e))
+            return False
+
+    def remove_client_container_and_save_logs(self, container_name, log_file_name):
+        client = docker.from_env()
+        container = client.containers.get(container_name)
+        text_file = open(ARCHIVES_PATH + container_name + '_' + log_file_name + ".log", "w")
+        text_file.write(container.logs())
+        text_file.close()
+        container.remove()
+        self.remove_mount_dir()
+
+    def create_mount_dir(self):
+        if not os.path.exists(self.mount_path):
+            os.makedirs(self.mount_path)
+
+    def remove_mount_dir(self):
+        shutil.rmtree(self.mount_path)
+
+    @staticmethod
+    def get_pkcs12(pass_file_path, jks_file_path):
+        # Load PKCS12 Object
+        password = open(pass_file_path, 'rb').read()
+        p12 = crypto.load_pkcs12(open(jks_file_path, 'rb').read(), password)
+        return p12
+
+    @staticmethod
+    def execute_bash_config(cmd, container_name):
+        # Run command with arguments. Wait for command to complete or timeout, return code attribute.
+        try:
+            resp_code = subprocess.call(["%s %s" % (cmd, container_name)], shell=True, stdout=DEV_NULL,
+                                        stderr=subprocess.STDOUT)
+            print("Response Code from Config.sh execution: {0}".format(resp_code))
+            return resp_code
+        except subprocess.CalledProcessError as e:
+            print("CalledProcessError Certificate installation failed in SDNC-ODL Container: {0}".format(e))
+            return 1  # Return Error Code
+
+    @staticmethod
+    def get_container_logs(container_name):
+        client = docker.from_env()
+        container = client.containers.get(container_name)
+        logs = container.logs()
+        return logs
+
+    @staticmethod
+    def read_env_list_from_file(path):
+        f = open(path, "r")
+        r_list = []
+        for line in f:
+            line = line.strip()
+            if line[0] != "#":
+                r_list.append(line)
+        return r_list
diff --git a/tests/sdnc/sdnc_netconf_tls_post_deploy/libraries/config.sh b/tests/sdnc/sdnc_netconf_tls_post_deploy/libraries/config.sh
new file mode 100755
index 00000000..cc6bf188
--- /dev/null
+++ b/tests/sdnc/sdnc_netconf_tls_post_deploy/libraries/config.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+
+#
+# ============LICENSE_START=======================================================
+#   Copyright (C) 2020 Nordix Foundation.
+# ================================================================================
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#  SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+# @author Ajay Deep Singh (ajay.deep.singh@est.tech)
+
+CONTAINER_NAME="$1"
+LOGFILE="${WORKSPACE}"/archives/config.log
+CONTAINER_ID=$(docker inspect --format="{{.Id}}" "$CONTAINER_NAME")
+
+OWNER="odl"
+DEST_DIR="/tmp"
+
+CERT_DIR="${WORKSPACE}"/tests/sdnc/sdnc_netconf_tls_post_deploy/cert-data/*
+
+function now_ms() {
+  date +"%Y-%m-%d %H:%M:%S.%3N"
+}
+
+function log() {
+  local level=$1
+  shift
+  local message="$*"
+  printf "%s %-5s %s\n" "$(now_ms)" "$level" "$message" >>"$LOGFILE"
+}
+
+# Copy [keystore.jks, truststore.jks, truststore.pass, keystore.pass] files into SDNC container.
+function docker_cp() {
+  local file=$1
+  docker cp "$file" "$CONTAINER_ID":"$DEST_DIR"
+  docker exec -u 0 "$CONTAINER_ID" chown "$OWNER":"$OWNER" "$DEST_DIR"/"${file##*/}"
+}
+
+# Run installCerts.py script to push X509 Certificates to SDNC-ODL Keystore/Truststore.
+function sdnc_conf() {
+  log INFO "Configuring SDNC-ODL Keystore..."
+  count=0
+  exit_code=false
+  for i in {1..4}; do
+    for file in $CERT_DIR; do
+      if [[ -f $file ]]; then
+        log INFO "Uploading file :" "$file"
+        docker_cp "$file"
+        count=$((count + 1))
+      fi
+    done
+    if [[ $count -eq 4 ]]; then
+      log INFO "SDNC JKS files upload successful"
+      exit_code=true
+      break
+    fi
+    log DEBUG "Waiting for JKS files to be uploaded to SDNC container.."
+    sleep 2m
+  done
+  if [[ "$exit_code" != "true" ]]; then
+    log DEBUG "JKS files Not found in $CERT_DIR"
+    exit 1 # Return error code
+  fi
+  sleep 2m
+  docker exec "$CONTAINER_ID" rm -rf /tmp/certs.properties
+  docker exec "$CONTAINER_ID" rm -rf /tmp/keys0.zip
+  if ! docker exec "$CONTAINER_ID" /usr/bin/python /opt/onap/sdnc/bin/installCerts.py; then
+    log DEBUG "Issue executing installCerts.py script"
+    docker cp "$CONTAINER_ID":/opt/opendaylight/data/log/installCerts.log "${WORKSPACE}"/archives
+    exit 1 # Return error code
+  fi
+  log INFO "Configuring SDNC-ODL Keystore successful"
+}
+
+# Copy [Server_key.pem, Server_cert.pem, Ca.pem] files into Netconf-Simulator container.
+# Reconfigure TLS config by invoking reconfigure-tls.sh script.
+function netconf-simulator_conf() {
+  log INFO "Configuring Netconf-Pnp-Simulator..."
+  count=0
+  exit_code=false
+  for i in {1..4}; do
+    for file in $CERT_DIR; do
+      if [[ -f $file && ${file: -4} == ".pem" ]]; then
+        log INFO "Uploading file :" "$file"
+        docker cp "$file" "$CONTAINER_ID":/config/tls
+        count=$((count + 1))
+      fi
+    done
+    if [[ $count -eq 3 ]]; then
+      log INFO "PEM files upload successful"
+      exit_code=true
+      break
+    fi
+    log DEBUG "Waiting for PEM files to be uploaded to Netconf-Pnp-Simulator.."
+    sleep 2m
+  done
+  if [[ "$exit_code" != "true" ]]; then
+    log DEBUG "PEM files Not found in $CERT_DIR"
+    exit 1 # Return error code
+  fi
+  sleep 2m
+  if ! docker exec "$CONTAINER_ID" /opt/bin/reconfigure-tls.sh; then
+    log DEBUG "Issue executing reconfigure-tls.sh script"
+    docker logs "$CONTAINER_ID" > "${WORKSPACE}"/archives/simulator.log
+    exit 1 # Return error code
+  fi
+  log INFO "Configuring Netconf-Pnp-Simulator successful"
+}
+
+# Push Config on SDNC, Netconf-Simulator.
+if [[ -n $CONTAINER_ID ]]; then
+  log INFO "Container Name: $CONTAINER_NAME, Container Id: $CONTAINER_ID"
+  if [[ "$CONTAINER_NAME" == "sdnc" ]]; then
+    sdnc_conf
+  elif [[ "$CONTAINER_NAME" == "netconf-simulator" ]]; then
+    netconf-simulator_conf
+  fi
+fi
diff --git a/tests/sdnc/sdnc_netconf_tls_post_deploy/resources/sdnc-keywords.robot b/tests/sdnc/sdnc_netconf_tls_post_deploy/resources/sdnc-keywords.robot
new file mode 100644
index 00000000..8e36e65f
--- /dev/null
+++ b/tests/sdnc/sdnc_netconf_tls_post_deploy/resources/sdnc-keywords.robot
@@ -0,0 +1,84 @@
+*** Settings ***
+
+Resource          ../../../common.robot
+Resource          ./sdnc-properties.robot
+
+Library           Collections
+Library 	      RequestsLibrary
+Library           HttpLibrary.HTTP
+Library           ../libraries/ClientManager.py  ${MOUNT_PATH}  ${TRUSTSTORE_PATH}
+
+*** Keywords ***
+
+Create sessions
+    [Documentation]  Create all required sessions
+    ${certs}=  Create List  ${CERTSERVICE_SERVER_CRT}  ${CERTSERVICE_SERVER_KEY}
+    Create Client Cert Session  alias  ${AAFCERT_URL}  client_certs=${certs}  verify=${ROOTCA}  disable_warnings=1
+    Set Suite Variable  ${https_valid_cert_session}  alias
+
+Run Healthcheck
+    [Documentation]  Run Healthcheck
+    ${resp}=  Get Request 	${https_valid_cert_session} 	/actuator/health
+    Should Be Equal As Strings 	${resp.status_code} 	200
+    Validate Recieved Response  ${resp}  status  UP
+
+Validate Recieved Response
+    [Documentation]  Validate message that has been received
+    [Arguments]  ${resp}  ${key}  ${expected_value}
+    ${json}=    Parse Json      ${resp.content}
+    ${value}=  Get From Dictionary  ${json}  ${key}
+    Should Be Equal As Strings    ${value}    ${expected_value}
+
+Send Get Request And Validate Response
+    [Documentation]   Send request to passed url and validate received response
+    [Arguments]   ${path}  ${resp_code}
+    ${resp}= 	Get Request 	${https_valid_cert_session}  ${path}
+    Should Be Equal As Strings 	${resp.status_code} 	${resp_code}
+
+Send Get Request And Validate Response Sdnc
+    [Documentation]   Send request to passed url and validate received response
+    [Arguments]   ${path}  ${resp_code}
+    Create Session   sdnc_restconf  ${SDNC_RESTCONF_URL}
+    &{headers}=  Create Dictionary    Authorization=Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==    Content-Type=application/json    Accept=application/json
+    ${resp}= 	Get Request    sdnc_restconf    ${path}    headers=${headers}
+    Should Be Equal As Strings 	${resp.status_code} 	${resp_code}
+
+Send Get Request And Validate TLS Connection Response
+    [Documentation]   Send request to passed url and validate received response
+    [Arguments]   ${path}  ${resp_code}
+    Create Session   sdnc_restconf  ${SDNC_RESTCONF_URL}
+    ${mount}=    Get File    ${REQUEST_DATA_PATH}${/}mount.xml
+    &{headers}=  Create Dictionary    Authorization=Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==    Content-Type=application/xml    Accept=application/xml
+    ${resp}=    Put Request    sdnc_restconf    ${path}    data=${mount}    headers=${headers}
+    Should Be Equal As Strings    ${resp.status_code}    201
+    Sleep  30
+    &{headers1}=  Create Dictionary    Authorization=Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==    Content-Type=application/json    Accept=application/json
+    ${resp1}=    Get Request    sdnc_restconf    ${PNFSIM_MOUNT_PATH}    headers=${headers1}
+    Should Be Equal As Strings    ${resp1.status_code}    ${resp_code}
+    Should Contain  ${resp1.content}     netconf-id
+    Should Contain  ${resp1.content}     netconf-param
+
+Send Delete Request And Validate PNF Mount Deleted
+    [Documentation]   Send request to passed url and validate received response
+    [Arguments]   ${path}  ${resp_code}
+    Create Session   sdnc_restconf  ${SDNC_RESTCONF_URL}
+    ${mount}=    Get File    ${REQUEST_DATA_PATH}${/}mount.xml
+    &{headers}=  Create Dictionary    Authorization=Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==    Content-Type=application/json    Accept=application/json
+    ${deleteresponse}=    Delete Request    sdnc_restconf    ${path}    data=${mount}    headers=${headers}
+    Should Be Equal As Strings 	${deleteresponse.status_code} 	${resp_code}
+    Sleep  30
+    ${del_topology}=    Delete Request    sdnc_restconf    ${SDNC_NETWORK_TOPOLOGY}
+    ${del_keystore}=    Delete Request    sdnc_restconf    ${SDNC_KEYSTORE_CONFIG_PATH}
+    Should Be Equal As Strings    ${del_keystore.status_code}    ${resp_code}
+    Should Be Equal As Strings    ${del_topology.status_code}    ${resp_code}
+
+Run Cert Service Client And Validate JKS File Creation And Client Exit Code
+    [Documentation]  Run Cert Service Client Container And Validate Exit Code For SDNC
+    [Arguments]   ${env_file}  ${CONTAINER_NAME}  ${expected_exit_code}
+    ${exit_code}=  Run Client Container  ${DOCKER_CLIENT_IMAGE}  ${CLIENT_CONTAINER_NAME}  ${env_file}  ${CERT_SERVICE_ADDRESS}${CERT_SERVICE_ENDPOINT}  ${CERT_SERVICE_NETWORK}
+    ${can_open}=  Can Open Keystore And Truststore With Pass
+    ${install_certs}=  Can Install Keystore And Truststore Certs  ${CONF_SCRIPT}  ${CONTAINER_NAME}
+    Remove Client Container And Save Logs  ${CLIENT_CONTAINER_NAME}  positive_path
+    Should Be Equal As Strings  ${exit_code}  ${expected_exit_code}  Client return: ${exitcode} exit code, but expected: ${expected_exit_code}
+    Should Be True  ${can_open}  Cannot Open Keystore/TrustStore by Passphrase
+    Should Be True  ${install_certs}  Cannot Install Keystore/Truststore
\ No newline at end of file
diff --git a/tests/sdnc/sdnc_netconf_tls_post_deploy/resources/sdnc-properties.robot b/tests/sdnc/sdnc_netconf_tls_post_deploy/resources/sdnc-properties.robot
new file mode 100644
index 00000000..131a52f9
--- /dev/null
+++ b/tests/sdnc/sdnc_netconf_tls_post_deploy/resources/sdnc-properties.robot
@@ -0,0 +1,37 @@
+*** Variables ***
+
+# AAF CertService
+${NEXUS_DOCKER_REPO}                     nexus3.onap.org:10001
+
+${RA_CA_NAME}                            RA
+${CERT_SERVICE_PORT}                     8443
+${CERT_SERVICE_CONTAINER_NAME}           aaf-cert-service
+${CERT_SERVICE_NETWORK}                  certservice_certservice
+${AAFCERT_URL}                           https://localhost:${CERT_SERVICE_PORT}
+${CERT_SERVICE_ENDPOINT}                 /v1/certificate/
+${CERT_SERVICE_ADDRESS}                  https://${CERT_SERVICE_CONTAINER_NAME}:${CERT_SERVICE_PORT}
+${ROOTCA}                                %{WORKSPACE}/tests/sdnc/sdnc_netconf_tls_post_deploy/certs/root.crt
+${CERTSERVICE_SERVER_CRT}                %{WORKSPACE}/tests/sdnc/sdnc_netconf_tls_post_deploy/certs/certServiceServer.crt
+${CERTSERVICE_SERVER_KEY}                %{WORKSPACE}/tests/sdnc/sdnc_netconf_tls_post_deploy/certs/certServiceServer.key
+
+#AAF CerService Client
+${CLIENT_CONTAINER_NAME}                 %{CLIENT_CONTAINER_NAME}
+${DOCKER_CLIENT_IMAGE}                   nexus3.onap.org:10001/onap/org.onap.aaf.certservice.aaf-certservice-client:latest
+${TRUSTSTORE_PATH}                       %{WORKSPACE}/plans/sdnc/sdnc_netconf_tls_post_deploy/certs
+
+# SDNC Configuration
+${REQUEST_DATA_PATH}                     %{REQUEST_DATA_PATH}
+${SDNC_CONTAINER_NAME}                   %{SDNC_CONTAINER_NAME}
+${SDNC_RESTCONF_URL}                     http://localhost:8282/restconf
+${SDNC_KEYSTORE_CONFIG_PATH}             /config/netconf-keystore:keystore
+${SDNC_NETWORK_TOPOLOGY}                 /config/network-topology:network-topology
+${MOUNT_PATH}                            %{WORKSPACE}/tests/sdnc/sdnc_netconf_tls_post_deploy/cert-data
+${SDNC_CSR_FILE}                         %{WORKSPACE}/tests/sdnc/sdnc_netconf_tls_post_deploy/csr/sdnc_csr.env
+${SDNC_MOUNT_PATH}                       /config/network-topology:network-topology/topology/topology-netconf/node/PNFDemo
+${PNFSIM_MOUNT_PATH}                     /config/network-topology:network-topology/topology/topology-netconf/node/PNFDemo/yang-ext:mount/mynetconf:netconflist
+
+# Netconf-Pnp-Simulator
+${NETCONF_PNP_SIM_CONTAINER_NAME}        %{NETCONF_PNP_SIM_CONTAINER_NAME}
+${NETCONF_PNP_SIM_CSR_FILE}              %{WORKSPACE}/tests/sdnc/sdnc_netconf_tls_post_deploy/csr/netconf_pnp_simulator_csr.env
+${CONF_SCRIPT}                           %{WORKSPACE}/tests/sdnc/sdnc_netconf_tls_post_deploy/libraries/config.sh
+${CONF_TLS_SCRIPT}                       %{WORKSPACE}/tests/sdnc/sdnc_netconf_tls_post_deploy/libraries/config_tls.sh
\ No newline at end of file
diff --git a/tests/sdnc/sdnc_netconf_tls_post_deploy/sdnc_post_deploy_cert_check.robot b/tests/sdnc/sdnc_netconf_tls_post_deploy/sdnc_post_deploy_cert_check.robot
index 75283dcb..c2b35e12 100644
--- a/tests/sdnc/sdnc_netconf_tls_post_deploy/sdnc_post_deploy_cert_check.robot
+++ b/tests/sdnc/sdnc_netconf_tls_post_deploy/sdnc_post_deploy_cert_check.robot
@@ -1,39 +1,60 @@
 *** Settings ***
-Library     Collections
-Library     RequestsLibrary
-Library     OperatingSystem
-Library     json
-Library     String
-
-*** Variables ***
-${SDNC_KEYSTORE_CONFIG_PATH}    /config/netconf-keystore:keystore
-${SDNC_MOUNT_PATH}    /config/network-topology:network-topology/topology/topology-netconf/node/netopeer2
-${PNFSIM_MOUNT_PATH}    /config/network-topology:network-topology/topology/topology-netconf/node/netopeer2/yang-ext:mount/mynetconf:netconflist
-
- *** Test Cases ***
- Test SDNC Keystore
-      [Documentation]    Checking keystore after SDNC installation
-      Create Session   sdnc  http://localhost:8282/restconf
-      &{headers}=  Create Dictionary    Authorization=Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==    Content-Type=application/json    Accept=application/json
-      ${resp}=    Get Request    sdnc    ${SDNC_KEYSTORE_CONFIG_PATH}    headers=${headers}
-      Should Be Equal As Strings    ${resp.status_code}    200
-      ${keystoreContent}=    Convert To String    ${resp.content}
-      Log to console  *************************
-      Log to console  ${resp.content}
-      Log to console  *************************
-
-# Test SDNC PNF Mount
-#     [Documentation]    Checking PNF mount after SDNC installation
-#     Create Session   sdnc  http://localhost:8282/restconf
-#     ${mount}=    Get File     ${CURDIR}${/}data${/}mount.xml
-#     Log to console  ${mount}
-#     &{headers}=  Create Dictionary    Authorization=Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==    Content-Type=application/xml    Accept=application/xml
-#     ${resp}=    Put Request    sdnc    ${SDNC_MOUNT_PATH}    data=${mount}    headers=${headers}
-#     Should Be Equal As Strings    ${resp.status_code}    201
-#     Sleep  30
-#     &{headers1}=  Create Dictionary    Authorization=Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ==    Content-Type=application/json    Accept=application/json
-#     ${resp1}=    Get Request    sdnc    ${PNFSIM_MOUNT_PATH}    headers=${headers1}
-#     Should Be Equal As Strings    ${resp1.status_code}    200
-#     Log to console  ${resp1.content}
-#     Should Contain  ${resp1.content}     netconf-id
-#     Should Contain  ${resp1.content}     netconf-param
\ No newline at end of file
+
+Documentation     SDNC, Netconf-Pnp-Simulator E2E Test Case Scenarios
+
+Library 	      RequestsLibrary
+Resource          ./resources/sdnc-keywords.robot
+
+Suite Setup       Create sessions
+
+*** Test Cases ***
+
+Health Check AAF CertService
+    [Tags]      AAF-CERT-SERVICE
+    [Documentation]   Service is Up and Running
+    Run health check
+
+Reload AAF CertService Configuration
+    [Tags]      AAF-CERT-SERVICE
+    [Documentation]   Configuration is Reloaded
+    Send Get Request And Validate Response  /reload  200
+
+Check AAF CertService Container Is Ready
+    [Tags]      AAF-CERT-SERVICE
+    [Documentation]   Send Request to /ready Endpoint and Expect 200
+    Send Get Request And Validate Response  /ready  200
+
+Check SDNC Keystore For Netopeer2 Certificates
+    [Tags]      SDNC-NETOPEER2-CERT-DEPLOYMENT
+    [Documentation]    Checking Keystore after SDNC istallation
+    Send Get Request And Validate Response Sdnc  ${SDNC_KEYSTORE_CONFIG_PATH}  200
+
+Check SDNC And PNF TLS Connection Over Netopeer2 Certificates
+    [Tags]      SDNC-PNF-TLS-CONNECTION-CHECK
+    [Documentation]    Checking PNF Mount after SDNC Installation
+    Send Get Request And Validate TLS Connection Response  ${SDNC_MOUNT_PATH}  200
+
+Check PNF Delete And Remove Netopeer2 Certificates From Keystore
+    [Tags]      SDNC-PNF-MOUNT-DELETE-CLEAR-KEYSTORE
+    [Documentation]    Checking PNF Mount Delete from SDNC
+    Send Delete Request And Validate PNF Mount Deleted  ${SDNC_MOUNT_PATH}  200
+
+Check AAF-CertService Successfully Creates Certificates for SDNC
+    [Tags]      AAF-CERT-SERVICE-SDNC
+    [Documentation]  Run with SDNC CSR and Expected Exit Code 0
+    Run Cert Service Client And Validate JKS File Creation And Client Exit Code  ${SDNC_CSR_FILE}  ${SDNC_CONTAINER_NAME}  0
+
+Check SDNC-ODL Certificates Installation In Keystore And Truststore
+    [Tags]      SDNC-ODL-CERTIFICATE-KEYSTORE-VALIDATE
+    [Documentation]  Validate Certificates Got Installed in SDNC-ODL Keystore
+    Send Get Request And Validate Response Sdnc  ${SDNC_KEYSTORE_CONFIG_PATH}  200
+
+Check AAF-CertService Successfully Creates Certificates for Netconf-Pnp-Simulator
+    [Tags]      AAF-CERT-SERVICE-NETCONF_PNP_SIMULATOR
+    [Documentation]  Run with NETCONF-PNP-SIMULATOR CSR and Expect Exit Code 0
+    Run Cert Service Client And Validate JKS File Creation And Client Exit Code  ${NETCONF_PNP_SIM_CSR_FILE}  ${NETCONF_PNP_SIM_CONTAINER_NAME}  0
+
+Check SDNC-ODL Netconf-Pnp-Simulatore TLS Connection Establishment
+    [Tags]      SDNC-ODL-NETCONF-PNP_SIMULATION-TLS-CONNECTION
+    [Documentation]  Validate SDNC-ODL and Netconf-Pnp-Simulation TLS Connection Establishment
+    Send Get Request And Validate TLS Connection Response  ${SDNC_MOUNT_PATH}  200
\ No newline at end of file