Add SLA Validations for Apex-PDP

- Add multiple apex microservice SLA tests
- Add single instance apex SLA tests
- Single instance SLA tests to be ran as part of CSIT runs
- Multiple microservice will not be ran as part of CSITs due to high resource consumption
- Multiple microservice SLA test results can be found in the linked JIRA
- Added setup scripts for multiple instance SLA tests
- nginx added for load balancing purposes for multiple apex instances

Issue-ID: POLICY-4530
Issue-ID: POLICY-4164
Change-Id: Ib86e96c57f4b7bf2b4f5e930fd7d4a3805b50687
Signed-off-by: Wayne Dunican <wayne.dunican@est.tech>
Signed-off-by: adheli.tavares <adheli.tavares@est.tech>

16 files changed, 579 insertions, 111 deletions

diff --git a/compose/config/nginx/nginx.conf b/compose/config/nginx/nginx.conf
new file mode 100644
index 00000000..08d65627
--- /dev/null
+++ b/compose/config/nginx/nginx.conf
@@ -0,0 +1,18 @@
+user nginx;
+
+events {
+    worker_connections 1000;
+}
+
+http {
+    upstream pdps {
+        server apexpdp:6969;
+        server apexpdp:23324;
+    }
+    server {
+        listen 30001;
+        location / {
+            proxy_pass http://pdps;
+        }
+    }
+}
diff --git a/compose/docker-compose.pdp.scale.yml b/compose/docker-compose.pdp.scale.yml
new file mode 100644
index 00000000..53376ab2
--- /dev/null
+++ b/compose/docker-compose.pdp.scale.yml
@@ -0,0 +1,52 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+version: '2'
+services:
+  apexpdp:
+    image: ${CONTAINER_LOCATION}onap/policy-apex-pdp:${POLICY_APEX_PDP_VERSION}
+    depends_on:
+      - mariadb
+      - simulator
+      - pap
+    hostname: policy-apex-pdp
+    expose:
+      - 6969
+      - 23324
+    deploy:
+        mode: replicated
+        replicas: ${REPLICAS}
+    volumes:
+      - ./config/apex-pdp/OnapPfConfig.json:/opt/app/policy/apex-pdp/etc/onappf/config/OnapPfConfig.json:ro
+      - ./config/apex-pdp/logback.xml:/opt/app/policy/apex-pdp/etc/logback.xml:ro
+      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
+    entrypoint: /opt/app/policy/bin/wait_for_port.sh
+    command: [
+      '-c', '/opt/app/policy/apex-pdp/bin/apexOnapPf.sh -c /opt/app/policy/apex-pdp/etc/onappf/config/OnapPfConfig.json',
+      'mariadb', '3306',
+      'message-router', '3904',
+      'pap', '6969'
+      ]
+  nginx:
+    image: ${CONTAINER_LOCATION}library/nginx:latest
+    volumes:
+      - ./config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      - apexpdp
+    ports:
+       - ${APEX_PORT}:${APEX_PORT}
diff --git a/compose/metrics/prometheus.yml b/compose/metrics/prometheus.yml
index 789cfc2a..c3d87f57 100644
--- a/compose/metrics/prometheus.yml
+++ b/compose/metrics/prometheus.yml
@@ -24,78 +24,74 @@ global:
 # Alertmanager configuration
 alerting:
   alertmanagers:
-    - static_configs:
-        - targets:
-          # - alertmanager:9093
+  - static_configs:
+    - targets:
+      # - alertmanager:9093
 
 # scrape config
 scrape_configs:
 
-  - job_name: "api-metrics"
-    metrics_path: /policy/api/v1/metrics
-    static_configs:
-      - targets: ["policy-api:6969"]
-    basic_auth:
-      username: "policyadmin"
-      password: "zb!XztG34"
+- job_name: "api-metrics"
+  metrics_path: /policy/api/v1/metrics
+  static_configs:
+  - targets: ["policy-api:6969"]
+  basic_auth:
+    username: "policyadmin"
+    password: "zb!XztG34"
 
-  - job_name: "pap-metrics"
-    metrics_path: /policy/pap/v1/metrics
-    static_configs:
-      - targets: ["policy-pap:6969"]
-    basic_auth:
-      username: "policyadmin"
-      password: "zb!XztG34"
+- job_name: "pap-metrics"
+  metrics_path: /policy/pap/v1/metrics
+  static_configs:
+  - targets: ["policy-pap:6969"]
+  basic_auth:
+    username: "policyadmin"
+    password: "zb!XztG34"
 
-  - job_name: "apex-pdp-metrics"
-    static_configs:
-      - targets:
-          - "policy-apex-pdp:6969"
-    basic_auth:
-      username: "policyadmin"
-      password: "zb!XztG34"
+- job_name: "apex-pdp-metrics"
+  static_configs:
+  - targets:
+    - "policy-apex-pdp:6969"
+  basic_auth:
+    username: "policyadmin"
+    password: "zb!XztG34"
 
-  - job_name: "drools-apps-metrics"
-    static_configs:
-      - targets:
-          - "drools-apps:9696"
-    basic_auth:
-      username: "demo@people.osaaf.org"
-      password: "demo123456!"
+- job_name: "drools-apps-metrics"
+  static_configs:
+  - targets:
+    - "drools-apps:9696"
+  basic_auth:
+    username: "demo@people.osaaf.org"
+    password: "demo123456!"
 
-  - job_name: "drools-pdp-metrics"
-    static_configs:
-      - targets:
-          - "drools:9696"
-    basic_auth:
-      username: "demo@people.osaaf.org"
-      password: "demo123456!"
+- job_name: "drools-pdp-metrics"
+  static_configs:
+  - targets:
+    - "drools:9696"
+  basic_auth:
+    username: "demo@people.osaaf.org"
+    password: "demo123456!"
 
-  - job_name: "distribution-metrics"
-    static_configs:
-      - targets:
-          - "policy-distribution:6969"
-    basic_auth:
-      username: "policyadmin"
-      password: "zb!XztG34"
+- job_name: "distribution-metrics"
+  static_configs:
+  - targets:
+    - "policy-distribution:6969"
+  basic_auth:
+    username: "policyadmin"
+    password: "zb!XztG34"
 
-  - job_name: "xacml-pdp-metrics"
-    static_configs:
-      - targets:
-          - "policy-xacml-pdp:6969"
-    basic_auth:
-      username: "policyadmin"
-      password: "zb!XztG34"
+- job_name: "xacml-pdp-metrics"
+  static_configs:
+  - targets:
+    - "policy-xacml-pdp:6969"
+  basic_auth:
+    username: "policyadmin"
+    password: "zb!XztG34"
 
-  - job_name: "acm-metrics"
-    metrics_path: "/onap/policy/clamp/acm/prometheus"
-    static_configs:
-      - targets:
-          - "policy-clamp-runtime-acm:6969"
-    basic_auth:
-      username: "runtimeUser"
-      password: "zb!XztG34"
-
-  - job_name: "node"
-    static_configs:
-    - targets: ["node-exporter:9100"]
+- job_name: "acm-metrics"
+  metrics_path: "/onap/policy/clamp/acm/prometheus"
+  static_configs:
+  - targets:
+    - "policy-clamp-runtime-acm:6969"
+  basic_auth:
+    username: "runtimeUser"
+    password: "zb!XztG34"
diff --git a/compose/start-multiple-pdp.sh b/compose/start-multiple-pdp.sh
new file mode 100755
index 00000000..30572875
--- /dev/null
+++ b/compose/start-multiple-pdp.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# ============LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END======================================================
+
+if [ -z "${WORKSPACE}" ]; then
+    WORKSPACE=$(git rev-parse --show-toplevel)
+    export WORKSPACE
+fi
+COMPOSE_FOLDER="${WORKSPACE}"/compose
+
+cd ${COMPOSE_FOLDER}
+
+echo "Configuring docker compose..."
+source export-ports.sh > /dev/null 2>&1
+source get-versions.sh > /dev/null 2>&1
+
+export REPLICAS=${1}
+docker-compose -f docker-compose.yml -f docker-compose.pdp.scale.yml up -d apexpdp nginx grafana
+
+cd ${WORKSPACE}
\ No newline at end of file
diff --git a/compose/stop-compose.sh b/compose/stop-compose.sh
index 810c9fe0..a8185215 100755
--- a/compose/stop-compose.sh
+++ b/compose/stop-compose.sh
@@ -33,4 +33,4 @@ docker-compose -f "${COMPOSE_FOLDER}"/docker-compose.yml logs > docker_compose.l
 cat docker_compose.log
 
 echo "Tearing down containers..."
-docker-compose -f "${COMPOSE_FOLDER}"/docker-compose.yml down -v
+docker-compose -f "${COMPOSE_FOLDER}"/docker-compose.yml down -v --remove-orphans
diff --git a/csit/resources/scripts/setup-apex-pdp-large.sh b/csit/resources/scripts/setup-apex-pdp-large.sh
new file mode 100644
index 00000000..83d78841
--- /dev/null
+++ b/csit/resources/scripts/setup-apex-pdp-large.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+export PROJECT="apex-pdp"
+source "${SCRIPTS}"/node-templates.sh
+
+source "${WORKSPACE}"/compose/start-multiple-pdp.sh 10
+
+sleep 10
+unset http_proxy https_proxy
+
+# wait for the app to start up
+bash "${SCRIPTS}"/wait_for_rest.sh localhost ${PAP_PORT}
+bash "${SCRIPTS}"/wait_for_rest.sh localhost ${APEX_PORT}
+
+sleep 20
+
+healthy=false
+
+while [ $healthy = false ]
+do
+    msg=`curl -s -k --user 'policyadmin:zb!XztG34' http://localhost:${APEX_PORT}/policy/apex-pdp/v1/healthcheck`
+    echo "${msg}" | grep -q true
+    if [ "${?}" -eq 0 ]
+    then
+        healthy=true
+        break
+    fi
+    sleep 10s
+done
+
+export DMAAP_IP="localhost:${DMAAP_PORT}"
+export SUITES="apex-slas-10.robot"
+
+ROBOT_VARIABLES="-v POLICY_PAP_IP:localhost:${PAP_PORT} -v POLICY_API_IP:localhost:${API_PORT}
+-v PROMETHEUS_IP:localhost:${PROMETHEUS_PORT} -v DATA:${DATA} -v NODETEMPLATES:${NODETEMPLATES}
+-v APEX_IP:localhost:${APEX_PORT} -v DMAAP_IP:${DMAAP_IP}
+-v APEX_EVENTS_IP:localhost:${APEX_PORT}"
diff --git a/csit/resources/scripts/setup-apex-pdp-medium.sh b/csit/resources/scripts/setup-apex-pdp-medium.sh
new file mode 100644
index 00000000..e1f3776a
--- /dev/null
+++ b/csit/resources/scripts/setup-apex-pdp-medium.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+export PROJECT="apex-pdp"
+source "${SCRIPTS}"/node-templates.sh
+
+source "${WORKSPACE}"/compose/start-multiple-pdp.sh 3
+
+sleep 10
+unset http_proxy https_proxy
+
+# wait for the app to start up
+bash "${SCRIPTS}"/wait_for_rest.sh localhost ${PAP_PORT}
+bash "${SCRIPTS}"/wait_for_rest.sh localhost ${APEX_PORT}
+
+sleep 20
+
+healthy=false
+
+while [ $healthy = false ]
+do
+    msg=`curl -s -k --user 'policyadmin:zb!XztG34' http://localhost:${APEX_PORT}/policy/apex-pdp/v1/healthcheck`
+    echo "${msg}" | grep -q true
+    if [ "${?}" -eq 0 ]
+    then
+        healthy=true
+        break
+    fi
+    sleep 10s
+done
+
+export DMAAP_IP="localhost:${DMAAP_PORT}"
+export SUITES="apex-slas-3.robot"
+
+ROBOT_VARIABLES="-v POLICY_PAP_IP:localhost:${PAP_PORT} -v POLICY_API_IP:localhost:${API_PORT}
+-v PROMETHEUS_IP:localhost:${PROMETHEUS_PORT} -v DATA:${DATA} -v NODETEMPLATES:${NODETEMPLATES}
+-v APEX_IP:localhost:${APEX_PORT} -v DMAAP_IP:${DMAAP_IP}
+-v APEX_EVENTS_IP:localhost:${APEX_PORT}"
diff --git a/csit/resources/scripts/setup-apex-pdp.sh b/csit/resources/scripts/setup-apex-pdp.sh
index acebb284..b9b1a78a 100755
--- a/csit/resources/scripts/setup-apex-pdp.sh
+++ b/csit/resources/scripts/setup-apex-pdp.sh
@@ -24,10 +24,11 @@
 source "${SCRIPTS}"/setup-pap.sh
 
 # wait for the app to start up
-"${SCRIPTS}"/wait_for_rest.sh localhost "${APEX_PORT}"
+bash "${SCRIPTS}"/wait_for_rest.sh localhost ${APEX_PORT}
 
 export DMAAP_IP="localhost:${DMAAP_PORT}"
-export SUITES="apex-pdp-test.robot"
+export SUITES="apex-pdp-test.robot
+apex-slas.robot"
 
 ROBOT_VARIABLES="${ROBOT_VARIABLES} -v APEX_IP:localhost:${APEX_PORT} -v DMAAP_IP:${DMAAP_IP}
 -v APEX_EVENTS_IP:localhost:${APEX_EVENTS_PORT}"
diff --git a/csit/resources/tests/apex-pdp-common.robot b/csit/resources/tests/apex-pdp-common.robot
new file mode 100644
index 00000000..cf3f1f49
--- /dev/null
+++ b/csit/resources/tests/apex-pdp-common.robot
@@ -0,0 +1,64 @@
+*** Settings ***
+Library     Collections
+Library     RequestsLibrary
+Library     OperatingSystem
+Library     json
+Library     Process
+Resource    ${CURDIR}/common-library.robot
+
+*** Keywords ***
+
+DeployPolicy
+     [Documentation]    Deploy the policy in apex-pdp engine
+     ${postjson}=    Get File  ${CURDIR}/data/policy_deploy.json
+     ${postjson}=    evaluate    json.loads('''${postjson}''')    json
+     set to dictionary    ${postjson['groups'][0]['deploymentSubgroups'][0]['policies'][0]}    name=${policyName}
+     ${postjson}=    evaluate    json.dumps(${postjson})    json
+     ${policyadmin}=  PolicyAdminAuth
+     PerformPostRequest  ${POLICY_PAP_IP}  /policy/pap/v1/pdps/deployments/batch  202  ${postjson}  null  ${policyadmin}
+
+RunEventOnApexEngine
+    [Documentation]    Send event to verify policy execution
+    Create Session   apexSession  http://${APEX_EVENTS_IP}   max_retries=1
+    ${data}=    Get Binary File     ${CURDIR}/data/event.json
+    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
+    ${resp}=    PUT On Session    apexSession    /apex/FirstConsumer/EventIn    data=${data}   headers=${headers}
+    Should Be Equal As Strings    ${resp.status_code}   200
+
+VerifyPdpStatistics
+     [Documentation]    Verify pdp statistics after policy execution
+     [Arguments]    ${deployCount}    ${deploySuccessCount}    ${executedCount}    ${executedSuccessCount}
+     ${policyadmin}=  PolicyAdminAuth
+     ${resp}=  PerformGetRequest  ${POLICY_PAP_IP}  /policy/pap/v1/pdps/statistics/defaultGroup/apex/${pdpName}  200  null  ${policyadmin}
+     Should Be Equal As Strings    ${resp.status_code}     200
+     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['pdpInstanceId']}  ${pdpName}
+     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['pdpGroupName']}  defaultGroup
+     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['pdpSubGroupName']}  apex
+     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyDeployCount']}  ${deployCount}
+     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyDeploySuccessCount']}  ${deploySuccessCount}
+     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyDeployFailCount']}  0
+     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyExecutedCount']}  ${executedCount}
+     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyExecutedSuccessCount']}  ${executedSuccessCount}
+     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyExecutedFailCount']}  0
+
+CheckLogMessage
+    [Documentation]    Read log messages received and check for expected content.
+    [Arguments]    ${status}    ${expectedMsg}
+    ${result}=     CheckTopic     APEX-CL-MGT    ${status}
+    Should Contain    ${result}    ${expectedMsg}
+
+ValidatePolicyExecution
+    [Arguments]  ${url}  ${executionTime}
+    [Documentation]  Check that policy execution under X milliseconds
+    ${resp}=  QueryPrometheus  ${url}
+    ${rawNumber}=  Evaluate  ${resp['data']['result'][0]['value'][1]}
+    ${actualTime}=   Set Variable  ${rawNumber * ${1000}}
+    Should Be True   ${actualTime} <= ${executionTime}
+
+ValidateEventExecution
+    [Arguments]  ${eventStartTime}  ${eventEndTime}  ${eventsNo}
+    [Documentation]    Check that X amount of events were exeuted per second
+    ${eventTimeTaken}=  Subtract Date From Date  ${eventEndTime}  ${eventStartTime}
+    ${eventResult}=  Set Variable  ${eventTimeTaken * ${1000}}
+    ${eventsPerSecond}=  Set Variable  ${${1000} / ${eventResult}}
+    Should Be True  ${eventsPerSecond} >= ${eventsNo}
\ No newline at end of file
diff --git a/csit/resources/tests/apex-pdp-test.robot b/csit/resources/tests/apex-pdp-test.robot
index 57fc93b3..f30af62d 100644
--- a/csit/resources/tests/apex-pdp-test.robot
+++ b/csit/resources/tests/apex-pdp-test.robot
@@ -5,7 +5,7 @@ Library     OperatingSystem
 Library     json
 Library     Process
 Resource    ${CURDIR}/common-library.robot
-
+Resource    ${CURDIR}/apex-pdp-common.robot
 
 *** Test Cases ***
 
@@ -79,23 +79,6 @@ Metrics
 
 *** Keywords ***
 
-DeployPolicy
-     [Documentation]    Deploy the policy in apex-pdp engine
-     ${postjson}=    Get File  ${CURDIR}/data/policy_deploy.json
-     ${postjson}=    evaluate    json.loads('''${postjson}''')    json
-     set to dictionary    ${postjson['groups'][0]['deploymentSubgroups'][0]['policies'][0]}    name=${policyName}
-     ${postjson}=    evaluate    json.dumps(${postjson})    json
-     ${policyadmin}=  PolicyAdminAuth
-     PerformPostRequest  ${POLICY_PAP_IP}  /policy/pap/v1/pdps/deployments/batch  202  ${postjson}  null  ${policyadmin}
-
-RunEventOnApexEngine
-    [Documentation]    Send event to verify policy execution
-    Create Session   apexSession  http://${APEX_EVENTS_IP}   max_retries=1
-    ${data}=    Get Binary File     ${CURDIR}/data/event.json
-    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
-    ${resp}=    PUT On Session    apexSession    /apex/FirstConsumer/EventIn    data=${data}   headers=${headers}
-    Should Be Equal As Strings    ${resp.status_code}   200
-
 TriggerAndVerifyTestPnfPolicy
     [Documentation]    Send TestPnf policy trigger event to DMaaP and read notifications to verify policy execution
     Create Session   apexSession  http://${DMAAP_IP}   max_retries=1
@@ -118,25 +101,3 @@ TriggerAndVerifyTestVnfPolicy
     Run Keyword    CheckLogMessage    SUCCESS    VNF details are received from AAI successfully. Sending ConfigModify request to CDS.
     Run Keyword    CheckLogMessage    SUCCESS    ConfigModify request is successful. Sending restart request to CDS.
     Run Keyword    CheckLogMessage    FINAL_SUCCESS    Successfully processed the VES Event. Restart is complete.
-
-CheckLogMessage
-    [Documentation]    Read log messages received and check for expected content.
-    [Arguments]    ${status}    ${expectedMsg}
-    ${result}=     CheckTopic     APEX-CL-MGT    ${status}
-    Should Contain    ${result}    ${expectedMsg}
-
-VerifyPdpStatistics
-     [Documentation]    Verify pdp statistics after policy execution
-     [Arguments]    ${deployCount}    ${deploySuccessCount}    ${executedCount}    ${executedSuccessCount}
-     ${policyadmin}=  PolicyAdminAuth
-     ${resp}=  PerformGetRequest  ${POLICY_PAP_IP}  /policy/pap/v1/pdps/statistics/defaultGroup/apex/${pdpName}  200  null  ${policyadmin}
-     Should Be Equal As Strings    ${resp.status_code}     200
-     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['pdpInstanceId']}  ${pdpName}
-     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['pdpGroupName']}  defaultGroup
-     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['pdpSubGroupName']}  apex
-     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyDeployCount']}  ${deployCount}
-     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyDeploySuccessCount']}  ${deploySuccessCount}
-     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyDeployFailCount']}  0
-     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyExecutedCount']}  ${executedCount}
-     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyExecutedSuccessCount']}  ${executedSuccessCount}
-     Should Be Equal As Strings    ${resp.json()['defaultGroup']['apex'][0]['policyExecutedFailCount']}  0
diff --git a/csit/resources/tests/apex-slas-10.robot b/csit/resources/tests/apex-slas-10.robot
new file mode 100644
index 00000000..7639ce95
--- /dev/null
+++ b/csit/resources/tests/apex-slas-10.robot
@@ -0,0 +1,79 @@
+*** Settings ***
+Library    Collections
+Library    RequestsLibrary
+Library    OperatingSystem
+Library    DateTime
+Library    Process
+Library    json
+Resource    ${CURDIR}/common-library.robot
+Resource    ${CURDIR}/apex-pdp-common.robot
+
+*** Test Cases ***
+Healthcheck
+     [Documentation]    Runs Apex PDP Health check
+     ${hcauth}=  PolicyAdminAuth
+     ${resp}=  PerformGetRequest  ${APEX_IP}  /policy/apex-pdp/v1/healthcheck  200  null  ${hcauth}
+     Should Be Equal As Strings    ${resp.json()['code']}    200
+     Set Suite Variable    ${pdpName}    ${resp.json()['name']}
+
+ValidatePolicyExecutionAndEventRateLowComplexity
+    [Documentation]  Validate that a moderate complexity policity can be executed in less than 100ms and minimum 100 events triggered per second
+    Set Test Variable    ${policyName}    onap.policies.apex.pnf.metadataSet.Test
+    ${postjson}=  Get File  ${CURDIR}/data/${policyName}.json
+    CreatePolicy  /policy/api/v1/policytypes/onap.policies.native.Apex/versions/1.0.0/policies  200  ${postjson}  ${policyName}  1.0.0
+    ${postjson}=  Get File  ${CURDIR}/data/onap.pnf.metadataSet.Test.json
+    CreateNodeTemplate  /policy/api/v1/nodetemplates  200  ${postjson}  1
+    DeployPolicy
+    Wait Until Keyword Succeeds    2 min    5 sec    QueryPolicyStatus  ${policyName}  defaultGroup  apex  ${pdpName}  onap.policies.native.Apex
+    GetTopic     APEX-CL-MGT2
+    Create Session   apexSession  http://${DMAAP_IP}   max_retries=1
+    ${data}=    Get Binary File     ${CURDIR}/data/VesEventForVnfPolicy.json
+    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
+    ${eventStartTime}=  Get Current Date
+    ${resp}=    POST On Session    apexSession    /events/unauthenticated.DCAE_POLICY_EXAMPLE_OUTPUT    data=${data}   headers=${headers}
+    ${eventEndTime}=  Get Current Date
+    Should Be Equal As Strings    ${resp.status_code}   200
+    ValidateEventExecution    ${eventStartTime}  ${eventEndTime}  100
+
+ValidatePolicyExecutionAndEventRateHighComplexity
+    [Documentation]  Validate that a high complexity policity can be executed in less than 5000ms and minimum 2 events triggered per second
+    Set Test Variable    ${policyName}    onap.policies.native.apex.Sampledomain
+    ${postjson}=  Get File  ${CURDIR}/data/${policyName}.json
+    CreatePolicy  /policy/api/v1/policytypes/onap.policies.native.Apex/versions/1.0.0/policies  200  ${postjson}  ${policyName}  1.0.0
+    DeployPolicy
+    Create Session   apexSession  http://${APEX_EVENTS_IP}   max_retries=1
+    ${data}=    Get Binary File     ${CURDIR}/data/event.json
+    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
+    Sleep  60s
+    ${eventStartTime}=  Get Current Date
+    ${resp}=    PUT On Session    apexSession    /apex/FirstConsumer/EventIn    data=${data}   headers=${headers}
+    ${eventEndTime}=  Get Current Date
+    Should Be Equal As Strings    ${resp.status_code}   200
+    ValidateEventExecution    ${eventStartTime}  ${eventEndTime}  2
+
+ValidatePolicyExecutionAndEventRateModerateComplexity
+    [Documentation]  Validate that a low complexity policity can be executed in less than 1000ms and minimum 10 events triggered per second
+    Set Test Variable    ${policyName}    onap.policies.apex.pnf.Test
+    ${postjson}=  Get File  ${CURDIR}/data/${policyName}.json
+    CreatePolicy  /policy/api/v1/policytypes/onap.policies.native.Apex/versions/1.0.0/policies  200  ${postjson}  ${policyName}  1.0.0
+    DeployPolicy
+    Wait Until Keyword Succeeds    2 min    5 sec    QueryPolicyStatus  ${policyName}  defaultGroup  apex  ${pdpName}  onap.policies.native.Apex
+    GetTopic     APEX-CL-MGT
+    Create Session   apexSession  http://${DMAAP_IP}   max_retries=1
+    ${data}=    Get Binary File     ${CURDIR}/data/VesEventForPnfPolicy.json
+    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
+    ${eventStartTime}=  Get Current Date
+    ${resp}=    POST On Session    apexSession    /events/unauthenticated.DCAE_CL_OUTPUT    data=${data}   headers=${headers}
+    ${eventEndTime}=  Get Current Date
+    Should Be Equal As Strings    ${resp.status_code}   200
+    ValidateEventExecution    ${eventStartTime}  ${eventEndTime}  10
+
+WaitForPrometheusServer
+    [Documentation]  Sleep time to wait for Prometheus server to gather all metrics
+    Sleep    2 minutes
+
+ValidatePolicyExecutionTimes
+    [Documentation]    Validate policy execution times using prometheus metrics
+    ValidatePolicyExecution   pdpa_engine_average_execution_time_seconds{engine_instance_id="NSOApexEngine-0:0.0.1", instance="policy-apex-pdp:6969", job="apex-pdp-metrics"}  5000
+    ValidatePolicyExecution   pdpa_engine_average_execution_time_seconds{engine_instance_id="MyApexEngine-0:0.0.1"}  1000
+    ValidatePolicyExecution   pdpa_engine_average_execution_time_seconds{engine_instance_id="NSOApexEngine-1:0.0.1", instance="policy-apex-pdp:6969", job="apex-pdp-metrics"}  100
diff --git a/csit/resources/tests/apex-slas-3.robot b/csit/resources/tests/apex-slas-3.robot
new file mode 100644
index 00000000..13d58957
--- /dev/null
+++ b/csit/resources/tests/apex-slas-3.robot
@@ -0,0 +1,80 @@
+*** Settings ***
+Library    Collections
+Library    RequestsLibrary
+Library    OperatingSystem
+Library    DateTime
+Library    Process
+Library    json
+Resource    ${CURDIR}/common-library.robot
+Resource    ${CURDIR}/apex-pdp-common.robot
+
+*** Test Cases ***
+Healthcheck
+     [Documentation]    Runs Apex PDP Health check
+     ${hcauth}=  PolicyAdminAuth
+     ${resp}=  PerformGetRequest  ${APEX_IP}  /policy/apex-pdp/v1/healthcheck  200  null  ${hcauth}
+     Should Be Equal As Strings    ${resp.json()['code']}    200
+     Set Suite Variable    ${pdpName}    ${resp.json()['name']}
+
+ValidatePolicyExecutionAndEventRateLowComplexity
+    [Documentation]  Validate that a moderate complexity policity can be executed in less than 100ms and minimum 30 events triggered per second
+    Set Test Variable    ${policyName}    onap.policies.apex.pnf.Test
+    ${postjson}=  Get File  ${CURDIR}/data/${policyName}.json
+    CreatePolicy  /policy/api/v1/policytypes/onap.policies.native.Apex/versions/1.0.0/policies  200  ${postjson}  ${policyName}  1.0.0
+    DeployPolicy
+    Wait Until Keyword Succeeds    2 min    5 sec    QueryPolicyStatus  ${policyName}  defaultGroup  apex  ${pdpName}  onap.policies.native.Apex
+    GetTopic     APEX-CL-MGT
+    Create Session   apexSession  http://${DMAAP_IP}   max_retries=1
+    ${data}=    Get Binary File     ${CURDIR}/data/VesEventForPnfPolicy.json
+    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
+    ${eventStartTime}=  Get Current Date
+    ${resp}=    POST On Session    apexSession    /events/unauthenticated.DCAE_CL_OUTPUT    data=${data}   headers=${headers}
+    Should Be Equal As Strings    ${resp.status_code}   200
+    ${eventEndTime}=  Get Current Date
+    ValidateEventExecution    ${eventStartTime}  ${eventEndTime}  30
+
+ValidatePolicyExecutionAndEventRateHighComplexity
+    [Documentation]  Validate that a high complexity policity can be executed in less than 5000ms and minimum 0.6 events triggered per second
+    Set Test Variable    ${policyName}    onap.policies.apex.pnf.metadataSet.Test
+    ${postjson}=  Get File  ${CURDIR}/data/${policyName}.json
+    CreatePolicy  /policy/api/v1/policytypes/onap.policies.native.Apex/versions/1.0.0/policies  200  ${postjson}  ${policyName}  1.0.0
+    ${postjson}=  Get File  ${CURDIR}/data/onap.pnf.metadataSet.Test.json
+    CreateNodeTemplate  /policy/api/v1/nodetemplates  200  ${postjson}  1
+    DeployPolicy
+    Wait Until Keyword Succeeds    2 min    5 sec    QueryPolicyStatus  ${policyName}  defaultGroup  apex  ${pdpName}  onap.policies.native.Apex
+    GetTopic     APEX-CL-MGT2
+    Create Session   apexSession  http://${DMAAP_IP}   max_retries=1
+    ${data}=    Get Binary File     ${CURDIR}/data/VesEventForVnfPolicy.json
+    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
+    ${eventStartTime}=  Get Current Date
+    ${resp}=    POST On Session    apexSession    /events/unauthenticated.DCAE_POLICY_EXAMPLE_OUTPUT    data=${data}   headers=${headers}
+    Should Be Equal As Strings    ${resp.status_code}   200
+    ${eventEndTime}=  Get Current Date
+    ValidateEventExecution    ${eventStartTime}  ${eventEndTime}  0.6
+
+ValidatePolicyExecutionAndEventRateModerateComplexity
+    [Documentation]  Validate that a low complexity policity can be executed in less than 1000ms and minimum 3 events triggered per second
+    Set Test Variable    ${policyName}    onap.policies.native.apex.Sampledomain
+    ${postjson}=  Get File  ${CURDIR}/data/${policyName}.json
+    CreatePolicy  /policy/api/v1/policytypes/onap.policies.native.Apex/versions/1.0.0/policies  200  ${postjson}  ${policyName}  1.0.0
+    DeployPolicy
+    #Wait Until Keyword Succeeds    4 min    5 sec    RunEventOnApexEngine
+    Create Session   apexSession  http://${APEX_EVENTS_IP}   max_retries=1
+    ${data}=    Get Binary File     ${CURDIR}/data/event.json
+    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
+    Sleep  60s
+    ${eventStartTime}=  Get Current Date
+    ${resp}=    PUT On Session    apexSession    /apex/FirstConsumer/EventIn    data=${data}   headers=${headers}
+    Should Be Equal As Strings    ${resp.status_code}   200
+    ${eventEndTime}=  Get Current Date
+    ValidateEventExecution    ${eventStartTime}  ${eventEndTime}  3
+
+WaitForPrometheusServer
+    [Documentation]  Sleep time to wait for Prometheus server to gather all metrics
+    Sleep    2 minutes
+
+ValidatePolicyExecutionTimes
+    [Documentation]    Validate policy execution times using prometheus metrics
+    ValidatePolicyExecution   pdpa_engine_average_execution_time_seconds{engine_instance_id="NSOApexEngine-0:0.0.1", instance="policy-apex-pdp:6969", job="apex-pdp-metrics"}  5000
+    ValidatePolicyExecution   pdpa_engine_average_execution_time_seconds{engine_instance_id="MyApexEngine-0:0.0.1"}  1000
+    ValidatePolicyExecution   pdpa_engine_average_execution_time_seconds{engine_instance_id="NSOApexEngine-1:0.0.1", instance="policy-apex-pdp:6969", job="apex-pdp-metrics"}  100
diff --git a/csit/resources/tests/apex-slas.robot b/csit/resources/tests/apex-slas.robot
new file mode 100644
index 00000000..408b0add
--- /dev/null
+++ b/csit/resources/tests/apex-slas.robot
@@ -0,0 +1,74 @@
+*** Settings ***
+Library    Collections
+Library    RequestsLibrary
+Library    OperatingSystem
+Library    DateTime
+Library    Process
+Library    json
+Resource    ${CURDIR}/common-library.robot
+Resource    ${CURDIR}/apex-pdp-common.robot
+
+*** Test Cases ***
+Healthcheck
+     [Documentation]    Runs Apex PDP Health check
+     ${hcauth}=  PolicyAdminAuth
+     ${resp}=  PerformGetRequest  ${APEX_IP}  /policy/apex-pdp/v1/healthcheck  200  null  ${hcauth}
+     Should Be Equal As Strings    ${resp.json()['code']}    200
+     Set Suite Variable    ${pdpName}    ${resp.json()['name']}
+
+ValidatePolicyExecutionAndEventRateLowComplexity
+    [Documentation]  Validate that a moderate complexity policity can be executed in less than 100ms and minimum 10 events triggered per second
+    Set Test Variable    ${policyName}    onap.policies.apex.pnf.Test
+    ${postjson}=  Get File  ${CURDIR}/data/${policyName}.json
+    CreatePolicy  /policy/api/v1/policytypes/onap.policies.native.Apex/versions/1.0.0/policies  200  ${postjson}  ${policyName}  1.0.0
+    DeployPolicy
+    Wait Until Keyword Succeeds    2 min    5 sec    QueryPolicyStatus  ${policyName}  defaultGroup  apex  ${pdpName}  onap.policies.native.Apex
+    GetTopic     APEX-CL-MGT
+    Create Session   apexSession  http://${DMAAP_IP}   max_retries=1
+    ${data}=    Get Binary File     ${CURDIR}/data/VesEventForPnfPolicy.json
+    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
+    ${eventStartTime}=  Get Current Date
+    ${resp}=    POST On Session    apexSession    /events/unauthenticated.DCAE_CL_OUTPUT    data=${data}   headers=${headers}
+    Should Be Equal As Strings    ${resp.status_code}   200
+    ${eventEndTime}=  Get Current Date
+    ValidateEventExecution    ${eventStartTime}  ${eventEndTime}  10
+
+ValidatePolicyExecutionAndEventRateModerateComplexity
+    [Documentation]  Validate that a low complexity policity can be executed in less than 1000ms and minimum 1 events triggered per second
+    Set Test Variable    ${policyName}    onap.policies.native.apex.Sampledomain
+    ${postjson}=  Get File  ${CURDIR}/data/${policyName}.json
+    CreatePolicy  /policy/api/v1/policytypes/onap.policies.native.Apex/versions/1.0.0/policies  200  ${postjson}  ${policyName}  1.0.0
+    DeployPolicy
+    ${eventStartTime}=  Get Current Date
+    Wait Until Keyword Succeeds    4 min    5 sec    RunEventOnApexEngine
+    ${eventEndTime}=  Get Current Date
+    ValidateEventExecution    ${eventStartTime}  ${eventEndTime}  1
+
+ValidatePolicyExecutionAndEventRateHighComplexity
+    [Documentation]  Validate that a high complexity policity can be executed in less than 5000ms and minimum 0.2 events triggered per second
+    Set Test Variable    ${policyName}    onap.policies.apex.pnf.metadataSet.Test
+    ${postjson}=  Get File  ${CURDIR}/data/${policyName}.json
+    CreatePolicy  /policy/api/v1/policytypes/onap.policies.native.Apex/versions/1.0.0/policies  200  ${postjson}  ${policyName}  1.0.0
+    ${postjson}=  Get File  ${CURDIR}/data/onap.pnf.metadataSet.Test.json
+    CreateNodeTemplate  /policy/api/v1/nodetemplates  200  ${postjson}  1
+    DeployPolicy
+    Wait Until Keyword Succeeds    2 min    5 sec    QueryPolicyStatus  ${policyName}  defaultGroup  apex  ${pdpName}  onap.policies.native.Apex
+    GetTopic     APEX-CL-MGT2
+    Create Session   apexSession  http://${DMAAP_IP}   max_retries=1
+    ${data}=    Get Binary File     ${CURDIR}/data/VesEventForVnfPolicy.json
+    &{headers}=  Create Dictionary    Content-Type=application/json    Accept=application/json
+    ${eventStartTime}=  Get Current Date
+    ${resp}=    POST On Session    apexSession    /events/unauthenticated.DCAE_POLICY_EXAMPLE_OUTPUT    data=${data}   headers=${headers}
+    Should Be Equal As Strings    ${resp.status_code}   200
+    ${eventEndTime}=  Get Current Date
+    ValidateEventExecution    ${eventStartTime}  ${eventEndTime}  0.2
+
+WaitForPrometheusServer
+    [Documentation]  Sleep time to wait for Prometheus server to gather all metrics
+    Sleep    1 minute
+
+ValidatePolicyExecutionTimes
+    [Documentation]    Validate policy execution times using prometheus metrics
+    ValidatePolicyExecution   pdpa_engine_average_execution_time_seconds{engine_instance_id="NSOApexEngine-0:0.0.1", instance="policy-apex-pdp:6969", job="apex-pdp-metrics"}  5000
+    ValidatePolicyExecution   pdpa_engine_average_execution_time_seconds{engine_instance_id="MyApexEngine-0:0.0.1"}  1000
+    ValidatePolicyExecution   pdpa_engine_average_execution_time_seconds{engine_instance_id="NSOApexEngine-1:0.0.1", instance="policy-apex-pdp:6969", job="apex-pdp-metrics"}  100
diff --git a/csit/resources/tests/common-library.robot b/csit/resources/tests/common-library.robot
index c43b5dca..8c279176 100644
--- a/csit/resources/tests/common-library.robot
+++ b/csit/resources/tests/common-library.robot
@@ -100,7 +100,6 @@ QueryPolicyStatus
     Should Be Equal As Strings    ${resp.status_code}     200
     Should Be Equal As Strings    ${responseEntry['pdpGroup']}  ${pdpGroup}
     Should Be Equal As Strings    ${responseEntry['pdpType']}  ${pdpType}
-    Should Be Equal As Strings    ${responseEntry['pdpId']}  ${pdpName}
     Should Be Equal As Strings    ${responseEntry['policy']['name']}  ${policyName}
     Should Be Equal As Strings    ${responseEntry['policy']['version']}  1.0.0
     Should Be Equal As Strings    ${responseEntry['policyType']['name']}  ${policyTypeName}
diff --git a/csit/run-k8s-csit-enable.sh b/csit/run-k8s-csit-enable.sh
index 5eacc62a..c6de8c0e 100755
--- a/csit/run-k8s-csit-enable.sh
+++ b/csit/run-k8s-csit-enable.sh
@@ -29,9 +29,9 @@ fi
 CSIT_SCRIPT="scripts/run-test.sh"
 ROBOT_DOCKER_IMAGE="policy-csit-robot"
 POLICY_CLAMP_ROBOT="policy-clamp-test.robot"
-POLICY_API_ROBOT="api-test.robot"
+POLICY_API_ROBOT="api-test.robot api-slas.robot"
 POLICY_PAP_ROBOT="pap-test.robot pap-slas.robot"
-POLICY_APEX_PDP_ROBOT="apex-pdp-test.robot"
+POLICY_APEX_PDP_ROBOT="apex-pdp-test.robot apex-slas.robot"
 POLICY_XACML_PDP_ROBOT="xacml-pdp-test.robot"
 POLICY_DROOLS_PDP_ROBOT="drools-pdp-test.robot"
 POLICY_DISTRIBUTION_ROBOT="distribution-test.robot"
diff --git a/csit/run-project-csit.sh b/csit/run-project-csit.sh
index 47ad3b15..32f80b63 100755
--- a/csit/run-project-csit.sh
+++ b/csit/run-project-csit.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash
 #
 # Copyright 2016-2017 Huawei Technologies Co., Ltd.
 # Modification Copyright 2019 © Samsung Electronics Co., Ltd.