From 9fc9fa480d5b18b0988b9388d13db8328032fc52 Mon Sep 17 00:00:00 2001 From: "adheli.tavares" Date: Thu, 9 Feb 2023 11:56:27 +0000 Subject: Add SLAs validations to docker csit robot tests Issue-ID: POLICY-4529 Change-Id: I197a51892e9135d20c2e8a49ce7787df3bfbd4fa Signed-off-by: adheli.tavares --- .gitignore | 1 + csit/common-library.robot | 19 +++++++++++--- csit/pap/plans/setup.sh | 9 ------- csit/pap/plans/testplan.txt | 1 + csit/pap/tests/pap-slas.robot | 60 +++++++++++++++++++++++++++++++++++++++++++ csit/pap/tests/pap-test.robot | 25 ++++++++++++++++++ csit/run-project-csit.sh | 18 +++++-------- csit/stop-all.sh | 2 +- csit/wait_for_port.sh | 4 +-- csit/wait_for_rest.sh | 2 +- 10 files changed, 113 insertions(+), 28 deletions(-) create mode 100644 csit/pap/tests/pap-slas.robot diff --git a/.gitignore b/.gitignore index d4aa5c29..fee07ae7 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ env.properties .pydevproject models .vscode/ +**/*.log diff --git a/csit/common-library.robot b/csit/common-library.robot index e8a9640c..871e7efe 100644 --- a/csit/common-library.robot +++ b/csit/common-library.robot @@ -120,7 +120,18 @@ GetMetrics Log Received response from policy ${resp.text} [return] ${resp} -QueryPrometheus ${query} - ${resp}= GET http://localhost:30259/api/v1/query?query=${query} expected_status=200 - Log Received response from policy ${resp.text} - [return] ${resp} +QueryPrometheus + [Arguments] ${query} + ${params}= Create Dictionary query=${query} + ${resp}= GET http://localhost:30259/api/v1/query ${params} + Status Should Be OK + Log Received response from Prometheus ${resp.text} + [return] ${resp.json()} + +ValidateResponseTime + [Arguments] ${job} ${uri} ${method} ${timeLimit} + [Documentation] Check if uri response is under the required time + ${resp}= QueryPrometheus http_server_requests_seconds_sum{uri="${uri}",method="${method}",job="${job}"}/http_server_requests_seconds_count{uri="${uri}",method="${method}",job="${job}"} + ${rawNumber}= Evaluate ${resp['data']['result'][0]['value'][1]} + ${actualTime}= Set Variable ${rawNumber * ${1000}} + Should Be True ${actualTime} <= ${timeLimit} diff --git a/csit/pap/plans/setup.sh b/csit/pap/plans/setup.sh index b656ef5a..4c77accf 100755 --- a/csit/pap/plans/setup.sh +++ b/csit/pap/plans/setup.sh @@ -37,15 +37,8 @@ docker-compose -f "${SCRIPTS}"/docker-compose-all.yml up -d pap apex-pdp grafana sleep 10 unset http_proxy https_proxy -POLICY_PAP_IP=$(get-instance-ip.sh policy-pap) POLICY_PAP_PORT=30442 -POLICY_API_IP=$(get-instance-ip.sh policy-api) POLICY_API_PORT=30440 -MARIADB_IP=$(get-instance-ip.sh mariadb) - -echo PAP IP IS "${POLICY_PAP_IP}" -echo API IP IS "${POLICY_API_IP}" -echo MARIADB IP IS "${MARIADB_IP}" # wait for the app to start up "${SCRIPTS}"/wait_for_rest.sh localhost "${POLICY_PAP_PORT}" @@ -55,8 +48,6 @@ DATA=${WORKSPACE}/models/models-examples/src/main/resources/policies NODETEMPLATES=${WORKSPACE}/models/models-examples/src/main/resources/nodetemplates ROBOT_VARIABLES="" -ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_PAP_IP:${POLICY_PAP_IP}" -ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_API_IP:${POLICY_API_IP}" ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_PAP_PORT:${POLICY_PAP_PORT}" ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_API_PORT:${POLICY_API_PORT}" ROBOT_VARIABLES="${ROBOT_VARIABLES} -v DATA:${DATA}" diff --git a/csit/pap/plans/testplan.txt b/csit/pap/plans/testplan.txt index 1e5b1f94..4fac1027 100644 --- a/csit/pap/plans/testplan.txt +++ b/csit/pap/plans/testplan.txt @@ -1,3 +1,4 @@ # Test suites are relative paths under [policy/docker.git]/csit/[project]/tests. # Place the suites in run order. pap-test.robot +pap-slas.robot diff --git a/csit/pap/tests/pap-slas.robot b/csit/pap/tests/pap-slas.robot new file mode 100644 index 00000000..045530ae --- /dev/null +++ b/csit/pap/tests/pap-slas.robot @@ -0,0 +1,60 @@ +*** Settings *** +Library Collections +Library RequestsLibrary +Library OperatingSystem +Library json +Resource ${CURDIR}/../../common-library.robot + +*** Keywords *** +ValidateResponseTimeForPap + [Arguments] ${uri} ${method} + [Documentation] Check if uri response is under the 500ms required time for pap metrics + ValidateResponseTime pap-metrics ${uri} ${method} 500 + +*** Test Cases *** +WaitForPrometheusServer + [Documentation] Sleep time to wait for Prometheus server to gather all metrics + Sleep 1 minute + +ValidateResponseTimeForHealthcheck + [Documentation] Validate component healthcheck response time + ValidateResponseTimeForPap /healthcheck GET + +ValidateResponseTimeForSystemHealthcheck + [Documentation] Validate if system healthcheck response time is under 1000ms + ValidateResponseTime pap-metrics /components/healthcheck GET 10000 + +ValidateResponseTimeForStatistics + [Documentation] Validate statistics response time + ValidateResponseTimeForPap /statistics GET + +# TODO: includes notification, so always over 500ms +# ValidateResponseTimeCreateGroup +# [Documentation] Validate create group response time +# ValidateResponseTimeForPap /pdps/groups/batch POST + +ValidateResponseTimeQueryPolicyAudit + [Documentation] Validate query audits response time + ValidateResponseTimeForPap /policies/audit GET + +ValidateResponseTimeUpdateGroup + [Documentation] Validate pdps/group response time + ValidateResponseTimeForPap /pdps/groups/{name} PUT + +ValidatePolicyDeploymentTime + [Documentation] Check if deployment of policy is under 2000ms + ${resp}= QueryPrometheus pap_policy_deployments_seconds_sum{operation="deploy",status="SUCCESS"}/pap_policy_deployments_seconds_count{operation="deploy",status="SUCCESS"} + ${rawNumber}= Evaluate ${resp['data']['result'][0]['value'][1]} + ${actualTime}= Set Variable ${rawNumber * ${1000}} + Should Be True ${actualTime} <= ${2000} + +ValidateResponseTimeDeletePolicy + [Documentation] Check if undeployment of policy is under 2000ms + ${resp}= QueryPrometheus pap_policy_deployments_seconds_sum{operation="undeploy",status="SUCCESS"}/pap_policy_deployments_seconds_count{operation="undeploy",status="SUCCESS"} + ${rawNumber}= Evaluate ${resp['data']['result'][0]['value'][1]} + ${actualTime}= Set Variable ${rawNumber * ${1000}} + Should Be True ${actualTime} <= ${2000} + +ValidateResponseTimeDeleteGroup + [Documentation] Validate delete group response time + ValidateResponseTimeForPap /pdps/groups/{name} DELETE diff --git a/csit/pap/tests/pap-test.robot b/csit/pap/tests/pap-test.robot index c417a83f..a4f0853d 100644 --- a/csit/pap/tests/pap-test.robot +++ b/csit/pap/tests/pap-test.robot @@ -12,6 +12,18 @@ GetReq ${resp}= PerformGetRequest ${POLICY_PAP_PORT} ${url} 200 null ${auth} [return] ${resp} +ValidateResponseTimeForPap + [Arguments] ${uri} ${method} + [Documentation] Check if uri response is under the required time for pap metrics + ValidateResponseTime pap-metrics ${uri} ${method} 500 + +ValidateDeploymentTime + [Documentation] Check if deployment of policy is under 2000ms + ${resp}= QueryPrometheus pap_policy_deployments_seconds_sum{operation="deploy",status="SUCCESS"}/pap_policy_deployments_seconds_count{operation="deploy",status="SUCCESS"} + ${rawNumber}= Evaluate ${resp['data']['result'][0]['value'][1]} + ${actualTime}= Set Variable ${rawNumber * ${1000}} + Should Be True ${actualTime} <= ${2000} + *** Test Cases *** LoadPolicy [Documentation] Create a policy named 'onap.restart.tca' and version '1.0.0' using specific api @@ -126,3 +138,16 @@ DeletePdpGroups QueryPdpGroupsAfterDelete [Documentation] Verify PdpGroups after delete QueryPdpGroups 1 defaultGroup ACTIVE 0 null null null + +# ValidateSlaForPap +# [Documentation] Run checks against Prometheus server to check response time +# Sleep 30s +# ValidateDeploymentTime +# ValidateResponseTime pap-metrics /components/healthcheck GET 10000 +# ValidateResponseTimeForPap /healthcheck GET +# ValidateResponseTimeForPap /statistics GET +# ValidateResponseTimeForPap /policies/audit GET +# ValidateResponseTimeForPap /pdps/groups/{name} PUT +# ValidateResponseTimeForPap /pdps/policies/{name} DELETE +# ValidateResponseTimeForPap /pdps/groups/{name} DELETE +# ValidateResponseTimeForPap /pdps/groups/batch POST diff --git a/csit/run-project-csit.sh b/csit/run-project-csit.sh index a3837b03..0f15c75b 100755 --- a/csit/run-project-csit.sh +++ b/csit/run-project-csit.sh @@ -27,17 +27,18 @@ function on_exit(){ rc=$? if [[ ${WORKSPACE} ]]; then - if [[ ${WORKDIR} ]]; then - rsync -av "${WORKDIR}/" "${WORKSPACE}/csit/archives/${PROJECT}" - fi # Record list of active docker containers - docker ps + docker ps --format "table {{ .Names }}\t{{ .Status }}" # Show the logs from all containers - docker-compose -f "${WORKSPACE}/csit/docker-compose-all.yml" logs + docker-compose -f "${WORKSPACE}/csit/docker-compose-all.yml" logs > docker_compose.log # show memory consumption after all docker instances initialized docker_stats + + if [[ ${WORKDIR} ]]; then + rsync -av "${WORKDIR}/" "${WORKSPACE}/csit/archives/${PROJECT}" + fi fi # Run teardown script plan if it exists cd "${TESTPLANDIR}/plans/" @@ -57,25 +58,20 @@ function docker_stats(){ # General memory details if [ "$(uname -s)" == "Darwin" ] then - echo "> top -l1 | head -10" sh -c "top -l1 | head -10" echo else - echo "> top -bn1 | head -3" sh -c "top -bn1 | head -3" echo - echo "> free -h" sh -c "free -h" echo fi # Memory details per Docker - echo "> docker ps" - docker ps + docker ps --format "table {{ .Names }}\t{{ .Status }}" echo - echo "> docker stats --no-stream" docker stats --no-stream echo } diff --git a/csit/stop-all.sh b/csit/stop-all.sh index 4e6b0aa6..9adfcf3c 100755 --- a/csit/stop-all.sh +++ b/csit/stop-all.sh @@ -23,4 +23,4 @@ export SCRIPTS="${SCRIPTS}"/csit source "${SCRIPTS}"/get-versions.sh -docker-compose -f "${SCRIPTS}"/docker-compose-all.yml down +docker-compose -f "${SCRIPTS}"/docker-compose-all.yml down -v diff --git a/csit/wait_for_port.sh b/csit/wait_for_port.sh index 5a8be250..6dcb3cab 100755 --- a/csit/wait_for_port.sh +++ b/csit/wait_for_port.sh @@ -1,7 +1,7 @@ #!/bin/sh # ============LICENSE_START==================================================== # Copyright (C) 2021 AT&T Intellectual Property. All rights reserved. -# Modifications Copyright (C) 2022 Nordix Foundation. +# Modifications Copyright (C) 2022-2023 Nordix Foundation. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -64,7 +64,7 @@ do do if command -v docker > /dev/null 2>&1 then - docker ps + docker ps --format "table {{ .Names }}\t{{ .Status }}" fi nc -vz "$host" "$port" diff --git a/csit/wait_for_rest.sh b/csit/wait_for_rest.sh index bf47ccf1..a64c3063 100755 --- a/csit/wait_for_rest.sh +++ b/csit/wait_for_rest.sh @@ -54,7 +54,7 @@ do do if command -v docker > /dev/null 2>&1 then - docker ps + docker ps --format "table {{ .Names }}\t{{ .Status }}" fi curl "http://$host:$port" > /dev/null 2>&1 rc=$? -- cgit 1.2.3-korg