Add test suit for SDC helm chart validator

Issue-ID: SDC-3185
Signed-off-by: Joanna Jeremicz <joanna.jeremicz@nokia.com>
Change-Id: Id53318a912766ff8bd76637614f65533dfd62ed6

7 files changed, 313 insertions, 0 deletions

diff --git a/plans/sdc/sdc-helm-validator/setup.sh b/plans/sdc/sdc-helm-validator/setup.sh
new file mode 100644
index 00000000..6d079679
--- /dev/null
+++ b/plans/sdc/sdc-helm-validator/setup.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+export SDC_HELM_VALIDATOR="sdc-helm-validator"
+export SDC_HELM_VALIDATOR_IMAGE="nexus3.onap.org:10001/onap/org.onap.sdc.sdc-helm-validator:latest"
+
+unset http_proxy
+unset https_proxy
+
+docker run -p 8080:8080 -d --name ${SDC_HELM_VALIDATOR} ${SDC_HELM_VALIDATOR_IMAGE}
+
+# Wait container ready
+HELM_VALIDATOR_IP='none'
+for i in {1..5}
+do
+  HELM_VALIDATOR_IP=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${SDC_HELM_VALIDATOR})
+  RESP_CODE=$(curl -s http://${HELM_VALIDATOR_IP}:8080/actuator/health | python2 -c 'import json,sys;obj=json.load(sys.stdin);print obj["status"]')
+   if [[ "$RESP_CODE" == "UP" ]]; then
+       echo 'SDC Helm Validator is ready'
+       break
+   fi
+
+  echo 'Waiting for SDC Helm Validator to start up...'
+  sleep 10s
+done
+
+echo HELM_VALIDATOR_IP=${HELM_VALIDATOR_IP}
+
+# Pass any variables required by Robot test suites in ROBOT_VARIABLES
+ROBOT_VARIABLES="-v VALIDATOR:${HELM_VALIDATOR_IP}:8080"
diff --git a/plans/sdc/sdc-helm-validator/teardown.sh b/plans/sdc/sdc-helm-validator/teardown.sh
new file mode 100644
index 00000000..9916a424
--- /dev/null
+++ b/plans/sdc/sdc-helm-validator/teardown.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+echo "Starting teardown script"
+
+running_container=$(docker ps --filter name=sdc-helm-validator -qa)
+
+docker stop $running_container
+docker rm $running_container
diff --git a/plans/sdc/sdc-helm-validator/testplan.txt b/plans/sdc/sdc-helm-validator/testplan.txt
new file mode 100644
index 00000000..9133005c
--- /dev/null
+++ b/plans/sdc/sdc-helm-validator/testplan.txt
@@ -0,0 +1,3 @@
+# Test suites are relative paths under [integration/csit.git]/tests/.
+# Place the suites in run order.
+sdc/sdc-helm-validator
diff --git a/tests/sdc/sdc-helm-validator/assets/charts/correct-apiVersion-v2.tgz b/tests/sdc/sdc-helm-validator/assets/charts/correct-apiVersion-v2.tgz
new file mode 100644
index 00000000..a1a99bab
--- /dev/null
+++ b/tests/sdc/sdc-helm-validator/assets/charts/correct-apiVersion-v2.tgz
diff --git a/tests/sdc/sdc-helm-validator/assets/charts/one-lint-one-render-error-apiVersion-v2.tgz b/tests/sdc/sdc-helm-validator/assets/charts/one-lint-one-render-error-apiVersion-v2.tgz
new file mode 100644
index 00000000..874dd3c0
--- /dev/null
+++ b/tests/sdc/sdc-helm-validator/assets/charts/one-lint-one-render-error-apiVersion-v2.tgz
diff --git a/tests/sdc/sdc-helm-validator/assets/charts/one-lint-warning-apiVersion-v2.tgz b/tests/sdc/sdc-helm-validator/assets/charts/one-lint-warning-apiVersion-v2.tgz
new file mode 100644
index 00000000..51f4dd1d
--- /dev/null
+++ b/tests/sdc/sdc-helm-validator/assets/charts/one-lint-warning-apiVersion-v2.tgz
diff --git a/tests/sdc/sdc-helm-validator/sdc-helm-validator.robot b/tests/sdc/sdc-helm-validator/sdc-helm-validator.robot
new file mode 100644
index 00000000..04b570f9
--- /dev/null
+++ b/tests/sdc/sdc-helm-validator/sdc-helm-validator.robot
@@ -0,0 +1,274 @@
+*** Settings ***
+Documentation     Testing SDC Helm Validator
+Suite Setup       Create Validator Session
+Library           RequestsLibrary
+Library           OperatingSystem
+Library           Collections
+
+*** Variables ***
+${VALIDATOR_URL}                    http://${VALIDATOR}
+${VERSIONS_ENDPOINT}                /versions
+${VALIDATE_ENDPOINT}                /validate
+
+${CHARTS_PATH}                      %{WORKSPACE}/tests/sdc/sdc-helm-validator/assets/charts/
+
+${CHART_CORRECT_V2}                 /correct-apiVersion-v2.tgz
+${CHART_LINT_WARNING_V2}            /one-lint-warning-apiVersion-v2.tgz
+${CHART_LINT_RENDER_ERROR_V2}       /one-lint-one-render-error-apiVersion-v2.tgz
+${CHART_INCORRECT}                  /incorrect-chart.tgz
+
+${REQ_KEY_VERSION_DESIRED}          versionDesired
+${REQ_KEY_IS_LINTED}                isLinted
+${REQ_KEY_IS_STRICT_LINTED}         isStrictLinted
+${REQ_KEY_FILE}                     file
+
+${RESP_KEY_VERSIONS}                versions
+${RESP_KEY_VERSION_USED}            versionUsed
+${RESP_KEY_DEPLOYABLE}              deployable
+${RESP_KEY_VALID}                   valid
+${RESP_KEY_RENDER_ERRORS}           renderErrors
+${RESP_KEY_LINT_ERROR}              lintError
+${RESP_KEY_LINT_WARNING}            lintWarning
+
+
+*** Test Cases ***
+
+Verify That Sdc Helm Validator Correctly Responds With Supported Versions Array
+    [Tags]                          SDC_HELM_VALIDATOR_1
+    [Documentation]                 Verify that validator correctly responds with supported helm versions array.
+    ...                             Send GET request to ask for supported versions array.
+    ...                             Should reply with JSON containing an array of Helm versions that are supported by the validator.
+    [Timeout]                       5 minute
+
+    ${resp}=                        GET On Session                 ${VALIDATOR_SESSION}          ${VERSIONS_ENDPOINT}
+    Status Should Be                200                            ${resp}
+
+    @{versions}=                    Get From Dictionary            ${resp.json()}                ${RESP_KEY_VERSIONS}
+    Should Not Be Empty             ${versions}
+
+    FOR                             ${version}                     IN                            @{versions}
+                                    Should Match Regexp            ${version}                    \\d+\.\\d+\.\\d+
+    END
+
+Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Default Version
+    [Tags]                          SDC_HELM_VALIDATOR_2
+    [Documentation]                 Verify that validator correctly responds for correct chart validation request with default version.
+    ...                             Send POST request to validate correct chart. Input: Helm chart with api version v2, no additional data.
+    ...                             Should reply with JSON containing the following information: used version = 3.x.x, deployable = true, render errors = []
+    [Timeout]                       5 minute
+
+    ${other_data}=                  Create Dictionary
+    ${resp}=                        Send Post Request              ${CHART_CORRECT_V2}            ${other_data}
+
+    Status Should Be                200                            ${resp}
+
+    Dictionary Should Not Contain Key  ${resp.json()}              ${RESP_KEY_LINT_ERROR}
+    Dictionary Should Not Contain Key  ${resp.json()}              ${RESP_KEY_LINT_WARNING}
+    Dictionary Should Not Contain Key  ${resp.json()}              ${RESP_KEY_VALID}
+
+    ${version}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VERSION_USED}
+    Should Start With               ${version}                     3.
+
+    ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
+    Should Be True                  ${isDeployable}
+
+    ${errors}=                      Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
+    Should Be Empty                 ${errors}
+
+Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Given V3 Version
+    [Tags]                          SDC_HELM_VALIDATOR_3
+    [Documentation]                 Verify that validator correctly responds for correct chart validation request with given v3 version.
+    ...                             Send POST request to validate correct chart. Input: Helm chart with api version v2, desired version = v3.
+    ...                             Should reply with JSON containing the following information: used version = 3.x.x, deployable = true, render errors = [].
+    [Timeout]                       5 minute
+
+    ${other_data}=                  Create Dictionary              ${REQ_KEY_VERSION_DESIRED}=v3
+    ${resp}=                        Send Post Request              ${CHART_CORRECT_V2}            ${other_data}
+
+    Status Should Be                200                            ${resp}
+
+    ${version}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VERSION_USED}
+    Should Start With               ${version}                     3.
+
+    ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
+    Should Be True                  ${isDeployable}
+
+    ${errors}=                      Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
+    Should Be Empty                 ${errors}
+
+Verify That Sdc Helm Validator Responds With Error For Chart Validation Request With Invalid Version
+    [Tags]                          SDC_HELM_VALIDATOR_4
+    [Documentation]                 Verify that validator responds with error and 400 status code for validation request with invalid version.
+    ...                             Send POST request with correct chart but not supported Helm version. Input: Correct helm chart, desired version = v10.
+    ...                             Should reply with JSON containing error message with information regarding not supported Helm version. Response code should be 400.
+    [Timeout]                       5 minute
+
+    ${chart_path}                   Catenate                       SEPARATOR=                    ${CHARTS_PATH}                    ${CHART_CORRECT_V2}
+    ${files}=                       Create Multi Part              ${chart_path}
+    ${other_data}=                  Create Dictionary              ${REQ_KEY_VERSION_DESIRED}=v10
+    ${resp}=                        Post Request                   ${VALIDATOR_SESSION}          ${VALIDATE_ENDPOINT}              files=${files}    data=${other_data}
+
+    Should Be Equal As Strings      ${resp.status_code}            400
+    Should Be Equal As Strings      ${resp.text}                   {"message":"Version: 10 is not supported"}
+
+Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Random Supported Version
+    [Tags]                          SDC_HELM_VALIDATOR_5
+    [Documentation]                 Verify that validator correctly responds for correct chart validation request with random supported version.
+    ...                             Send GET request to ask for supported versions array.
+    ...                             Should reply with JSON containing an array of Helm versions that are supported by the validator.
+    ...                             Select random version from the returned array.
+    ...                             Send POST request with correct chart and randomly chosen supported Helm version. Input: Correct helm chart, desired version = [randomly selected].
+    ...                             Response code should be 200.
+    [Timeout]                       5 minute
+
+    ${resp}=                        GET On Session                 ${VALIDATOR_SESSION}          ${VERSIONS_ENDPOINT}
+    ${versions}=                    Get From Dictionary            ${resp.json()}                ${RESP_KEY_VERSIONS}
+    ${list_size}=                   Get length                     ${versions}
+    ${random_index}=                Evaluate                       random.randint(0, ${list_size}-1)
+    ${version}=                     Get From List                  ${versions}                   ${random_index}
+    Status Should Be                200                            ${resp}
+
+    ${other_data}=                  Create Dictionary              ${REQ_KEY_VERSION_DESIRED}=${version}
+    ${resp}=                        Send Post Request              ${CHART_CORRECT_V2}            ${other_data}
+    Status Should Be                200                            ${resp}
+
+Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Lint
+    [Tags]                          SDC_HELM_VALIDATOR_6
+    [Documentation]                 Verify that validator correctly responds for correct chart validation request with lint.
+    ...                             Send POST request to validate correct chart and lint. Input: Helm chart with api version v2, linted = true.
+    ...                             Should reply with JSON containing the following information: deployable = true, valid = true, render errors = [], lint errors = [], lint warnings = [].
+    ...                             Status code should be 200.
+    [Timeout]                       5 minute
+
+    ${other_data}=                  Create Dictionary              ${REQ_KEY_IS_LINTED}=true
+    ${resp}=                        Send Post Request              ${CHART_CORRECT_V2}            ${other_data}
+    Status Should Be                200                            ${resp}
+
+    ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
+    Should Be True                  ${isDeployable}
+
+    ${isValid}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VALID}
+    Should Be True                  ${isValid}
+
+    ${renderErrors}=                Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
+    Should Be Empty                 ${renderErrors}
+
+    ${lintErrors}=                  Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_ERROR}
+    Should Be Empty                 ${lintErrors}
+
+    ${lintWarnings}=                Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_WARNING}
+    Should Be Empty                 ${lintWarnings}
+
+Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint Warnings
+    [Tags]                          SDC_HELM_VALIDATOR_7
+    [Documentation]                 Verify that validator correctly responds for chart validation request with lint warnings.
+    ...                             Send POST request to validate chart and lint. Input: Helm chart that should cause lint warning, linted = true.
+    ...                             Should reply with JSON containing the following information: deployable = true, valid = true, render errors = [], lint warning = [not empty]
+    ...                             Status code should be 200.
+    [Timeout]                       5 minute
+
+    ${other_data}=                  Create Dictionary              ${REQ_KEY_IS_LINTED}=true
+    ${resp}=                        Send Post Request              ${CHART_LINT_WARNING_V2}      ${other_data}
+    Status Should Be                200                            ${resp}
+
+    ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
+    Should Be True                  ${isDeployable}
+
+    ${isValid}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VALID}
+    Should Be True                  ${isValid}
+
+    ${renderErrors}=                Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
+    Should Be Empty                 ${renderErrors}
+
+    ${lintErrors}=                  Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_ERROR}
+    Should Be Empty                 ${lintErrors}
+
+    @{lintWarnings}=                Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_WARNING}
+    Should Not Be Empty             @{lintWarnings}
+    Should Contain                  @{lintWarnings}                [WARNING] templates/: directory not found
+
+Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint Strict Checking
+    [Tags]                          SDC_HELM_VALIDATOR_8
+    [Documentation]                 Verify that validator correctly responds for chart validation request with lint strict checking.
+    ...                             Send POST request to validate chart and strictly lint. Input: Helm chart that should cause lint warning, linted = true, strict linted = true.
+    ...                             Should reply with JSON containing the following information: deployable = true, valid = false, render errors = [], lint warning = [not empty].
+    ...                             Status code should be 200.
+    [Timeout]                       5 minute
+
+    ${other_data}=                  Create Dictionary              ${REQ_KEY_IS_LINTED}=true     ${REQ_KEY_IS_STRICT_LINTED}=true
+    ${resp}=                        Send Post Request              ${CHART_LINT_WARNING_V2}      ${other_data}
+    Status Should Be                200                            ${resp}
+
+    ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
+    Should Be True                  ${isDeployable}
+
+    ${isValid}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VALID}
+    Should Not Be True              ${isValid}                     There should be a lint warning, which in strict mode on should make the chart invalid
+
+    ${renderErrors}=                Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
+    Should Be Empty                 ${renderErrors}
+
+    ${lintErrors}=                  Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_ERROR}
+    Should Be Empty                 ${lintErrors}
+
+    ${lintWarnings}=                Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_WARNING}
+    Should Not Be Empty             ${lintWarnings}
+    Should Contain                  @{lintWarnings}                [WARNING] templates/: directory not found
+
+Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint And Render Errors
+    [Tags]                          SDC_HELM_VALIDATOR_9
+    [Documentation]                 Verify that validator correctly responds for chart validation request with lint and render errors.
+    ...                             Send POST request to validate chart and strictly lint. Input: Helm chart that should cause lint and render errors, linted = true, strict linted = true.
+    ...                             Should reply with JSON containing the following information: deployable = false, valid = false, render errors = [not empty], lint errors = [not empty], lint warnings = [].
+    ...                             Status code should be 200.
+    [Timeout]                       5 minute
+
+    ${other_data}=                  Create Dictionary              ${REQ_KEY_IS_LINTED}=true     ${REQ_KEY_IS_STRICT_LINTED}=true
+    ${resp}=                        Send Post Request              ${CHART_LINT_RENDER_ERROR_V2}  ${other_data}
+    Status Should Be                200                            ${resp}
+
+    ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
+    Should Not Be True              ${isDeployable}                There should be render errors which should make the chart not deployable
+
+    ${isValid}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VALID}
+    Should Not Be True              ${isValid}                     There should be lint errors which should make the chart invalid
+
+    @{renderErrors}=                Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
+    Should Not Be Empty             @{renderErrors}
+    Should Contain                  @{renderErrors}                Error: template: mychartname/templates/test.yaml:2:18: executing "mychartname/templates/test.yaml" at <.Values.image.repository>: nil pointer evaluating interface {}.repository
+
+    @{lintErrors}=                  Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_ERROR}
+    Should Not Be Empty             @{lintErrors}
+    Should Contain                  @{lintErrors}                  [ERROR] templates/: template: mychartname/templates/test.yaml:2:18: executing "mychartname/templates/test.yaml" at <.Values.image.repository>: nil pointer evaluating interface {}.repository
+
+    ${lintWarnings}=                Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_WARNING}
+    Should Be Empty                 ${lintWarnings}
+
+*** Keywords ***
+
+Create Validator Session
+    Create Session                  validator_session              ${VALIDATOR_URL}
+    Set Suite Variable              ${VALIDATOR_SESSION}           validator_session
+
+Send Post Request
+    [Arguments]                     ${chart_name}                  ${data_dictionary}
+    ${chart_path}                   Catenate                       SEPARATOR=                    ${CHARTS_PATH}                    ${chart_name}
+    ${files}=                       Create Multi Part              ${chart_path}
+
+    ${resp}=                        POST On Session                ${VALIDATOR_SESSION}          ${VALIDATE_ENDPOINT}              files=${files}    data=${data_dictionary}
+    [Return]                        ${resp}
+
+Create Multi Part
+    [Arguments]                     ${path}
+    ${data}=                        Get Binary File                ${path}
+    ${files}=                       Create Dictionary
+    ${fileDir}  ${fileName}=        Split Path                     ${path}
+    ${partData}=                    Create List                    ${fileName}                 ${data}
+    Set To Dictionary               ${files}                       ${REQ_KEY_FILE}=${partData}
+    [Return]                        ${files}
+
+Get By Key
+    [Arguments]                     ${dict}                        ${key}
+    Dictionary Should Contain Key   ${dict}                        ${key}
+    ${value}=                       Get From Dictionary            ${dict}                     ${key}
+    [Return]                        ${value}