From 1592b0854a5552091afcb12564bc891e6e8e8a5f Mon Sep 17 00:00:00 2001 From: Krzysztof Kuzmicki Date: Fri, 27 Mar 2020 14:03:43 +0100 Subject: ROBOT e2e test cases improvements for csar/zip tosca package validation in VNFSDK and SDC Onbarding Issue-ID: INT-1240 Signed-off-by: Krzysztof Kuzmicki Change-Id: I87dd001822e2fe2f61a731167250eb56e724b606 --- docker/Dockerfile | 3 +- .../vnfsdk/vnfsdk_validation_request.jinja | 9 ++ robot/resources/sdc_interface.robot | 1 + .../vnfsdk_validation_template.robot | 143 +++++++++++++++++++++ robot/resources/vnfsdk_interface.robot | 7 + robot/testsuites/vnfsdk_validation.robot | 99 ++++++++++++++ 6 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 robot/assets/templates/vnfsdk/vnfsdk_validation_request.jinja create mode 100644 robot/resources/test_templates/vnfsdk_validation_template.robot create mode 100644 robot/testsuites/vnfsdk_validation.robot diff --git a/docker/Dockerfile b/docker/Dockerfile index 188a8e15..1a732033 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,7 +34,8 @@ RUN apt-get update \ x11-xserver-utils \ xvfb \ xxd \ - vim + vim \ + zip # install python 3 after so it isn't default python version RUN apt-get install \ diff --git a/robot/assets/templates/vnfsdk/vnfsdk_validation_request.jinja b/robot/assets/templates/vnfsdk/vnfsdk_validation_request.jinja new file mode 100644 index 00000000..18f77b5b --- /dev/null +++ b/robot/assets/templates/vnfsdk/vnfsdk_validation_request.jinja @@ -0,0 +1,9 @@ +[{ + "scenario": "{{scenario}}", + "testSuiteName": "{{test_suite_name}}", + "testCaseName": "{{test_case_name}}", + "parameters": { + "csar": "file://{{file_name}}", + "pnf":"{{pnf}}" + } + }] diff --git a/robot/resources/sdc_interface.robot b/robot/resources/sdc_interface.robot index a85ba27f..4780a35a 100644 --- a/robot/resources/sdc_interface.robot +++ b/robot/resources/sdc_interface.robot @@ -778,6 +778,7 @@ Upload SDC Heat Package Create Multi Part ${files} upload ${file_path} contentType=application/zip ${resp}= SDC.Run Post Files Request ${SDC_BE_ONBOARD_ENDPOINT} ${SDC_VENDOR_SOFTWARE_PRODUCT_PATH}/${software_product_id}/versions/${version_id}${SDC_VENDOR_SOFTWARE_UPLOAD_PATH} ${files} ${SDC_DESIGNER_USER_ID} auth=${GLOBAL_SDC_AUTHENTICATION} Should Be Equal As Strings ${resp.status_code} 200 + [Return] ${resp} Add SDC Catalog Service [Documentation] Creates an SDC Catalog Service and returns its id diff --git a/robot/resources/test_templates/vnfsdk_validation_template.robot b/robot/resources/test_templates/vnfsdk_validation_template.robot new file mode 100644 index 00000000..56cd0b96 --- /dev/null +++ b/robot/resources/test_templates/vnfsdk_validation_template.robot @@ -0,0 +1,143 @@ +*** Settings *** +Documentation This test template encapsulates the VNF Orchestration use case. + +Library OperatingSystem +Library ArchiveLibrary +Library Collections + +Library ONAPLibrary.Templating WITH NAME Templating +Resource ../global_properties.robot +Resource ../vnfsdk_interface.robot +Resource ../sdc_interface.robot + +*** Variables *** +${VNFSDK_TEST} vnfsdk/vnfsdk_validation_request.jinja + + + +*** Variables *** +${VNFSDK_TOSCA_ONBOARDING_PACKAGES_DIRECTORY} ${GLOBAL_TOSCA_ONBOARDING_PACKAGES_FOLDER}/vnfsdk +${VNFSDK_CSAR_DIRECTORY} ${VNFSDK_TOSCA_ONBOARDING_PACKAGES_DIRECTORY}/temp +@{empty_list} +*** Keywords *** + +Validate Onboarding Package + [Arguments] ${package_folder} ${scenario}=onap-vtp ${test_suite_name}=validation ${test_case_name}=csar-validate ${pnf}=TRUE ${secured_package}=FALSE ${negative_test_case}=TRUE ${failed_vnfreqNames}=@{empty_list} ${sdc_response}=@{empty_list} ${secure_type}=CMS ${sdc_cert}=sdc-valid + Create Directory ${VNFSDK_CSAR_DIRECTORY} + ${onboarding_package_path}= Run Keyword If "${secured_package}"=='FALSE' Create CSAR Package ${package_folder} + ... ELSE Create Secured CSAR ZIP Package ${package_folder} ${secure_type} ${sdc_cert} + Run Keyword If "${negative_test_case}"=='FALSE' Validate Valid Onboarding Package ${package_folder} ${onboarding_package_path} ${scenario} ${test_suite_name} ${test_case_name} ${pnf} ${secured_package} + ... ELSE Validate Not Valid Onboarding Package ${package_folder} ${onboarding_package_path} ${scenario} ${test_suite_name} ${test_case_name} ${pnf} ${failed_vnfreqNames} ${secured_package} + Validate Onboarding Package In SDC ${onboarding_package_path} ${package_folder} ${sdc_cert} ${negative_test_case} ${sdc_response} ${sdc_cert} + + +Create CSAR Package + [Arguments] ${package_folder} + Empty Directory ${VNFSDK_CSAR_DIRECTORY} + ${csar}= Catenate ${VNFSDK_CSAR_DIRECTORY}/${package_folder}.csar + Copy File ${GLOBAL_TOSCA_ONBOARDING_PACKAGES_FOLDER}/vnfsdk/${package_folder}.csar ${csar} + [Return] ${csar} + +Create Secured CSAR ZIP Package + [Arguments] ${package_folder} ${secure_type} ${sdc_cert} + ${zip}= Run Keyword If "${secure_type}"=='CMS' Create Secured CSAR ZIP Package with CMS ${package_folder} ${sdc_cert} + ... ELSE Create Secured CSAR ZIP Package with CMS and CERT ${package_folder} ${sdc_cert} + [Return] ${zip} + +Create Secured CSAR ZIP Package with CMS + [Arguments] ${package_folder} ${cert} + ${zip}= Catenate ${VNFSDK_CSAR_DIRECTORY}/${package_folder}.zip + ${cms}= Catenate ${VNFSDK_CSAR_DIRECTORY}/${package_folder}.cms + ${csar}= Create CSAR Package ${package_folder} + ${rc} = Run and Return RC openssl cms -sign -signer /tmp/package-robot-${cert}.cert -inkey /tmp/package-private-robot-${cert}.key -outform PEM -binary -in ${csar} -out ${cms} + Should Be Equal As Integers ${rc} 0 + ${rc} = Run and Return RC cd ${VNFSDK_CSAR_DIRECTORY}; zip -r ${zip} * + Should Be Equal As Integers ${rc} 0 + [Return] ${zip} + +Create Secured CSAR ZIP Package with CMS and CERT + [Arguments] ${package_folder} ${cert} + ${zip}= Catenate ${VNFSDK_CSAR_DIRECTORY}/${package_folder}.zip + ${cms}= Catenate ${VNFSDK_CSAR_DIRECTORY}/${package_folder}.cms + ${csar}= Create CSAR Package ${package_folder} + Copy File /tmp/package-robot-${cert}.cert ${VNFSDK_CSAR_DIRECTORY}/${package_folder}.cert + ${rc} = Run and Return RC openssl cms -sign -signer /tmp/package-robot-${cert}.cert -inkey /tmp/package-private-robot-${cert}.key -outform PEM -binary -nocerts -in ${csar} -out ${cms} + Should Be Equal As Integers ${rc} 0 + ${rc} = Run and Return RC cd ${VNFSDK_CSAR_DIRECTORY}; zip -r ${zip} * + Should Be Equal As Integers ${rc} 0 + [Return] ${zip} + +Validate Valid Onboarding Package + [Arguments] ${package_folder} ${onboarding_package_path} ${scenario} ${test_suite_name} ${test_case_name} ${pnf} ${secured_package} + ${onboarding_package_name}= Run Keyword If "${secured_package}"=='FALSE' Catenate ${package_folder}.csar + ... ELSE Catenate ${package_folder}.zip + ${result}= Run VNFSDK Validate Request ${onboarding_package_name} ${onboarding_package_path} ${scenario} ${test_suite_name} ${test_case_name} ${pnf} + Log ${result.json()} + ${json}= Set Variable ${result.json()} + ${status}= Set Variable ${json[0]['results']['criteria']} + Should Be Equal As Strings ${status} PASS + +Run VNFSDK Validate Request + [Arguments] ${onboarding_package_name} ${onboarding_package_path} ${scenario} ${test_suite_name} ${test_case_name} ${pnf} + ${arguments}= Create Dictionary scenario=${scenario} onboarding_package_path=${onboarding_package_path} test_suite_name=${test_suite_name} test_case_name=${test_case_name} pnf=${pnf} file_name=${onboarding_package_name} + Templating.Create Environment vnfsdk ${GLOBAL_TEMPLATE_FOLDER} + ${executions}= Templating.Apply Template vnfsdk ${VNFSDK_TEST} ${arguments} + ${fileData}= Get Binary File ${onboarding_package_path} + ${fileDir} ${fileName}= Split Path ${onboarding_package_path} + ${file_part}= Create List ${fileName} ${fileData} application/octet-stream + ${executions_parts}= Create List ${executions} + ${fileParts}= Create Dictionary + Set to Dictionary ${fileParts} file=${file_part} + Set to Dictionary ${fileParts} executions=${executions} + ${resp}= Run VNFSDK Post Request /onapapi/vnfsdk-marketplace/v1/vtp/executions ${fileParts} + [Return] ${resp} + +Validate Not Valid Onboarding Package + [Arguments] ${package_folder} ${onboarding_package_path} ${scenario} ${test_suite_name} ${test_case_name} ${pnf} ${failed_req_list} ${secured_package} + ${onboarding_package_name}= Run Keyword If "${secured_package}"=='FALSE' Catenate ${package_folder}.csar + ... ELSE Catenate ${package_folder}.zip + ${result}= Run VNFSDK Validate Request ${onboarding_package_name} ${onboarding_package_path} ${scenario} ${test_suite_name} ${test_case_name} ${pnf} + Log ${result.json()} + ${json}= Set Variable ${result.json()} + ${status}= Set Variable ${json[0]['results']['criteria']} + Should Be Equal As Strings ${status} FAILED + ${status_req_list}= Set Variable ${json[0]['results']['results']} + ${failed_req_from_test_run}= Create List + :FOR ${status_req} IN @{status_req_list} + \ ${req_status}= Get From Dictionary ${status_req} passed + \ Run Keyword If "${req_status}"=='False' Add Failed Requirement To List ${status_req} ${failed_req_from_test_run} vnfreqName + Log ${failed_req_from_test_run} + Lists Should Be Equal ${failed_req_from_test_run} ${failed_req_list} + +Add Failed Requirement To List + [Arguments] ${status_req} ${failed_req} ${param_name} + ${req}= Get From Dictionary ${status_req} ${param_name} + Append To List ${failed_req} ${req} + +Get And Comapre Error Responses From SDC API + [Arguments] ${resp} ${sdc_response} + ${json}= Set Variable ${resp.json()} + ${sdc_response_list} Set Variable ${json['errors']['uploadFile']} + ${failed_req_from_test_run}= Create List + :FOR ${message_item} IN @{sdc_response_list} + \ ${req_status}= Get From Dictionary ${message_item} level + \ Run Keyword If "${req_status}"=='ERROR' Add Failed Requirement To List ${message_item} ${failed_req_from_test_run} message + Log ${failed_req_from_test_run} + Lists Should Be Equal ${failed_req_from_test_run} ${sdc_response} + +Validate Onboarding Package In SDC + [Arguments] ${onboarding_package_path} ${package_folder} ${sdc_validation} ${negative_test_case} ${sdc_response} ${sdc_cert} + ${license_model_id} ${license_model_version_id}= Add SDC License Model + ${license_temp_date}= Get Current Date + ${license_start_date}= Get Current Date result_format=%m/%d/%Y + ${license_end_date}= Add Time To Date ${license_temp_date} 365 days result_format=%m/%d/%Y + ${key_group_id}= Add SDC License Group ${license_model_id} ${license_model_version_id} ${license_start_date} ${license_end_date} + ${pool_id}= Add SDC Entitlement Pool ${license_model_id} ${license_model_version_id} ${license_start_date} ${license_end_date} + ${feature_group_id}= Add SDC Feature Group ${license_model_id} ${key_group_id} ${pool_id} ${license_model_version_id} + ${license_agreement_id}= Add SDC License Agreement ${license_model_id} ${feature_group_id} ${license_model_version_id} + Submit SDC License Model ${license_model_id} ${license_model_version_id} + ${license_model_resp}= Get SDC License Model ${license_model_id} ${license_model_version_id} + ${software_product_id} ${software_product_version_id}= Add SDC Software Product ${license_agreement_id} ${feature_group_id} ${license_model_resp['vendorName']} ${license_model_id} ${license_model_version_id} ${package_folder} + ${resp}= Upload SDC Heat Package ${software_product_id} ${onboarding_package_path} ${software_product_version_id} + Run Keyword If "${negative_test_case}"=='TRUE' or "${sdc_cert}"=='sdc-invalid' Get And Comapre Error Responses From SDC API ${resp} ${sdc_response} + ... ELSE Validate SDC Software Product ${software_product_id} ${software_product_version_id} diff --git a/robot/resources/vnfsdk_interface.robot b/robot/resources/vnfsdk_interface.robot index 01cacf4d..fea6b31a 100644 --- a/robot/resources/vnfsdk_interface.robot +++ b/robot/resources/vnfsdk_interface.robot @@ -22,3 +22,10 @@ Run VNFSDK Get Request Log Received response from VNFSDK ${resp.text} [Return] ${resp} +Run VNFSDK Post Request + [Documentation] Runs VNFSDK Get request + [Arguments] ${data_path} ${files} ${headers} + ${session}= Create Session vnfsdk ${VNFSDK_ENDPOINT} + ${resp}= Post Request vnfsdk ${data_path} files=${files} + Log Received response from VNFSDK ${resp} + [Return] ${resp} diff --git a/robot/testsuites/vnfsdk_validation.robot b/robot/testsuites/vnfsdk_validation.robot new file mode 100644 index 00000000..993efca7 --- /dev/null +++ b/robot/testsuites/vnfsdk_validation.robot @@ -0,0 +1,99 @@ +*** Settings *** +Documentation This test suite creates csara and signed zip tosca VSP packages, validates them in VNFSDK and SDC +Test Timeout 1m + +Resource ../resources/test_templates/vnfsdk_validation_template.robot +Library String +Test Template Validate Onboarding Package +Default Tags vnfsdk + + +*** Variables *** +@{allMandatoryEntriesDefinedInTOSCAMeta} r146092 +@{no_pnfd_release_date_time_error} r57019 +@{non_mano_artifact_sets_is_mandatory} r146092 +@{noETSI-Entry-ManifestOrETSI-Entry-Change-Log} r293901 +@{PNFD_missing} SOL004 r10087 r87234 r35854 r15837 r17852 r293901 r146092 r57019 r787965 +@{allMandatoryEntriesDefinedInTOSCAMeta_sdc_message} Following entry not supported in TOSCA.meta Entry-Tests=Artifacts/Tests Manifest contains invalid line : #The manifest file shall include a list of all files contained in or referenced from the VNF package with their location +@{no_pnfd_release_date_time_error_sdc_message} Following entry not supported in TOSCA.meta Entry-Tests=Artifacts/Tests Manifest contains invalid line : #The manifest file shall include a list of all files contained in or referenced from the VNF package with their location +@{non_mano_artifact_sets_is_mandatory_sdc_message} Following entry not supported in TOSCA.meta Entry-Tests=Artifacts/Tests Manifest contains invalid line : #The manifest file shall include a list of all files contained in or referenced from the VNF package with their location +@{noETSI-Entry-ManifestOrETSI-Entry-Change-Log_sdc_message} Manifest contains invalid line : #The manifest file shall include a list of all files contained in or referenced from the VNF package with their location +@{PNFD_missing_sdc_message} TOSCA.meta file in TOSCA-metadata directory missing entry Created-By +@{invalid_certificate} Could not verify signature! + + +*** Test Cases *** +Validate Onboarding allMandatoryEntriesDefinedInTOSCAMeta + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/allMandatoryEntriesDefinedInTOSCAMeta + ... Imports it as csar VSP package to SDC and comapres with list of expected errors. + ... Runs VNFSDK validation and comapres with list of expected errors. + allMandatoryEntriesDefinedInTOSCAMeta failed_vnfreqNames=@{allMandatoryEntriesDefinedInTOSCAMeta} sdc_response=@{allMandatoryEntriesDefinedInTOSCAMeta_sdc_message} + +Validate Onboarding non_mano_artifact_sets_is_mandatory + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/non_mano_artifact_sets_is_mandatory + ... Imports it as csar VSP package to SDC and comapres with list of expected errors. + ... Runs VNFSDK validation and comapres with list of expected errors. + non_mano_artifact_sets_is_mandatory failed_vnfreqNames=@{non_mano_artifact_sets_is_mandatory} sdc_response=@{non_mano_artifact_sets_is_mandatory_sdc_message} + +Validate Onboarding no_pnfd_release_date_time_error + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/no_pnfd_release_date_time_error + ... Imports it as csar VSP package to SDC and comapres with list of expected errors. + ... Runs VNFSDK validation and comapres with list of expected errors. + no_pnfd_release_date_time_error failed_vnfreqNames=@{no_pnfd_release_date_time_error} sdc_response=@{no_pnfd_release_date_time_error_sdc_message} + +Validate Onboarding noETSI-Entry-ManifestOrETSI-Entry-Change-Log + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/noETSI-Entry-ManifestOrETSI-Entry-Change-Log + ... Imports it as csar VSP package to SDC and comapres with list of expected errors. + ... Runs VNFSDK validation and comapres with list of expected errors. + noETSI-Entry-ManifestOrETSI-Entry-Change-Log failed_vnfreqNames=@{noETSI-Entry-ManifestOrETSI-Entry-Change-Log} sdc_response=@{noETSI-Entry-ManifestOrETSI-Entry-Change-Log_sdc_message} + +Validate Onboarding PNFD_missing + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/PNFD_missing + ... Imports it as csar VSP package to SDC and comapres with list of expected errors. + ... Runs VNFSDK validation and comapres with list of expected errors. + PNFD_missing failed_vnfreqNames=@{PNFD_missing} sdc_response=@{PNFD_missing_sdc_message} + +Validate Onboarding test_SDC_and_VNFSDK_API + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/test_SDC_and_VNFSDK_API + ... Imports it as csar VSP package to SDC and expects success. + ... Runs VNFSDK validation and and expects success. + test_SDC_and_VNFSDK_API negative_test_case=FALSE + +Validate Onboarding test_SDC_and_VNFSDK_API, secured csar, CMS only, valid certificate + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/test_SDC_and_VNFSDK_API + ... Next sign with certificate that is imported to SDC onabarding POD using CMS only method (CMS file signs csar, zip contains only csar and cms files ) + ... Imports it as zip VSP package to SDC and expects success. + ... Runs VNFSDK validation and and expects success. + test_SDC_and_VNFSDK_API secured_package=TRUE negative_test_case=FALSE secure_type=CMS sdc_cert=sdc-valid + +Validate Onboarding test_SDC_and_VNFSDK_API, secured csar, CMS and CERT, valid certificate + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/test_SDC_and_VNFSDK_API + ... Next sign with certificate that is imported to SDC onabarding POD using CMS and CERT method (CMS file signs csar, zip contains csar, certificate and cms files ) + ... Imports it as zip VSP package to SDC and expects success. + ... Runs VNFSDK validation and and expects success. + test_SDC_and_VNFSDK_API secured_package=TRUE negative_test_case=FALSE secure_type=CMS_AND_CERT sdc_cert=sdc-valid + +Validate Onboarding test_SDC_and_VNFSDK_API secured csar, CMS only, invalid certificate + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/test_SDC_and_VNFSDK_API + ... Next sign with certificate that is not imported to SDC onabarding POD using CMS only method (CMS file signs csar, zip contains only csar and cms files ) + ... Imports it as zip VSP package to SDC and expects error due to issues with certificate validation. + ... Runs VNFSDK validation and and expects success. + test_SDC_and_VNFSDK_API secured_package=TRUE negative_test_case=FALSE sdc_response=@{invalid_certificate} secure_type=CMS sdc_cert=sdc-invalid + +Validate Onboarding test_SDC_and_VNFSDK_API secured csar, CMS and CERT, invalid certificate + [Documentation] + ... This test case creates TOSCA csar software package for PNF, based on /var/opt/ONAP/demo/tosca/vnfsdk/test_SDC_and_VNFSDK_API + ... Next sign with certificate that is not imported to SDC onabarding POD using CMS and CERT method (CMS file signs csar, zip contains csar, certificate and cms files ) + ... Imports it as zip VSP package to SDC and expects error due to issues with certificate validation. + ... Runs VNFSDK validation and and expects success + test_SDC_and_VNFSDK_API secured_package=TRUE negative_test_case=FALSE sdc_response=@{invalid_certificate} secure_type=CMS_AND_CERT sdc_cert=sdc-invalid + -- cgit 1.2.3-korg