From f5e39dd27dd12b4f90c39ea267a509eabe1cfe4c Mon Sep 17 00:00:00 2001 From: Tony Hansen Date: Wed, 11 Oct 2017 18:08:57 +0000 Subject: fix inputs, template expander, add tests rewrite the template expander: 1) faster 2) handle multiple line variables 3) add new ONAPTEMPLATE_STANDARD_INPUTS_TYPES variable fix inputs directives in pgaas templates by using ONAPTEMPLATE_STANDARD_INPUTS_TYPES add call to check-blueprint-vs-input to test_templates Change-Id: I4ffc220d7e1f4820b271d055b5de3961ca236983 Signed-off-by: Tony Hansen Issue-ID: DCAEGEN2-128 Signed-off-by: Tony Hansen --- blueprints/pgaas-cluster.yaml-template | 32 ++------ blueprints/pgaas-database.yaml-template | 24 +----- blueprints/pgaas-disk.yaml-template | 26 +------ blueprints/pgaas.yaml-template | 34 ++------- check-blueprint-vs-input/lib/sample-inputs.yaml | 4 +- mvn-phase-lib.sh | 99 +++++++++++++------------ 6 files changed, 77 insertions(+), 142 deletions(-) diff --git a/blueprints/pgaas-cluster.yaml-template b/blueprints/pgaas-cluster.yaml-template index 59b188a..a1885f1 100644 --- a/blueprints/pgaas-cluster.yaml-template +++ b/blueprints/pgaas-cluster.yaml-template @@ -42,35 +42,17 @@ imports: - "{{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }}/type_files/pgaas/pgaas_types.yaml" inputs: - # environment info blueprint_version: type: string default: 'VERSION-STAMP-GOES-HERE' - flavor_id: - type: string - key_filename: - type: string - keypair: - type: string - location_domain: - type: string - location_prefix: - type: string - openstack: {} - private_net: - type: string - public_net: - type: string - security_group: - type: string - ubuntu1604image_id: - type: string # pgaas-specific info pgaas_cluster_name: type: string default: pgcl +{{ ONAPTEMPLATE_STANDARD_INPUTS_TYPES }} + vm_init_pgrs: type: string default: | @@ -183,7 +165,7 @@ node_templates: args: floating_network_name: { get_input: public_net } dns_pgrs00: - type: dcae.nodes.dns.arecord + type: ccsdk.nodes.dns.arecord properties: fqdn: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '00.', { get_input: location_domain } ] } openstack: *open_conf @@ -285,7 +267,7 @@ node_templates: args: floating_network_name: { get_input: public_net } dns_pgrs01: - type: dcae.nodes.dns.arecord + type: ccsdk.nodes.dns.arecord properties: fqdn: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '01.', { get_input: location_domain } ] } openstack: *open_conf @@ -389,7 +371,7 @@ node_templates: # CNAME records dns_pgrs_rw: - type: dcae.nodes.dns.cnamerecord + type: ccsdk.nodes.dns.cnamerecord properties: fqdn: { concat: [ { get_input: location_prefix }, '-', { get_input: pgaas_cluster_name }, '-write.', { get_input: location_domain } ] } openstack: *open_conf @@ -401,7 +383,7 @@ node_templates: cname: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '00.', { get_input: location_domain } ] } dns_pgrs_ro: - type: dcae.nodes.dns.cnamerecord + type: ccsdk.nodes.dns.cnamerecord properties: fqdn: { concat: [ { get_input: location_prefix }, '-', { get_input: pgaas_cluster_name }, '.', { get_input: location_domain } ] } openstack: *open_conf @@ -437,5 +419,5 @@ outputs: value: { get_property: [ dns_pgrs00, fqdn ] } dns_pgrs01: value: { get_property: [ dns_pgrs01, fqdn ] } - version: + blueprint_version: value: { get_input: blueprint_version } diff --git a/blueprints/pgaas-database.yaml-template b/blueprints/pgaas-database.yaml-template index 6ee5297..e1b7450 100644 --- a/blueprints/pgaas-database.yaml-template +++ b/blueprints/pgaas-database.yaml-template @@ -38,29 +38,9 @@ imports: - "{{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }}/type_files/pgaas/pgaas_types.yaml" inputs: - # environment info blueprint_version: type: string default: 'VERSION-STAMP-GOES-HERE' - flavor_id: - type: string - key_filename: - type: string - keypair: - type: string - location_domain: - type: string - location_prefix: - type: string - openstack: {} - private_net: - type: string - public_net: - type: string - security_group: - type: string - ubuntu1604image_id: - type: string # pgaas-specific info pgaas_cluster_name: @@ -75,7 +55,9 @@ inputs: database_name: type: string default: sample - + +{{ ONAPTEMPLATE_STANDARD_INPUTS_TYPES }} + node_templates: pgaasdbtest: type: dcae.nodes.pgaas.database diff --git a/blueprints/pgaas-disk.yaml-template b/blueprints/pgaas-disk.yaml-template index 7fbe790..fc97893 100644 --- a/blueprints/pgaas-disk.yaml-template +++ b/blueprints/pgaas-disk.yaml-template @@ -44,29 +44,9 @@ imports: - "{{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }}/type_files/pgaas/pgaas_types.yaml" inputs: - # environment info blueprint_version: type: string default: 'VERSION-STAMP-GOES-HERE' - flavor_id: - type: string - key_filename: - type: string - keypair: - type: string - location_domain: - type: string - location_prefix: - type: string - openstack: {} - private_net: - type: string - public_net: - type: string - security_group: - type: string - ubuntu1604image_id: - type: string # pgaas-specific info pgaas_cluster_name: @@ -76,6 +56,8 @@ inputs: type: string default: 300 +{{ ONAPTEMPLATE_STANDARD_INPUTS_TYPES }} + node_templates: key_pair: type: cloudify.openstack.nodes.KeyPair @@ -87,7 +69,7 @@ node_templates: get_input: openstack sharedsshkey_pgrs: - type: dcae.nodes.ssh.keypair + type: ccsdk.nodes.ssh.keypair private_net: type: cloudify.openstack.nodes.Network @@ -137,5 +119,5 @@ node_templates: target: sharedsshkey_pgrs outputs: - version: + blueprint_version: value: { get_input: blueprint_version } diff --git a/blueprints/pgaas.yaml-template b/blueprints/pgaas.yaml-template index 16e3496..5dbf932 100644 --- a/blueprints/pgaas.yaml-template +++ b/blueprints/pgaas.yaml-template @@ -35,35 +35,17 @@ imports: - "{{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }}/type_files/pgaas/pgaas_types.yaml" inputs: - # environment info blueprint_version: type: string default: 'VERSION-STAMP-GOES-HERE' - flavor_id: - type: string - key_filename: - type: string - keypair: - type: string - location_domain: - type: string - location_prefix: - type: string - openstack: {} - private_net: - type: string - public_net: - type: string - security_group: - type: string - ubuntu1604image_id: - type: string # pgaas-specific info pgaas_cluster_name: type: string default: pstg +{{ ONAPTEMPLATE_STANDARD_INPUTS_TYPES }} + vm_init_pgrs: type: string default: | @@ -155,7 +137,7 @@ node_templates: openstack_config: *open_conf sharedsshkey_pgrs: - type: dcae.nodes.ssh.keypair + type: ccsdk.nodes.ssh.keypair fixedip_pgrs00: type: cloudify.openstack.nodes.Port @@ -179,7 +161,7 @@ node_templates: args: floating_network_name: { get_input: public_net } dns_pgrs00: - type: dcae.nodes.dns.arecord + type: ccsdk.nodes.dns.arecord properties: fqdn: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '00.', { get_input: location_domain } ] } openstack: *open_conf @@ -281,7 +263,7 @@ node_templates: args: floating_network_name: { get_input: public_net } dns_pgrs01: - type: dcae.nodes.dns.arecord + type: ccsdk.nodes.dns.arecord properties: fqdn: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '01.', { get_input: location_domain } ] } openstack: *open_conf @@ -364,7 +346,7 @@ node_templates: # CNAME records dns_pgrs_rw: - type: dcae.nodes.dns.cnamerecord + type: ccsdk.nodes.dns.cnamerecord properties: fqdn: { concat: [ { get_input: location_prefix }, '-', { get_input: pgaas_cluster_name }, '-write.', { get_input: location_domain } ] } openstack: *open_conf @@ -376,7 +358,7 @@ node_templates: cname: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '00.', { get_input: location_domain } ] } dns_pgrs_ro: - type: dcae.nodes.dns.cnamerecord + type: ccsdk.nodes.dns.cnamerecord properties: fqdn: { concat: [ { get_input: location_prefix }, '-', { get_input: pgaas_cluster_name }, '.', { get_input: location_domain } ] } openstack: *open_conf @@ -414,5 +396,5 @@ outputs: value: { get_property: [ dns_pgrs00, fqdn ] } dns_pgrs01: value: { get_property: [ dns_pgrs01, fqdn ] } - version: + blueprint_version: value: { get_input: blueprint_version } diff --git a/check-blueprint-vs-input/lib/sample-inputs.yaml b/check-blueprint-vs-input/lib/sample-inputs.yaml index 48e5e24..6e888b0 100644 --- a/check-blueprint-vs-input/lib/sample-inputs.yaml +++ b/check-blueprint-vs-input/lib/sample-inputs.yaml @@ -1,5 +1,5 @@ -centos7image_id: '7c8d7524-de1f-490b-8418-db294bfa2d65' -ubuntu1604image_id: '4b09c18b-d69e-4ba8-a1bd-562cab91ff20' +centos7image_id: 'some-value-in-guid-format' +ubuntu1604image_id: 'another-value-in-guid-format' flavor_id: '4' security_group: '55a11193-6559-4f6c-b2d2-0119a9817062' public_net: 'admin_floating_228_net' diff --git a/mvn-phase-lib.sh b/mvn-phase-lib.sh index a71d316..1db6fd8 100755 --- a/mvn-phase-lib.sh +++ b/mvn-phase-lib.sh @@ -101,6 +101,7 @@ clean_tox_files() expand_templates() { + set +x # set up env variables, get ready for template resolution # NOTE: CCSDK artifacts do not distinguish REALESE vs SNAPSHOTs export ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases="$MVN_RAWREPO_BASEURL_DOWNLOAD/org.onap.ccsdk.platform.plugins" @@ -116,64 +117,60 @@ expand_templates() export ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2_platform_blueprints_snapshots="$MVN_RAWREPO_BASEURL_DOWNLOAD/org.onap.dcaegen2.platform.blueprints/snapshots" export ONAPTEMPLATE_PYPIURL_org_onap_dcaegen2="${MVN_PYPISERVER_BASEURL}" + export ONAPTEMPLATE_STANDARD_INPUTS_TYPES=" # standard inputs list + centos7image_id: + type: string + ubuntu1604image_id: + type: string + flavor_id: + type: string + security_group: + type: string + public_net: + type: string + private_net: + type: string + openstack: {} + keypair: + type: string + key_filename: + type: string + location_prefix: + type: string + location_domain: + type: string + codesource_url: + type: string + codesource_version: + type: string" # docker registry templates are for poll, so use PUBLIC registry export ONAPTEMPLATE_DOCKERREGURL_org_onap_dcaegen2_releases="$MVN_DOCKERREGISTRY_PUBLIC" export ONAPTEMPLATE_DOCKERREGURL_org_onap_dcaegen2_snapshots="${MVN_DOCKERREGISTRY_PUBLIC}/snapshots" + TEMPLATES=$(env |grep ONAPTEMPLATE | sed 's/=.*//' | sort -u) + if [ -z "$TEMPLATES" ]; then + echo "No template variables found!" + return 0 + fi TEMPLATE_FILES=$(find . -name "*-template") for F in $TEMPLATE_FILES; do F2=$(echo "$F" | sed 's/-template$//') - cp "$F" "$F2" - MOD=$(stat --format '%a' "$F") - chmod "$MOD" "$F2" - done + cp -p "$F" "$F2" + chmod u+w "$F2" - - TEMPLATES=$(env |grep ONAPTEMPLATE) - if [ -z "$TEMPLATES" ]; then - return 0 - fi - - echo "====> Resolving the following temaplate from environment variables " - echo "[$TEMPLATES]" - SELFFILE=$(echo "$0" | rev | cut -f1 -d '/' | rev) - for TEMPLATE in $TEMPLATES; do - KEY=$(echo "$TEMPLATE" | cut -f1 -d'=') - VALUE=$(echo "$TEMPLATE" | cut -f2 -d'=') - VALUE2=$(echo "$TEMPLATE" | cut -f2 -d'=' |sed 's/\//\\\//g') - set +e - FILES=$(grep -rl "$KEY") - set -e - - if [ -z "$FILES" ]; then - continue - fi - - # assuming FILES is not longer than 2M bytes, the limit for variable value max size on this VM - for F in $FILES; do - if [[ $F == *"$SELFFILE" ]]; then - continue - fi - if [[ "$F" == *-template ]]; then - continue - fi - - echo "======> Resolving template $KEY to value $VALUE for file $F" - sed -i "s/{{[[:space:]]*$KEY[[:space:]]*}}/$VALUE2/g" "$F" - #cat "$F" + echo "====> Resolving the following template from environment variables " + echo "$TEMPLATES" + for KEY in $TEMPLATES; do + VALUE1=$(eval 'echo "$"'"$KEY"'"' | sed 1q) + VALUE2=$(eval 'echo "$'"$KEY"'"' | sed -e 's/\//\\\//g' -e 's/$/\\/' -e '$s/\\$//') + + echo "======> Resolving template $KEY to value $VALUE1 for file $F2" + sed -i "s/{{[[:space:]]*$KEY[[:space:]]*}}/$VALUE2/g" "$F2" done - - #if [ ! -z "$FILES" ]; then - # echo "====> Resolving template $VALUE to value $VALUE" - # #CMD="grep -rl \"$VALUE\" | tr '\n' '\0' | xargs -0 sed -i \"s/{{[[:space:]]*$VALUE[[:space:]]*}}/$VALUE/g\"" - # grep -rl "$KEY" | tr '\n' '\0' | xargs -0 sed -i 's/$KEY/$VALUE2/g' - # #echo $CMD - # #eval $CMD - #fi done - echo "====> Done template reolving" + echo "====> Done template resolving" } test_templates() @@ -181,6 +178,8 @@ test_templates() # make certain that the type references exist TMP=$(mktemp) trap 'rm -f $TMP' 0 1 2 3 15 + + echo Verify that all of the import URLs are correct find . -name '*-template' | sed -e 's/-template$//' | while read file do @@ -194,6 +193,14 @@ test_templates() * ) echo ">>>>>>>>>>>>>>>> $url not found <<<<<<<<<<<<<<<<" ;; esac done + + echo Verify that the inputs are correct + PATH=$PATH:$PWD/check-blueprint-vs-input/bin + find . -name '*-template' | sed -e 's/-template$//' | + while read blueprint + do + check-blueprint-vs-input -b $blueprint -i check-blueprint-vs-input/lib/sample-inputs.yaml || true + done } -- cgit 1.2.3-korg