fix inputs, template expander, add tests

rewrite the template expander:
1) faster
2) handle multiple line variables
3) add new ONAPTEMPLATE_STANDARD_INPUTS_TYPES variable

fix inputs directives in pgaas templates by using ONAPTEMPLATE_STANDARD_INPUTS_TYPES

add call to check-blueprint-vs-input to test_templates

Change-Id: I4ffc220d7e1f4820b271d055b5de3961ca236983
Signed-off-by: Tony Hansen <tony@att.com>
Issue-ID: DCAEGEN2-128
Signed-off-by: Tony Hansen <tony@att.com>

6 files changed, 77 insertions, 142 deletions

diff --git a/blueprints/pgaas-cluster.yaml-template b/blueprints/pgaas-cluster.yaml-template
index 59b188a..a1885f1 100644
--- a/blueprints/pgaas-cluster.yaml-template
+++ b/blueprints/pgaas-cluster.yaml-template
@@ -42,35 +42,17 @@ imports:
   - "{{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }}/type_files/pgaas/pgaas_types.yaml"
 
 inputs:
-  # environment info
   blueprint_version:
     type: string
     default: 'VERSION-STAMP-GOES-HERE'
-  flavor_id:
-    type: string
-  key_filename:
-    type: string
-  keypair:
-    type: string
-  location_domain:
-    type: string
-  location_prefix:
-    type: string
-  openstack: {}
-  private_net:
-    type: string
-  public_net:
-    type: string
-  security_group:
-    type: string
-  ubuntu1604image_id:
-    type: string
 
   # pgaas-specific info
   pgaas_cluster_name:
     type: string
     default: pgcl
 
+{{ ONAPTEMPLATE_STANDARD_INPUTS_TYPES }}
+
   vm_init_pgrs:
     type: string
     default: |
@@ -183,7 +165,7 @@ node_templates:
             args:
               floating_network_name: { get_input: public_net }
   dns_pgrs00:
-    type: dcae.nodes.dns.arecord
+    type: ccsdk.nodes.dns.arecord
     properties:
       fqdn: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '00.', { get_input: location_domain } ] }
       openstack: *open_conf
@@ -285,7 +267,7 @@ node_templates:
             args:
               floating_network_name: { get_input: public_net }
   dns_pgrs01:
-    type: dcae.nodes.dns.arecord
+    type: ccsdk.nodes.dns.arecord
     properties:
       fqdn: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '01.', { get_input: location_domain } ] }
       openstack: *open_conf
@@ -389,7 +371,7 @@ node_templates:
 
   # CNAME records
   dns_pgrs_rw:
-    type: dcae.nodes.dns.cnamerecord
+    type: ccsdk.nodes.dns.cnamerecord
     properties:
       fqdn: { concat: [ { get_input: location_prefix }, '-', { get_input: pgaas_cluster_name }, '-write.', { get_input: location_domain } ] }
       openstack: *open_conf
@@ -401,7 +383,7 @@ node_templates:
               cname: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '00.', { get_input: location_domain } ] }
 
   dns_pgrs_ro:
-    type: dcae.nodes.dns.cnamerecord
+    type: ccsdk.nodes.dns.cnamerecord
     properties:
       fqdn: { concat: [ { get_input: location_prefix }, '-', { get_input: pgaas_cluster_name }, '.', { get_input: location_domain } ] }
       openstack: *open_conf
@@ -437,5 +419,5 @@ outputs:
     value: { get_property: [ dns_pgrs00, fqdn ] }
   dns_pgrs01:
     value: { get_property: [ dns_pgrs01, fqdn ] }
-  version:
+  blueprint_version:
     value: { get_input: blueprint_version }
diff --git a/blueprints/pgaas-database.yaml-template b/blueprints/pgaas-database.yaml-template
index 6ee5297..e1b7450 100644
--- a/blueprints/pgaas-database.yaml-template
+++ b/blueprints/pgaas-database.yaml-template
@@ -38,29 +38,9 @@ imports:
   - "{{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }}/type_files/pgaas/pgaas_types.yaml"
 
 inputs:
-  # environment info
   blueprint_version:
     type: string
     default: 'VERSION-STAMP-GOES-HERE'
-  flavor_id:
-    type: string
-  key_filename:
-    type: string
-  keypair:
-    type: string
-  location_domain:
-    type: string
-  location_prefix:
-    type: string
-  openstack: {}
-  private_net:
-    type: string
-  public_net:
-    type: string
-  security_group:
-    type: string
-  ubuntu1604image_id:
-    type: string
 
   # pgaas-specific info
   pgaas_cluster_name:
@@ -75,7 +55,9 @@ inputs:
   database_name:
     type: string
     default: sample
-  
+
+{{ ONAPTEMPLATE_STANDARD_INPUTS_TYPES }}
+
 node_templates:
   pgaasdbtest:
     type: dcae.nodes.pgaas.database
diff --git a/blueprints/pgaas-disk.yaml-template b/blueprints/pgaas-disk.yaml-template
index 7fbe790..fc97893 100644
--- a/blueprints/pgaas-disk.yaml-template
+++ b/blueprints/pgaas-disk.yaml-template
@@ -44,29 +44,9 @@ imports:
   - "{{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }}/type_files/pgaas/pgaas_types.yaml"
 
 inputs:
-  # environment info
   blueprint_version:
     type: string
     default: 'VERSION-STAMP-GOES-HERE'
-  flavor_id:
-    type: string
-  key_filename:
-    type: string
-  keypair:
-    type: string
-  location_domain:
-    type: string
-  location_prefix:
-    type: string
-  openstack: {}
-  private_net:
-    type: string
-  public_net:
-    type: string
-  security_group:
-    type: string
-  ubuntu1604image_id:
-    type: string
 
   # pgaas-specific info
   pgaas_cluster_name:
@@ -76,6 +56,8 @@ inputs:
     type: string
     default: 300
 
+{{ ONAPTEMPLATE_STANDARD_INPUTS_TYPES }}
+
 node_templates:
   key_pair:
     type: cloudify.openstack.nodes.KeyPair
@@ -87,7 +69,7 @@ node_templates:
         get_input: openstack
 
   sharedsshkey_pgrs:
-    type: dcae.nodes.ssh.keypair
+    type: ccsdk.nodes.ssh.keypair
 
   private_net:
     type: cloudify.openstack.nodes.Network
@@ -137,5 +119,5 @@ node_templates:
         target: sharedsshkey_pgrs
 
 outputs:
-  version:
+  blueprint_version:
     value: { get_input: blueprint_version }
diff --git a/blueprints/pgaas.yaml-template b/blueprints/pgaas.yaml-template
index 16e3496..5dbf932 100644
--- a/blueprints/pgaas.yaml-template
+++ b/blueprints/pgaas.yaml-template
@@ -35,35 +35,17 @@ imports:
   - "{{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }}/type_files/pgaas/pgaas_types.yaml"
 
 inputs:
-  # environment info
   blueprint_version:
     type: string
     default: 'VERSION-STAMP-GOES-HERE'
-  flavor_id:
-    type: string
-  key_filename:
-    type: string
-  keypair:
-    type: string
-  location_domain:
-    type: string
-  location_prefix:
-    type: string
-  openstack: {}
-  private_net:
-    type: string
-  public_net:
-    type: string
-  security_group:
-    type: string
-  ubuntu1604image_id:
-    type: string
 
   # pgaas-specific info
   pgaas_cluster_name:
     type: string
     default: pstg
 
+{{ ONAPTEMPLATE_STANDARD_INPUTS_TYPES }}
+
   vm_init_pgrs:
     type: string
     default: |
@@ -155,7 +137,7 @@ node_templates:
       openstack_config: *open_conf
 
   sharedsshkey_pgrs:
-    type: dcae.nodes.ssh.keypair
+    type: ccsdk.nodes.ssh.keypair
 
   fixedip_pgrs00:
     type: cloudify.openstack.nodes.Port
@@ -179,7 +161,7 @@ node_templates:
             args:
               floating_network_name: { get_input: public_net }
   dns_pgrs00:
-    type: dcae.nodes.dns.arecord
+    type: ccsdk.nodes.dns.arecord
     properties:
       fqdn: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '00.', { get_input: location_domain } ] }
       openstack: *open_conf
@@ -281,7 +263,7 @@ node_templates:
             args:
               floating_network_name: { get_input: public_net }
   dns_pgrs01:
-    type: dcae.nodes.dns.arecord
+    type: ccsdk.nodes.dns.arecord
     properties:
       fqdn: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '01.', { get_input: location_domain } ] }
       openstack: *open_conf
@@ -364,7 +346,7 @@ node_templates:
 
   # CNAME records
   dns_pgrs_rw:
-    type: dcae.nodes.dns.cnamerecord
+    type: ccsdk.nodes.dns.cnamerecord
     properties:
       fqdn: { concat: [ { get_input: location_prefix }, '-', { get_input: pgaas_cluster_name }, '-write.', { get_input: location_domain } ] }
       openstack: *open_conf
@@ -376,7 +358,7 @@ node_templates:
               cname: { concat: [ { get_input: location_prefix }, { get_input: pgaas_cluster_name }, '00.', { get_input: location_domain } ] }
 
   dns_pgrs_ro:
-    type: dcae.nodes.dns.cnamerecord
+    type: ccsdk.nodes.dns.cnamerecord
     properties:
       fqdn: { concat: [ { get_input: location_prefix }, '-', { get_input: pgaas_cluster_name }, '.', { get_input: location_domain } ] }
       openstack: *open_conf
@@ -414,5 +396,5 @@ outputs:
     value: { get_property: [ dns_pgrs00, fqdn ] }
   dns_pgrs01:
     value: { get_property: [ dns_pgrs01, fqdn ] }
-  version:
+  blueprint_version:
     value: { get_input: blueprint_version }
diff --git a/check-blueprint-vs-input/lib/sample-inputs.yaml b/check-blueprint-vs-input/lib/sample-inputs.yaml
index 48e5e24..6e888b0 100644
--- a/check-blueprint-vs-input/lib/sample-inputs.yaml
+++ b/check-blueprint-vs-input/lib/sample-inputs.yaml
@@ -1,5 +1,5 @@
-centos7image_id: '7c8d7524-de1f-490b-8418-db294bfa2d65'
-ubuntu1604image_id: '4b09c18b-d69e-4ba8-a1bd-562cab91ff20'
+centos7image_id: 'some-value-in-guid-format'
+ubuntu1604image_id: 'another-value-in-guid-format'
 flavor_id: '4'
 security_group: '55a11193-6559-4f6c-b2d2-0119a9817062'
 public_net: 'admin_floating_228_net'
diff --git a/mvn-phase-lib.sh b/mvn-phase-lib.sh
index a71d316..1db6fd8 100755
--- a/mvn-phase-lib.sh
+++ b/mvn-phase-lib.sh
@@ -101,6 +101,7 @@ clean_tox_files()
 
 expand_templates() 
 {
+  set +x
   # set up env variables, get ready for template resolution
   # NOTE: CCSDK artifacts do not distinguish REALESE vs SNAPSHOTs
   export ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases="$MVN_RAWREPO_BASEURL_DOWNLOAD/org.onap.ccsdk.platform.plugins"
@@ -116,64 +117,60 @@ expand_templates()
   export ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2_platform_blueprints_snapshots="$MVN_RAWREPO_BASEURL_DOWNLOAD/org.onap.dcaegen2.platform.blueprints/snapshots"
 
   export ONAPTEMPLATE_PYPIURL_org_onap_dcaegen2="${MVN_PYPISERVER_BASEURL}"
+  export ONAPTEMPLATE_STANDARD_INPUTS_TYPES="  # standard inputs list
+  centos7image_id:
+    type: string
+  ubuntu1604image_id:
+    type: string
+  flavor_id:
+    type: string
+  security_group:
+    type: string
+  public_net:
+    type: string
+  private_net:
+    type: string
+  openstack: {}
+  keypair:
+    type: string
+  key_filename:
+    type: string
+  location_prefix:
+    type: string
+  location_domain:
+    type: string
+  codesource_url:
+    type: string
+  codesource_version:
+    type: string"
 
   # docker registry templates are for poll, so use PUBLIC registry
   export ONAPTEMPLATE_DOCKERREGURL_org_onap_dcaegen2_releases="$MVN_DOCKERREGISTRY_PUBLIC"
   export ONAPTEMPLATE_DOCKERREGURL_org_onap_dcaegen2_snapshots="${MVN_DOCKERREGISTRY_PUBLIC}/snapshots"
 
+  TEMPLATES=$(env |grep ONAPTEMPLATE | sed 's/=.*//' | sort -u)
+  if [ -z "$TEMPLATES" ]; then
+    echo "No template variables found!"
+    return 0
+  fi
 
   TEMPLATE_FILES=$(find . -name "*-template")
   for F in $TEMPLATE_FILES; do
     F2=$(echo "$F" | sed 's/-template$//')
-    cp "$F" "$F2"
-    MOD=$(stat --format '%a' "$F")
-    chmod "$MOD" "$F2"
-  done
+    cp -p "$F" "$F2"
+    chmod u+w "$F2"
    
-
-  TEMPLATES=$(env |grep ONAPTEMPLATE)
-  if [ -z "$TEMPLATES" ]; then
-    return 0
-  fi
-
-  echo "====> Resolving the following temaplate from environment variables "
-  echo "[$TEMPLATES]"
-  SELFFILE=$(echo "$0" | rev | cut -f1 -d '/' | rev)
-  for TEMPLATE in $TEMPLATES; do
-    KEY=$(echo "$TEMPLATE" | cut -f1 -d'=')
-    VALUE=$(echo "$TEMPLATE" | cut -f2 -d'=')
-    VALUE2=$(echo "$TEMPLATE" | cut -f2 -d'=' |sed 's/\//\\\//g')
-    set +e
-    FILES=$(grep -rl "$KEY")
-    set -e
-
-    if [ -z "$FILES" ]; then
-      continue
-    fi
-
-    # assuming FILES is not longer than 2M bytes, the limit for variable value max size on this VM
-    for F in $FILES; do
-      if [[ $F == *"$SELFFILE" ]]; then
-        continue
-      fi
-      if [[ "$F" == *-template ]]; then
-        continue
-      fi
-
-      echo "======> Resolving template $KEY to value $VALUE for file $F"
-      sed -i "s/{{[[:space:]]*$KEY[[:space:]]*}}/$VALUE2/g" "$F"
-      #cat "$F"
+    echo "====> Resolving the following template from environment variables "
+    echo "$TEMPLATES"
+    for KEY in $TEMPLATES; do
+      VALUE1=$(eval 'echo "$"'"$KEY"'"' | sed 1q)
+      VALUE2=$(eval 'echo "$'"$KEY"'"' | sed -e 's/\//\\\//g' -e 's/$/\\/' -e '$s/\\$//')
+
+      echo "======> Resolving template $KEY to value $VALUE1 for file $F2"
+      sed -i "s/{{[[:space:]]*$KEY[[:space:]]*}}/$VALUE2/g" "$F2"
     done
-
-    #if [ ! -z "$FILES" ]; then
-    #   echo "====> Resolving template $VALUE to value $VALUE"
-    #   #CMD="grep -rl \"$VALUE\" | tr '\n' '\0' | xargs -0 sed -i \"s/{{[[:space:]]*$VALUE[[:space:]]*}}/$VALUE/g\""
-    #   grep -rl "$KEY" | tr '\n' '\0' | xargs -0 sed -i 's/$KEY/$VALUE2/g'
-    #   #echo $CMD
-    #   #eval $CMD
-    #fi
   done
-  echo "====> Done template reolving"
+  echo "====> Done template resolving"
 }
 
 test_templates()
@@ -181,6 +178,8 @@ test_templates()
     # make certain that the type references exist
     TMP=$(mktemp)
     trap 'rm -f $TMP' 0 1 2 3 15
+
+    echo Verify that all of the import URLs are correct
     find . -name '*-template' | sed -e 's/-template$//' |
     while read file
     do
@@ -194,6 +193,14 @@ test_templates()
 	    * ) echo ">>>>>>>>>>>>>>>> $url not found <<<<<<<<<<<<<<<<" ;;
 	esac
     done
+
+    echo Verify that the inputs are correct
+    PATH=$PATH:$PWD/check-blueprint-vs-input/bin
+    find . -name '*-template' | sed -e 's/-template$//' |
+    while read blueprint
+    do
+	check-blueprint-vs-input -b $blueprint -i check-blueprint-vs-input/lib/sample-inputs.yaml || true
+    done
 }