Implement MultiCloud proxyed Designate

This submission contains implementation for supporting MultiCloud
proxy-ed Designate solution. The changes, mostly in dcae2_vm_init.sh,
are for registering Designate proxy by MultiCloud with A&AI; register
new DNS zone with the MultiCloud proxyed Designate service; configure
DCAE to use the new DNS zone, etc.
The submission for corresponding Heat template changes will be made separtely.

Issue-Id: DCAEGEN2-128
Change-Id: Ief8e91152c7d2f117aec3f21df54c1ed74ceef0f
Signed-off-by: Lusheng Ji <lji@research.att.com>

3 files changed, 557 insertions, 16 deletions

diff --git a/boot/dcae2_install.sh b/boot/dcae2_install.sh
index a03e30bd..c1dbaa7a 100644
--- a/boot/dcae2_install.sh
+++ b/boot/dcae2_install.sh
@@ -1,4 +1,20 @@
 #!/bin/bash
+#############################################################################
+#
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#############################################################################
 
 set -ex 
 
@@ -23,10 +39,11 @@ then
 fi
 
 # Download dependencies
-echo "deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu $(lsb_release -c -s) main" >>  /etc/apt/sources.list.d/java.list
-echo "deb-src http://ppa.launchpad.net/openjdk-r/ppa/ubuntu $(lsb_release -c -s) main" >>  /etc/apt/sources.list.d/java.list
+#echo "deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu $(lsb_release -c -s) main" >>  /etc/apt/sources.list.d/java.list
+#echo "deb-src http://ppa.launchpad.net/openjdk-r/ppa/ubuntu $(lsb_release -c -s) main" >>  /etc/apt/sources.list.d/java.list
 apt-get update
-apt-get install --allow-unauthenticated -y apt-transport-https ca-certificates wget make openjdk-8-jdk git ntp ntpdate python python-pip
+#apt-get install --allow-unauthenticated -y apt-transport-https ca-certificates wget make openjdk-8-jdk git ntp ntpdate python python-pip
+apt-get install --allow-unauthenticated -y apt-transport-https ca-certificates wget git ntp ntpdate python python-pip
 
 # Download scripts from Nexus
 curl -k "$NEXUS_REPO/org.onap.demo/boot/$ARTIFACTS_VERSION/dcae2_vm_init.sh" -o /opt/dcae2_vm_init.sh
@@ -80,7 +97,7 @@ sed -e 's/\\n/\n/g' /opt/config/priv_key | sed -e 's/^[ \t]*//g; s/[ \t]*$//g' >
 chmod 777 /opt/app/config/key
 
 # move keystone url file
-cp /opt/config/keystone_url.txt /opt/app/config/keystone_url.txt
+#cp /opt/config/keystone_url.txt /opt/app/config/keystone_url.txt
 
 # download blueprint input template files
 rm -rf /opt/app/inputs-templates
@@ -89,7 +106,10 @@ wget -P /opt/app/inputs-templates https://nexus.onap.org/service/local/repositor
 wget -P /opt/app/inputs-templates https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.blueprints/releases/input-templates/phinputs.yaml
 wget -P /opt/app/inputs-templates https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.blueprints/releases/input-templates/dhinputs.yaml
 wget -P /opt/app/inputs-templates https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.blueprints/releases/input-templates/invinputs.yaml
-
+wget -P /opt/app/inputs-templates https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.blueprints/releases/input-templates/vesinput.yaml
+wget -P /opt/app/inputs-templates https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.blueprints/releases/input-templates/tcainputs.yaml
+wget -P /opt/app/inputs-templates https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.blueprints/releases/input-templates/he-ip.yaml
+wget -P /opt/app/inputs-templates https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.blueprints/releases/input-templates/hr-ip.yaml
 
 # generate blueprint input files
 pip install jinja2
diff --git a/boot/dcae2_serv.sh b/boot/dcae2_serv.sh
index a86906a3..30e647e4 100644
--- a/boot/dcae2_serv.sh
+++ b/boot/dcae2_serv.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 #############################################################################
 #
-# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,13 +17,13 @@
 #############################################################################
 
 ### BEGIN INIT INFO
-# Provides:
-# Required-Start:    $remote_fs $syslog
-# Required-Stop:     $remote_fs $syslog
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
+# Provides: dcaegen2-bootstrap
+# Required-Start: $remote_fs $syslog
+# Required-Stop: $remote_fs $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
 # Short-Description: Start daemon at boot time
-# Description:       Enable service provided by daemon.
+# Description: Enable service provided by daemon.
 ### END INIT INFO
 
 dir="/opt"
diff --git a/boot/dcae2_vm_init.sh b/boot/dcae2_vm_init.sh
index 129a6d82..d5df44e4 100644
--- a/boot/dcae2_vm_init.sh
+++ b/boot/dcae2_vm_init.sh
@@ -1,24 +1,543 @@
 #!/bin/bash
+#############################################################################
+#
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#############################################################################
+
+
+# prepare a curl command
+# parameters: URL METHOD CURLOPTIONS EXTRA_HEADERS_AS_A_STRING AUTH_AS_USER:PASS DATA
+assemble_curl_command()
+{
+    local URL="$1"
+    local METHOD="$2"
+    local CURLOPTIONS="$3"
+    local EXTRA_HEADERS="$4"
+    local AUTH="$5"
+    local DATA="$6"
+    local CMD=''
+    if [ ! -z "$METHOD" ]; then
+        CMD="curl $CURLOPTIONS $METHOD"
+    else
+        CMD="curl $CURLOPTIONS -X GET"
+    fi
+    if [ ! -z "$EXTRA_HEADERS" ]; then
+        CMD="$CMD $EXTRA_HEADERS"
+    fi
+    if [ ! -z "$AUTH" ]; then
+        CMD="$CMD $AUTH"
+    fi
+    if [ ! -z "$DATA" ]; then
+        CMD="$CMD $DATA"
+    fi
+    CMD="$CMD $URL"
+    echo "$CMD"
+}
+
+
+# Make a rest API call
+# parameters: URL METHOD expected_response_code EXTRA_HEADERS_AS_A_STRING AUTH_AS_USER:PASS DATA
+call_api_for_response_code()
+{
+    local CURLOPTIONS='-kIso /dev/null -w "%{http_code}"'
+    read -r CMDF <<-END
+$(assemble_curl_command "$1" "$2" "$CURLOPTIONS" "$4" "$5" "$6")
+END
+    eval "$CMDF";
+}
+call_api_for_response_body()
+{
+    local CURLOPTIONS='-ksb'
+    read -r CMDF <<-END
+$(assemble_curl_command "$1" "$2" "$CURLOPTIONS" "$4" "$5" "$6")
+END
+    eval "$CMDF"
+}
+call_api_for_response_header()
+{  
+    local CURLOPTIONS='-ks -o /dev/null -D -'
+    read -r CMDF <<-END
+$(assemble_curl_command "$1" "$2" "$CURLOPTIONS" "$4" "$5" "$6")
+END
+    eval "$CMDF"
+}
+call_api_for_verbose()
+{ 
+    local CURLOPTIONS='-kIv'
+    read -r CMDF <<-END
+$(assemble_curl_command "$1" "$2" "$CURLOPTIONS" "$4" "$5" "$6")
+END
+    eval "$CMDF"
+    #local TFILE=$(mktemp /tmp/curlcmd.XXXXXXXXX)
+    #echo $CMD > $TFILE
+    #eval $(cat $TFILE)
+    #rm -f $TFILE
+}
+
+
+# Wait till a web service API return specified response code
+# parameters: URL METHOD EXPECTED_RESP_CODE EXTRA_HEADERS_AS_A_STRING AUTH_AS_USER:PASS DATA
+wait_for_api()
+{
+    local RESP="$3" 
+    local ACTUALRESP
+    ACTUALRESP=$(call_api_for_response_code "$1" "$2" "$3" "$4" "$5" "$6")
+    while [ "$ACTUALRESP" != "$RESP" ]; do
+        echo "RESP CODE $ACTUALRESP, not as expected RESP CODE $RESP @ $(date)."
+        sleep 30
+        ACTUALRESP=$(call_api_for_response_code "$1" "$2" "$3" "$4" "$5" "$6")
+    done
+    echo "RESP CODE $ACTUALRESP, matches with expected RESP CODE $RESP."
+}
+
+# Wait till a TCP port is open
+# parameters: HOST PORT
+wait_for_tcp_port()
+{  
+    local DEST="$1"
+    local PORT="$2"
+    while ! nc -z -w 1 "$DEST" "$PORT"; do
+        sleep 4
+        echo '.'
+    done
+}
+
+
+
+
+wait_for_aai_ready()
+{
+    # wait till A&AI up and ready
+    local AAIHOST
+    AAIHOST=$(cat /opt/config/aai1_ip_addr.txt)
+    local AAIURL="https://$AAIHOST:8443/aai/v11/examples/cloud-regions"
+    local AAIMETHOD='-X GET'
+    local AAIRESP='200'
+    local AAIHEADERS='-H "X-FromAppId: AAI-Temp-Tool" -H "X-TransactionId: AAI-Temp-Tool" -H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
+    local AAIAUTH='-u AAI:AAI'
+    local AAIDATA=''
+    echo "===> Waiting for A&AI to get ready for getting $AAIRESP from $AAIURL @ $(date)"
+    wait_for_api "$AAIURL" "$AAIMETHOD" "$AAIRESP" "$AAIHEADERS" "$AAIAUTH" "$AAIDATA"
+    echo "===> A&AI ready @ $(date)"
+}
+
+
+
+wait_for_multicloud_ready()
+{
+    # wait till MultiCloud up and ready
+    local MCHOST
+    MCHOST=$(cat /opt/config/openo_ip_addr.txt)
+    local MCURL="http://$MCHOST:9005/api/multicloud-titanium_cloud/v0/swagger.json"
+    local MCMETHOD='-X GET'
+    local MCRESP='200'
+    local MCHEADERS='-H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
+    local MCAUTH=''
+    local MCDATA=''
+    echo "===> Waiting for MultiCloud to get ready for getting $MCRESP from $MCURL @ $(date)"
+    wait_for_api "$MCURL" "$MCMETHOD" "$MCRESP" "$MCHEADERS" "$MCAUTH" "$MCDATA"
+    echo "===> MultiCloud ready @ $(date)"
+}
+
+register_multicloud_pod25dns_with_aai()
+{
+    # Register MultiCloud with A&AI
+    local CLOUD_OWNER='pod25dns'
+    local CLOUD_VERSION='titanium_cloud'
+    local CLOUD_REGION
+    local CLOUD_ENV
+    local CLOUD_IDENTITY_URL
+    local DNSAAS_SERVICE_URL
+    local DNSAAS_USERNAME
+    local DNSAAS_PASSWORD
+    local DNSAAS_TENANT_ID
+
+    CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
+    CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
+    MCIP="$(cat /opt/config/openo_ip_addr.txt)"
+    CLOUD_IDENTITY_URL="http://${MCIP}/api/multicloud-titanium_cloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}/identity/v2.0"
+
+    local RESPCODE
+    DNSAAS_SERVICE_URL="$(cat /opt/config/dnsaas_keystone_url.txt)"
+    DNSAAS_USERNAME="$(cat /opt/config/dnsaas_username.txt)"
+    DNSAAS_PASSWORD="$(cat /opt/config/dnsaas_password.txt)"
+    DNSAAS_TENANT_ID="$(cat /opt/config/dnsaas_tenant_id.txt)"
+    cat >"/tmp/${CLOUD_OWNER}_${CLOUD_REGION}.json" <<EOL
+{
+    "cloud-owner" : "$CLOUD_OWNER",
+    "cloud-region-id" : "$CLOUD_REGION",
+    "cloud-region-version" : "$CLOUD_VERSION",
+    "cloud-type" : "$CLOUD_ENV",
+    "cloud-zone" : "cloud zone",
+    "complex-name" : "complex name2",
+    "identity-url": "$CLOUD_IDENTITY_URL",
+    "owner-defined-type" : "owner-defined-type",
+    "sriov-automation" : false,
+    "esr-system-info-list" : {
+        "esr-system-info" : [
+            {
+                "esr-system-info-id": "532ac032-e996-41f2-84ed-9c7a1766eb30",
+                "cloud-domain": "Default",
+                "default-tenant" : "$DNSAAS_TENANT_ID",
+                "user-name" : "$DNSAAS_USERNAME",
+                "password" : "$DNSAAS_PASSWORD",
+                "service-url" : "$DNSAAS_SERVICE_URL",
+                "ssl-cacert": "example-ssl-cacert-val-75021",
+                "ssl-insecure": true,
+                "system-name": "example-system-name-val-29071",
+                "system-type": "VIM",
+                "ip-address": "example-ip-address-val-44432",
+                "port": "example-port-val-93235",
+                "type": "example-type-val-85255",
+                "protocal": "example-protocal-val-52954",
+                "vendor": "example-vendor-val-94515",
+                "version": "example-version-val-71880"
+            }
+        ]
+    }
+}
+EOL
+
+
+    local REGHOST
+    local REGURL
+    local REGMETHOD='-X PUT'
+    local REGHEADERS='-H "X-FromAppId: AAI-Temp-Tool" -H "X-TransactionId: AAI-Temp-Tool" -H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
+    local REGRESP='201'
+    local REGAUTH='-u AAI:AAI'
+    local REGDATA
+
+    REGHOST="$(cat /opt/config/aai1_ip_addr.txt)"
+    REGURL="https://$REGHOST:8443/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/$CLOUD_OWNER/$CLOUD_REGION"
+    REGDATA="-T /tmp/${CLOUD_OWNER}_${CLOUD_REGION}.json"
+    echo "Register MultiCloud with A&AI owner $CLOUD_OWNER"
+    RESP=$(call_api_for_response_code "$REGURL" "$REGMETHOD" "$REGRESP" "$REGHEADERS" "$REGAUTH" "$REGDATA")
+    echo "RESP CODE: $RESP"
+}
+
+
+register_multicloud_pod25_with_aai()
+{ 
+    # Register MultiCloud with A&AI
+    local CLOUD_OWNER='pod25'
+    local CLOUD_VERSION='titanium_cloud'
+    local CLOUD_REGION
+    local CLOUD_ENV
+    local MCIP
+    local CLOUD_IDENTITY_URL
+    local KEYSTONE_URL
+    local USERNAME
+    local PASSWORD
+    local TENANT_ID
+
+    CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
+    CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
+    MCIP="$(cat /opt/config/openo_ip_addr.txt)"
+    CLOUD_IDENTITY_URL="http://${MCIP}/api/multicloud-titanium_cloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}/identity/v2.0"
+    KEYSTONE_URL="$(cat /opt/config/openstack_keystone_url.txt)"
+    USERNAME="$(cat /opt/config/openstack_user.txt)"
+    PASSWORD="$(cat /opt/config/openstack_password.txt)"
+    TENANT_ID="$(cat /opt/config/tenant_id.txt)"
+    cat >"/tmp/${CLOUD_OWNER}_${CLOUD_REGION}.json" <<EOL
+{ 
+    "cloud-owner" : "$CLOUD_OWNER",
+    "cloud-region-id" : "$CLOUD_REGION",
+    "cloud-region-version" : "$CLOUD_VERSION",
+    "cloud-type" : "$CLOUD_ENV",
+    "cloud-zone" : "cloud zone",
+    "complex-name" : "complex name",
+    "identity-url": "$CLOUD_IDENTITY_URL",
+    "owner-defined-type" : "owner-defined-type",
+    "sriov-automation" : false,
+    "cloud-extra-info" : "{\"epa-caps\":{\"huge_page\":\"true\",\"cpu_pinning\":\"true\",\"cpu_thread_policy\":\"true\",\"numa_aware\":\"true\",\"sriov\":\"true\",\"dpdk_vswitch\":\"true\",\"rdt\":\"false\",\"numa_locality_pci\":\"true\"},\"dns-delegate\":{\"cloud-owner\":\"pod25dns\",\"cloud-region-id\":\"RegionOne\"}}",
+    "esr-system-info-list" : {
+        "esr-system-info" : [
+            { 
+                "esr-system-info-id": "432ac032-e996-41f2-84ed-9c7a1766eb29",
+                "cloud-domain": "Default",
+                "default-tenant" : "$TENANT_ID",
+                "user-name" : "$USERNAME",
+                "password" : "$PASSWORD",
+                "service-url" : "$KEYSTONE_URL",
+                "ssl-cacert": "example-ssl-cacert-val-75021",
+                "ssl-insecure": true,
+                "system-name": "example-system-name-val-29070",
+                "system-type": "VIM",
+                "ip-address": "example-ip-address-val-44431",
+                "port": "example-port-val-93234",
+                "type": "example-type-val-85254",
+                "protocal": "example-protocal-val-52954",
+                "vendor": "example-vendor-val-94515",
+                "version": "example-version-val-71880"
+            }
+        ]
+    }
+}
+EOL
+ 
+    local REGHOST
+    local REGURL
+    local REGMETHOD='-X PUT'
+    local REGHEADERS='-H "X-FromAppId: AAI-Temp-Tool" -H "X-TransactionId: AAI-Temp-Tool" -H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
+    local REGRESP='201'
+    local REGAUTH='-u AAI:AAI'
+    local REGDATA
+ 
+    REGHOST="$(cat /opt/config/aai1_ip_addr.txt)"
+    REGURL="https://$REGHOST:8443/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/$CLOUD_OWNER/$CLOUD_REGION"
+    REGDATA="-T /tmp/${CLOUD_OWNER}_${CLOUD_REGION}.json"
+    echo "Register MultiCloud with A&AI owner $CLOUD_OWNER"
+    RESP=$(call_api_for_response_code "$REGURL" "$REGMETHOD" "$REGRESP" "$REGHEADERS" "$REGAUTH" "$REGDATA")
+    echo "RESP CODE: $RESP"
+}
+
+
+
+verify_multicloud_registration() 
+{
+    local CLOUD_OWNER='pod25'
+    local CLOUD_REGION
+    local CLOUD_VERSION='titanium_cloud'
+    local CLOUD_ENV
+    local REGHOST
+    local REGURL
+    local REGMETHOD='-X GET'
+    local REGHEADERS='-H "X-FromAppId: AAI-Temp-Tool" -H "X-TransactionId: AAI-Temp-Tool" -H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
+    local REGRESP='200'
+    local REGAUTH='-u AAI:AAI'
+    local REGDATA=''
+    local RESPCODE
+
+    CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
+    CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
+    REGHOST="$(cat /opt/config/aai1_ip_addr.txt)"
+
+    REGURL="https://$REGHOST:8443/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/${CLOUD_OWNER}/${CLOUD_REGION}?depth=all"
+    # Verify MultiCloud with A&AI
+    RESPCODE=$(call_api_for_response_code "$REGURL" "$REGMETHOD" "$REGRESP" "$REGHEADERS" "$REGAUTH" "$REGDATA")
+    echo "Register MultiCloud with A&AI owner $CLOUD_OWNER verify response code: $RESPCODE"
+
+    CLOUD_OWNER='pod25dns'
+    REGURL="https://$REGHOST:8443/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/${CLOUD_OWNER}/${CLOUD_REGION}?depth=all"
+    RESPCODE=$(call_api_for_response_code "$REGURL" "$REGMETHOD" "$REGRESP" "$REGHEADERS" "$REGAUTH" "$REGDATA")
+    echo "Register MultiCloud with A&AI owner $CLOUD_OWNER verify response code: $RESPCODE"
+}
+
+
+register_dns_zone()
+{
+    local CLOUD_OWNER='pod25' 
+    local CLOUD_REGION
+    local CLOUD_VERSION='titanium_cloud'
+    local CLOUD_ENV
+    local DCAE_ZONE
+    local DNSAAS_TENANT_ID
+    local MCHOST
+    local MCURL
+    local MCMETHOD='-X POST'
+    local MCRESP='200'
+    local MCHEADERS='-H "Content-Type: application/json" -H "Accept: application/json"'
+    local MCAUTH=''
+    local MCDATA=''
+
+    ## export endpoint prefix
+    local MULTICLOUD_PLUGIN_ENDPOINT
+
+    CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
+    CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
+    if [ -z "$1" ]; then DCAE_ZONE="$(cat /opt/config/dcae_zone.txt)"; else DCAE_ZONE="$1"; fi
+    DNSAAS_TENANT_ID="$(cat /opt/config/dnsaas_tenant_id.txt)"
+    MCHOST=$(cat /opt/config/openo_ip_addr.txt)
+    MCURL="http://$MCHOST:9005/api/multicloud-titanium_cloud/v0/swagger.json"
+
+    MULTICLOUD_PLUGIN_ENDPOINT=http://${MCHOST}/api/multicloud-titanium_cloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}
+    MULTICLOUD_PLUGIN_ENDPOINT=http://${MCHOST}:9005/api/multicloud-titanium_cloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}
+
+
+    ### Get Token
+    local TOKEN
+
+    MCURL="${MULTICLOUD_PLUGIN_ENDPOINT}/identity/v3/auth/tokens"
+    TOKEN=$(call_api_for_response_header "$MCURL" "$MCMETHOD" "$MCRESP" "$MCHEADERS" "$MCAUTH" "$MCDATA" | grep 'X-Subject-Token' | sed "s/^.*: //")
+    #TOKEN=$(curl -v -s -H "Content-Type: application/json" -X POST -d "{\"tenantName\": \"${DNSAAS_TENANT_ID}\"}"  "${MULTICLOUD_PLUGIN_ENDPOINT}/identity/v3/auth/tokens" 2>&1 | grep X-Subject-Token | sed "s/^.*: //")
+    echo "Received Keystone token $TOKEN from $MCURL"
+
+    ### zone operations
+    local ZONENAME
+    ZONENAME="${DCAE_ZONE}.dcaeg2.simpledemo.onap.org."
+
+    ### list zones
+    curl -sv -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones"
+
+    ### create a zone
+    echo "Creating zone $ZONENAME"
+    curl -sv -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X POST -d "{ \"name\": \"$ZONENAME\", \"email\": \"lji@research.att.com\"}" "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones"
+
+    ### query the zone with zone name
+    curl -s -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones?name=${ZONENAME}"
+
+    ### export ZONE id
+    local ZONEID
+    ZONEID=$(curl -v -s  -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones?name=${ZONENAME}" |sed 's/^.*"id":"\([a-zA-Z0-9-]*\)",.*$/\1/')
+    echo "After creation, zone $ZONENAME ID is $ZONEID"
+
+    ### query the zone with zone id
+    echo "Test listing zone info for zone $ZONENAME"
+    curl -sv -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones/${ZONEID}"
+}
+
+
+delete_dns_zone()
+{
+    local CLOUD_OWNER='pod25'
+    local CLOUD_REGION
+    local CLOUD_VERSION='titanium_cloud'
+    local CLOUD_ENV
+    local DCAE_ZONE
+    local DNSAAS_TENANT_ID
+    local MCHOST
+    local MCURL
+    local MCMETHOD='-X GET'
+    local MCRESP='200'
+    local MCHEADERS='-H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
+    local MCAUTH=''
+    local MCDATA=''
+    local MULTICLOUD_PLUGIN_ENDPOINT
+
+    CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
+    CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
+    DCAE_ZONE="$(cat /opt/config/dcae_zone.txt)"
+    DNSAAS_TENANT_ID="$(cat /opt/config/dnsaas_tenant_id.txt)"
+    MCHOST=$(cat /opt/config/openo_ip_addr.txt)
+    MCURL="http://$MCHOST:9005/api/multicloud-titanium_cloud/v0/swagger.json"
+
+    MULTICLOUD_PLUGIN_ENDPOINT=http://${MCHOST}/api/multicloud-titanium_cloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}
+    MULTICLOUD_PLUGIN_ENDPOINT=http://${MCHOST}:9005/api/multicloud-titanium_cloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}
+
+    ### Get Token
+    local TOKEN
+    TOKEN=$(curl -v -s -H "Content-Type: application/json" -X POST -d "{\"tenantName\": \"${DNSAAS_TENANT_ID}\"}"  "${MULTICLOUD_PLUGIN_ENDPOINT}/identity/v3/auth/tokens"  2>&1 | grep X-Subject-Token | sed "s/^.*: //")
+
+    local ZONENAME
+    ZONENAME="$1.dcae.simpledemo.onap.org."
+    local ZONEID
+    ZONEID=$(curl -v -s  -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones?name=${ZONENAME}" |sed 's/^.*"id":"\([a-zA-Z0-9-]*\)",.*$/\1/')
+
+    curl -s -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X DELETE "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones/${ZONEID}"
+}
+
+list_dns_zone() 
+{
+    local CLOUD_OWNER='pod25'
+    local CLOUD_REGION
+    local CLOUD_VERSION='titanium_cloud'
+    local CLOUD_ENV
+    local DCAE_ZONE
+    local DNSAAS_TENANT_ID
+    local MCHOST
+    local MCURL
+    local MCMETHOD='-X GET'
+    local MCRESP='200'
+    local MCHEADERS='-H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
+    local MCAUTH=''
+    local MCDATA=''
+    local MULTICLOUD_PLUGIN_ENDPOINT
+
+    CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
+    CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
+    DCAE_ZONE="$(cat /opt/config/dcae_zone.txt)"
+    DNSAAS_TENANT_ID="$(cat /opt/config/dnsaas_tenant_id.txt)"
+    MCHOST=$(cat /opt/config/openo_ip_addr.txt)
+    MCURL="http://$MCHOST:9005/api/multicloud-titanium_cloud/v0/swagger.json"
+
+    MULTICLOUD_PLUGIN_ENDPOINT=http://${MCHOST}/api/multicloud-titanium_cloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}
+    MULTICLOUD_PLUGIN_ENDPOINT=http://${MCHOST}:9005/api/multicloud-titanium_cloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}
+
+    ### Get Token
+    local TOKEN
+    TOKEN=$(curl -v -s -H "Content-Type: application/json" -X POST -d "{\"tenantName\": \"${DNSAAS_TENANT_ID}\"}"  "${MULTICLOUD_PLUGIN_ENDPOINT}/identity/v3/auth/tokens"  2>&1 | grep X-Subject-Token | sed "s/^.*: //")
+
+    local ZONENAME
+    ZONENAME="$1.dcae.simpledemo.onap.org."
+    local ZONEID
+    ZONEID=$(curl -v -s  -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones?name=${ZONENAME}" |sed 's/^.*"id":"\([a-zA-Z0-9-]*\)",.*$/\1/')
+
+    curl -v -s  -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones/${ZONEID}/recordsets"
+}
+
+
+
+
 
 NEXUS_USER=$(cat /opt/config/nexus_username.txt)
 NEXUS_PASSWORD=$(cat /opt/config/nexus_password.txt)
 NEXUS_DOCKER_REPO=$(cat /opt/config/nexus_docker_repo.txt)
 DOCKER_VERSION=$(cat /opt/config/docker_version.txt)
 ZONE=$(cat /opt/config/dcae_zone.txt)
+RANDSTR=$(cat /opt/config/rand_str.txt)
+MYFLOATIP=$(cat /opt/config/dcae_float_ip.txt)
+MYLOCALIP=$(cat /opt/config/dcae_ip_addr.txt)
+TENANTNAME=$(cat /opt/config/tenant_name.txt)
+OSUSERNAME=$(cat /opt/config/openstack_user.txt)
+OSPASSWORD=$(cat /opt/config/openstack_password.txt)
+
 
+#########################################
+# Wait for then register with A&AI
+########################################
 
+DNSAAS_PROXYED=$(tr '[:upper:]' '[:lower:]' < /opt/config/dnsaas_config_enabled.txt)
+if [ "$DNSAAS_PROXYED" == 'true' ]; then
+    echo "Using proxyed DNSaaS service, performing additional registration and configuration"
+    wait_for_aai_ready
+
+    register_multicloud_pod25_with_aai
+    register_multicloud_pod25dns_with_aai
+
+    verify_multicloud_registration
+
+    wait_for_multicloud_ready
+    register_dns_zone "$ZONE" 
+    echo "Registration and configuration for proxying DNSaaS completed."
+else
+    echo "Using proxyed DNSaaS service, performing additional registration and configuration"
+fi
+
+
+
+
+
+#########################################
+# Start DCAE Bootstrap container
+#########################################
+
+chmod 777 /opt/app/config
 rm -f /opt/config/runtime.ip.consul
 rm -f /opt/config/runtime.ip.cm
 
 
 docker login -u "$NEXUS_USER" -p "$NEXUS_PASSWORD" "$NEXUS_DOCKER_REPO"
 docker pull "$NEXUS_DOCKER_REPO/onap/org.onap.dcaegen2.deployments.bootstrap:$DOCKER_VERSION"
-#docker run -v /opt/config/priv_key:/opt/app/installer/config/key -v /opt/app/inputs.yaml:/opt/app/installer/config/inputs.yaml -e "LOCATION=$ZONE" $NEXUS_DOCKER_REPO/onap/org.onap.dcaegen2.deployments.bootstrap:$DOCKER_VERSION
 docker run -d --name boot -v /opt/app/config:/opt/app/installer/config -e "LOCATION=$ZONE" "$NEXUS_DOCKER_REPO/onap/org.onap.dcaegen2.deployments.bootstrap:$DOCKER_VERSION"
 
 
 # waiting for bootstrap to complete then starting nginx for proxying healthcheck calls
-while [ ! -f /opt/config/runtime.ip.consul ]; do sleep 30; done
+echo "Waiting for Consul to become accessible"
+while [ ! -f /opt/app/config/runtime.ip.consul ]; do echo "."; sleep 30; done
+
+
 
 # start proxy for consul's health check
 CONSULIP=$(head -1 /opt/config/runtime.ip.consul | sed 's/[[:space:]]//g')
@@ -29,8 +548,10 @@ server {
     listen 80;
     server_name dcae.simpledemo.onap.org;
     location /healthcheck {
-        proxy_pass http://"${CONSULIP}":8500/v1/health/state/passing;
+        proxy_pass http://${CONSULIP}:8500/v1/health/state/passing;
     }
 }
 EOL
-sudo docker run --name dcae-proxy -p 80:80 -v "$(pwd)/nginx.conf:/etc/nginx/conf.d/default.conf" -d nginx
+docker run --name dcae-proxy -p 8080:80 -v "$(pwd)/nginx.conf:/etc/nginx/conf.d/default.conf" -d nginx
+echo "Healthcheck API available at http://${MYFLOATIP}:8080/healthcheck"
+echo "                          or http://${MYLOCALIP}:8080/healthcheck"