Add code to enhance k8s bootstrap

- Make bootstrap container continue to run after initial bootstrap done
- Do deployments in parallel where feasible
- Accommodate deployments with no input files
- Test to see if actions (uploads, installs, etc.) are needed before doing them

Issue-ID: DCAEGEN2-594
Change-Id: Ie188c1fd69695479593aa82b516e5504a849099c
Signed-off-by: Jack Lucas <jflucas@research.att.com>

1 files changed, 150 insertions, 34 deletions

diff --git a/k8s-bootstrap-container/bootstrap.sh b/k8s-bootstrap-container/bootstrap.sh
index 6d39404..fb092ce 100755
--- a/k8s-bootstrap-container/bootstrap.sh
+++ b/k8s-bootstrap-container/bootstrap.sh
@@ -26,8 +26,99 @@
 # 	Blueprints for components to be installed in /blueprints
 #   Input files for components to be installed in /inputs
 #   Configuration JSON files that need to be loaded into Consul in /dcae-configs
+#   Consul is installed in /opt/consul/bin/consul, with base config in /opt/consul/config/00consul.json
 
-set -ex
+### FUNCTION DEFINITIONS ###
+
+# keep_running: Keep running after bootstrap finishes or after error
+keep_running() {
+    echo $1
+    sleep infinity &
+    wait
+}
+
+# cm_hasany: Query Cloudify Manager and return 0 (true) if there are any entities matching the query
+# Used to see if something is already present on CM
+# $1 -- query fragment, for instance "plugins?archive_name=xyz.wgn" to get
+#  the number of plugins that came from the archive file "xyz.wgn"
+function cm_hasany {
+    # We use _include=id to limit the amount of data the CM sends back
+    # We rely on the "metadata.pagination.total" field in the response
+    # for the total number of matching entities
+    COUNT=$(curl -Ss -H "Tenant: default_tenant" --user admin:${CMPASS} "${CMADDR}/api/v3.1/$1&_include=id" \
+             | /bin/jq .metadata.pagination.total)
+    if (( $COUNT > 0 ))
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# deploy: Deploy components if they're not already deployed
+# $1 -- name (for bp and deployment)
+# $2 -- blueprint file name
+# $3 -- inputs file name (optional)
+function deploy {
+    # Don't crash the script on error
+    set +e
+
+    # Upload blueprint if it's not already there
+    if cm_hasany "blueprints?id=$1"
+    then
+        echo blueprint $1 is already installed on ${CMADDR}
+    else
+        cfy blueprints upload -b $1  /blueprints/$2
+    fi
+
+    # Create deployment if it doesn't already exist
+    if cm_hasany "deployments?id=$1"
+    then
+       echo deployment $1 has already been created on ${CMADDR}
+    else
+        INPUTS=
+        if [ -n "$3" ]
+        then
+            INPUTS="-i/inputs/$3"
+        fi
+        cfy deployments create -b $1 ${INPUTS} $1
+    fi
+
+    # Run the install workflow if it hasn't been run already
+    # We don't have a completely certain way of determining this.
+    # We check to see if the deployment has any node instances
+    # that are in the 'uninitialized' or 'deleted' states.  (Note that
+    # the & in the query acts as a logical OR for the multiple state values.)
+    # We'll try to install when a deployment has node instances in those states
+    if cm_hasany "node-instances?deployment_id=$1&state=uninitialized&state=deleted"
+    then
+        cfy executions start -d $1 install
+    else
+        echo deployment $1 appears to have had an install workflow executed already or is not ready for an install
+    fi
+}
+
+# Install plugin if it's not already installed
+# $1 -- path to wagon file for plugin
+function install_plugin {
+    ARCHIVE=$(basename $1)
+    # See if it's already installed
+    if cm_hasany "plugins?archive_name=$ARCHIVE"
+    then
+        echo plugin $1 already installed on ${CMADDR}
+    else
+        cfy plugin upload $1
+    fi
+}
+
+### END FUNCTION DEFINTIONS ###
+
+set -x
+
+# Make sure we keep the container alive after an error
+trap keep_running ERR
+
+set -e
 
 # Consul service registration data
 CBS_REG='{"ID": "dcae-cbs0", "Name": "config_binding_service", "Address": "config-binding-service", "Port": 10000}'
@@ -48,20 +139,35 @@ then
 fi
 PH_REG="${PH_REG}\"}"
 
-# Deploy components
-# $1 -- name (for bp and deployment)
-# $2 -- blueprint name
-# $3 -- inputs file name
-function deploy {
-    cfy install -b $1 -d $1 -i /inputs/$3 /blueprints/$2
-}
+
+
 # Set up profile to access Cloudify Manager
 cfy profiles use -u admin -t default_tenant -p "${CMPASS}"  "${CMADDR}"
 
 # Output status, for debugging purposes
 cfy status
 
-# Load configurations into Consul
+# Check Consul readiness
+# The readiness container waits for a "consul-server" container to be ready,
+# but this isn't always enough.  We need the Consul API to be up and for
+# the cluster to be formed, otherwise our Consul accesses might fail.
+# (Note in ONAP R2, we never saw a problem, but occasionally in R3 we
+# have seen Consul not be fully ready, so we add these checks, originally
+# used in the R1 HEAT-based deployment.)
+# Wait for Consul API to come up
+until curl http://${CONSUL}/v1/agent/services
+do
+    echo Waiting for Consul API
+    sleep 60
+done
+# Wait for a leader to be elected
+until [[ "$(curl -Ss http://{$CONSUL}/v1/status/leader)" != '""' ]]
+do
+    echo Waiting for leader
+    sleep 30
+done
+
+# Load configurations into Consul KV store
 for config in /dcae-configs/*.json
 do
     # The basename of the file is the Consul key
@@ -71,47 +177,57 @@ do
     curl -v -X PUT -H "Content-Type: application/json" --data-binary @/tmp/dcae-upload ${CONSUL}/v1/kv/${key}
 done
 
-# For backward compatibility, load some platform services into Consul service registry
-# Some components still rely on looking up a service in Consul
-curl -v -X PUT -H "Content-Type: application/json" --data "${CBS_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${CBS_REG1}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${CM_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${INV_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${PH_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${HE_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${HR_REG}" ${CONSUL}/v1/agent/service/register
+# Put service registrations into the local Consul configuration directory
+for sr in CBS_REG CBS_REG1 INV_REG HE_REG HR_REG CM_REG PH_REG
+do
+  echo '{"service" : ' ${!sr}  ' }'> /opt/consul/config/${sr}.json
+done
+
+# Start the local consul agent instance
+/opt/consul/bin/consul agent --config-dir /opt/consul/config 2>&1 | tee /opt/consul/consul.log &
 
 # Store the CM password into a Cloudify secret
 cfy secret create -s ${CMPASS} cmpass
 
 # Load plugins onto CM
-# Allow "already loaded" error
-# (If there are other problems, will
-# be caught in deployments.)
-set +e
 for wagon in /wagons/*.wgn
 do
-    cfy plugins upload ${wagon}
+    install_plugin ${wagon}
 done
-set -e
 
 set +e
-# (don't let failure of one stop the script.  this is likely due to image pull taking too long)
+# (Don't let failure of one stop the script.  This is likely due to image pull taking too long.)
+
 # Deploy platform components
-deploy config_binding_service k8s-config_binding_service.yaml k8s-config_binding_service-inputs.yaml
-deploy inventory k8s-inventory.yaml k8s-inventory-inputs.yaml
-deploy deployment_handler k8s-deployment_handler.yaml k8s-deployment_handler-inputs.yaml
-deploy policy_handler k8s-policy_handler.yaml k8s-policy_handler-inputs.yaml
-deploy pgaas_initdb k8s-pgaas-initdb.yaml k8s-pgaas-initdb-inputs.yaml
+# Allow for some parallelism to speed up the process.  Probably could be somewhat more aggressive.
+# config_binding_service and pgaas_initdb needed by others, but can execute in parallel
+deploy config_binding_service k8s-config_binding_service.yaml k8s-config_binding_service-inputs.yaml &
+CBS_PID=$!
+deploy pgaas_initdb k8s-pgaas-initdb.yaml k8s-pgaas-initdb-inputs.yaml &
+PG_PID=$!
+wait ${CBS_PID} ${PG_PID}
+# inventory, deployment_handler, and policy_handler can be deployed simultaneously
+deploy inventory k8s-inventory.yaml k8s-inventory-inputs.yaml &
+INV_PID=$!
+deploy deployment_handler k8s-deployment_handler.yaml k8s-deployment_handler-inputs.yaml &
+DH_PID=$!
+deploy policy_handler k8s-policy_handler.yaml k8s-policy_handler-inputs.yaml&
+PH_PID=$!
+wait ${INV_PID} ${DH_PID} ${PH_PID}
 
 # Deploy service components
-deploy tca k8s-tca.yaml k8s-tca-inputs.yaml
-deploy ves k8s-ves.yaml k8s-ves-inputs.yaml
-deploy prh k8s-prh.yaml k8s-prh-inputs.yaml
-# holmes_rules must be deployed before holmes_engine
+# tca, ves, prh can be deployed simultaneously
+deploy tca k8s-tca.yaml k8s-tca-inputs.yaml &
+deploy ves k8s-ves.yaml k8s-ves-inputs.yaml &
+deploy prh k8s-prh.yaml &
+# holmes_rules must be deployed before holmes_engine, but holmes_rules can go in parallel with other service components
 deploy holmes_rules k8s-holmes-rules.yaml k8s-holmes_rules-inputs.yaml
 deploy holmes_engine k8s-holmes-engine.yaml k8s-holmes_engine-inputs.yaml
 set -e
 
 # Display deployments, for debugging purposes
 cfy deployments list
+
+# Continue running
+keep_running "Finished bootstrap steps."
+echo "Exiting!"
\ No newline at end of file