Add Consul support to 'OneClick' deployment.

The OOM is moving to using Consul to provide health checks for all ONAP
components.  This push adds deployment of a 3 node Consul Server cluster
and a single Consul agent to the 'OneClick' deployment.

As a first step, health check scripts for the A&AI microservices have
also been included.  Support for additional ONAP components will follow.

Issue-ID: OOM-86

Change-Id: Ib63f3d8b1b745551c9ec55b6529d022b32006e9c
Signed-off-by: gfraboni <gino.fraboni@amdocs.com>

19 files changed, 383 insertions, 0 deletions

diff --git a/kubernetes/config/docker/init/config-init.sh b/kubernetes/config/docker/init/config-init.sh
index 9217d2cd68..29167bf844 100755
--- a/kubernetes/config/docker/init/config-init.sh
+++ b/kubernetes/config/docker/init/config-init.sh
@@ -51,6 +51,9 @@ mkdir -p /config-init/$NAMESPACE/aai/data-router/logs/
 mkdir -p /config-init/$NAMESPACE/mso/mariadb/data
 mkdir -p /config-init/$NAMESPACE/clamp/mariadb/data
 mkdir -p /config-init/$NAMESPACE/log/elasticsearch/data
+mkdir -p /config-init/$NAMESPACE/consul/consul-agent-config/bin
+mkdir -p /config-init/$NAMESPACE/consul/consul-agent-config/scripts
+mkdir -p /config-init/$NAMESPACE/consul/consul-server-config
 
 echo "Setting permissions to container writeable directories"
 chmod -R 777 /config-init/$NAMESPACE/sdc/logs/
@@ -73,6 +76,7 @@ echo "Substituting configuration parameters"
 
 # replace the default 'onap' namespace qualification of K8s hostnames within the config files
 find /config-init/$NAMESPACE/ -type f -exec sed -i -e "s/\.onap-/\.$NAMESPACE-/g" {} \;
+find /config-init/$NAMESPACE/ -type f -exec sed -i -e "s/kubectl -n onap/kubectl -n $NAMESPACE/g" {} \;
 # set the ubuntu 14 image
 find /config-init/$NAMESPACE/ -type f -exec sed -i -e "s/UBUNTU_14_IMAGE_NAME_HERE/$OPENSTACK_UBUNTU_14_IMAGE/g" {} \;
 # set the openstack public network uuid
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-data-router-health.json b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-data-router-health.json
new file mode 100644
index 0000000000..a60203694a
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-data-router-health.json
@@ -0,0 +1,14 @@
+{
+  "service": {
+    "name": "A&AI Synapse Data Routing Service",
+    "checks": [
+      {
+        "id": "data-router-process",
+        "name": "Synapse Presence",
+        "script": "/consul/config/scripts/data-router-script.sh",
+        "interval": "15s",
+        "timeout": "1s"
+      }
+    ]
+  }
+}
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-model-loader-health.json b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-model-loader-health.json
new file mode 100644
index 0000000000..4e2e305afd
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-model-loader-health.json
@@ -0,0 +1,14 @@
+{
+  "service": {
+    "name": "A&AI Model Loader",
+    "checks": [
+      {
+        "id": "model-loader-process",
+        "name": "Model Loader Presence",
+        "script": "/consul/config/scripts/model-loader-script.sh",
+        "interval": "15s",
+        "timeout": "1s"
+      }
+    ]
+  }
+}
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-search-data-service-health.json b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-search-data-service-health.json
new file mode 100644
index 0000000000..c74fe8af26
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-search-data-service-health.json
@@ -0,0 +1,33 @@
+{
+  "service": {
+    "name": "A&AI Search Data Service",
+    "checks": [
+      {
+        "id": "elasticsearch",
+        "name": "Search Data Service Document Store",
+        "http": "http://elasticsearch.onap-aai:9200/_cat/indices?v",
+        "interval": "15s",
+        "timeout": "1s"
+      },
+      {
+        "id": "elasticsearch-write-health",
+        "name": "Search Data Service Document Store Write Test",
+        "script": "/consul/config/scripts/aai-search-storage-write-script.sh",
+        "interval": "60s"
+      },
+      {
+        "id": "search-data-service-availability",
+        "name": "Search Data Service Availability",
+        "script": "curl -k  --cert /consul/config/bin/client-cert-onap.crt.pem --cert-type PEM --key /consul/config/bin/client-cert-onap.key.pem --key-type PEM https://search-data-service.onap-aai:9509/services/search-data-service/v1/jaxrsExample/jaxrs-services/echo/up 2>&1 | grep 'Up'",
+        "interval": "15s"
+      },
+      {
+        "id": "search-data-service-api",
+        "name": "Search Data Service Operational Test",
+        "script": "/consul/config/scripts/search-data-service-availability.sh",
+        "interval": "15s",
+        "timeout": "1s"
+      }
+    ]
+  }
+}
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-services-health.json b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-services-health.json
new file mode 100644
index 0000000000..35f9371e8d
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-services-health.json
@@ -0,0 +1,53 @@
+{
+  "service": {
+    "name": "Active and Available Inventory",
+    "checks": [
+      {
+        "id": "aai-service",
+        "name": "Core A&AI",
+        "http": "https://aai-service.onap-aai:8443/aai/util/echo",
+        "header": {
+          "Authorization": ["Basic QUFJOkFBSQ=="],
+          "X-TransactionId": ["ConsulHealthCheck"],
+          "X-FromAppId": ["healthcheck"]
+        },
+        "tls_skip_verify": true,
+        "interval": "15s",
+        "timeout": "1s"
+      },
+      {
+        "id": "aai-resources",
+        "name": "Resources Microservice",
+        "http": "https://aai-resources.onap-aai:8447/aai/util/echo",
+        "header": {
+          "Authorization": ["Basic QUFJOkFBSQ=="],
+          "X-TransactionId": ["ConsulHealthCheck"],
+          "X-FromAppId": ["healthcheck"]
+        },
+        "tls_skip_verify": true,
+        "interval": "15s",
+        "timeout": "1s"
+      },
+      {
+        "id": "aai-traversal",
+        "name": "Traversal Microservice",
+        "http": "https://aai-traversal.onap-aai:8446/aai/util/echo",
+        "header": {
+          "Authorization": ["Basic QUFJOkFBSQ=="],
+          "X-TransactionId": ["ConsulHealthCheck"],
+          "X-FromAppId": ["healthcheck"]
+        },
+        "tls_skip_verify": true,
+        "interval": "15s",
+        "timeout": "1s"
+      },
+      {
+        "id": "gremlin-server",
+        "name": "Graph Data Store",
+        "script": "/consul/config/scripts/gremlin-script.sh",
+        "interval": "15s",
+        "timeout": "1s"
+      }
+    ]
+  }
+}
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-sparky-be-health.json b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-sparky-be-health.json
new file mode 100644
index 0000000000..6af58dbf4f
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-sparky-be-health.json
@@ -0,0 +1,14 @@
+{
+  "service": {
+    "name": "A&AI UI Backend Service",
+    "checks": [
+      {
+        "id": "sparky-be-process",
+        "name": "UI Backend Presence",
+        "script": "/consul/config/scripts/sparky-be-script.sh",
+        "interval": "15s",
+        "timeout": "1s"
+      }
+    ]
+  }
+}
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-tabular-backend-health.json b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-tabular-backend-health.json
new file mode 100644
index 0000000000..f76b33b3d8
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/aai-tabular-backend-health.json
@@ -0,0 +1,14 @@
+{
+  "service": {
+    "name": "A&AI Tabular Data Store",
+    "checks": [
+      {
+        "id": "tabular-backend",
+        "name": "Tabular Data Store Operational Test",
+        "script": "/consul/config/scripts/tabular-db-availability.sh",
+        "interval": "15s",
+        "timeout": "1s"
+      }
+    ]
+  }
+}
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/bin/client-cert-onap.crt.pem b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/bin/client-cert-onap.crt.pem
new file mode 100644
index 0000000000..5696aa3570
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/bin/client-cert-onap.crt.pem
@@ -0,0 +1,25 @@
+Bag Attributes
+    friendlyName: tomcat
+    localKeyID: 54 69 6D 65 20 31 34 39 33 33 32 33 39 32 32 37 35 31 
+subject=/C=CA/ST=Ontario/L=Ottawa/O=ONAP/OU=ONAP/CN=ONAP
+issuer=/C=CA/ST=Ontario/L=Ottawa/O=ONAP/OU=ONAP/CN=ONAP
+-----BEGIN CERTIFICATE-----
+MIIDWTCCAkGgAwIBAgIERWHcIzANBgkqhkiG9w0BAQsFADBdMQswCQYDVQQGEwJD
+QTEQMA4GA1UECBMHT250YXJpbzEPMA0GA1UEBxMGT3R0YXdhMQ0wCwYDVQQKEwRP
+TkFQMQ0wCwYDVQQLEwRPTkFQMQ0wCwYDVQQDEwRPTkFQMB4XDTE3MDQyNzIwMDUz
+N1oXDTM3MDExMjIwMDUzN1owXTELMAkGA1UEBhMCQ0ExEDAOBgNVBAgTB09udGFy
+aW8xDzANBgNVBAcTBk90dGF3YTENMAsGA1UEChMET05BUDENMAsGA1UECxMET05B
+UDENMAsGA1UEAxMET05BUDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AJsQpjB5U0exZHWKVt6xDzmBBhLiAtv7Qb8zsbAcIZPxuKsieOJykWDCaf+Ip7oe
++b86nf4LmKrNm4KMsDNnlU7Bg7+3HFa7m+tZgfILORv2HPMRXgvcqPFr1dxgTBkp
+xtlcGXHhA8oBpmqTmOCitE+ngVH+FBVxN93aHEDz+Dgc06PyzoP/xWI0GjvlOsv/
+qZeXCj6K4Hpu/FSPNk06Piq9M+rDwUMuyaRtY9FWjYMvkMCrRvlZUoAasrC0BGyR
+UAboHdk5aW3AZ0cVR6NMSlELcvCUFqzacAOWLgffX3b5vhkOaAsmnnzmxANV6s0t
+SqrD6Mmjg5OcYJW4VFKrwjUCAwEAAaMhMB8wHQYDVR0OBBYEFNji+IU70Qgptn4i
+boq/rOKNAg8tMA0GCSqGSIb3DQEBCwUAA4IBAQBc5mJLeeUUzJ4MujZjn0DS3Lvv
+THJTE54Id1euT3ddzfX3htF0Ewd90YzmLuj1y8r8PXj7b/8Bq+cvoKbmJ42c8h3X
+If0tqde+gYWx1X3NAWHwz00Cje9R0KY4Bx1Cvr39jTw/ESnuSQDKPHBnn8WyAS9K
+08ZhvrVSK54d3U7tDVut9UVva8Scdi12utTAWaOIlusLo3bU9Z6t+tgg7AnQBYc0
+N9oCMbq/MACFlLSdc1J6NITYS8XHY2RS8u88eLbWkCcEEx1glYz/PMX3+V1Ow9Uy
+MjenEx8ifl96ZSOe9XsI2gl2TCaevCY/QuREu4LZB9XmO0gncH7gF5w9Bw2b
+-----END CERTIFICATE-----
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/bin/client-cert-onap.key.pem b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/bin/client-cert-onap.key.pem
new file mode 100644
index 0000000000..c7e386e55f
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/bin/client-cert-onap.key.pem
@@ -0,0 +1,32 @@
+Bag Attributes
+    friendlyName: tomcat
+    localKeyID: 54 69 6D 65 20 31 34 39 33 33 32 33 39 32 32 37 35 31 
+Key Attributes: <No Attributes>
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCbEKYweVNHsWR1
+ilbesQ85gQYS4gLb+0G/M7GwHCGT8birInjicpFgwmn/iKe6Hvm/Op3+C5iqzZuC
+jLAzZ5VOwYO/txxWu5vrWYHyCzkb9hzzEV4L3Kjxa9XcYEwZKcbZXBlx4QPKAaZq
+k5jgorRPp4FR/hQVcTfd2hxA8/g4HNOj8s6D/8ViNBo75TrL/6mXlwo+iuB6bvxU
+jzZNOj4qvTPqw8FDLsmkbWPRVo2DL5DAq0b5WVKAGrKwtARskVAG6B3ZOWltwGdH
+FUejTEpRC3LwlBas2nADli4H3192+b4ZDmgLJp585sQDVerNLUqqw+jJo4OTnGCV
+uFRSq8I1AgMBAAECggEANFs6wcM1S0+qC8XZ7vb5nQDjfByzunLrkBN0O3JEJB/J
+qn7JMixcyb7a61zIxR8QVHEGR3DC62jgyQOXusOOtjjAs0qwVtihnKVsKr1/WuGO
+hMOobXjj0iAG5ZHeH+DrMxjVvo2rKdnExtdvFunY18xG7dhMD7Fam525THUTql4K
+yxhT7X6MrfS1eFjbR6oAIGNjoNTwyyEjEm4yvHO3PnG2NeyIeu7zIO2k+GimAAXT
+tN3AK30lmr3+35k6o+XQAhDE4/6msn6jBVSdLfK35ATFGwrojD0bCgALR4SUNEyd
+i33nuNLGyeI7DPWbqmjyWQW9uWLFJD85We2HzqBZQQKBgQDIrJ4PLvYE75dFWnSa
+lBr1HZbl/x5mP56MVEiwTabRbUsJoXKlX44lm9hwQaPbuoUAflb1ZtNKbyiRVsuN
+Ft5RToU9PWXyFtc2eyLCJToxHI4MhsuGRAaEeic5+l12wdpRxl74eeXdKJK4P/iU
+8wdhSxDG2ekkj6lyye5l5iwcBwKBgQDF0Pptcs+yPCz9FRqCmHT/I4QTK1VSD6mW
+F2Yd2KEUa4aocIb+L56ghJfYR+enIe9hHmb0ulomJaLLTicZJk6ffDfaQpCFBiS7
+BirDqHX8zlnBHePrBzZPyA5EfGMLxlP4uUk4g28JMFBJaZTEXAnQLUH0mIm0o0YR
+mbsaVo/Y4wKBgFsG8iuxAaf7hoLPJVV5GUFWyrxJnWCEO0csdEyE7MbS7NbRhU++
+qJwmtWc2Xz2svegbZxaqLe31vlEvLeYyGWaIV6gP0c6ezcDI2lt2x46/hS/pdSjS
+cqJlRqXmC79y77VoZmwP31USsnshiYEHPLHFeza4YilTgWmwb5OJdTjBAoGBAJBC
+0P7UhedjvyNqKoUnDdurWPxp07Ueuvw8YDpP61jq+a8JMUlaDQLe76XI+oWGV/6p
+n0fGR0weklRV0Gmk6B2jB1BizuZUDqFd4/4ActtE2WvekoKqJc+VA+KqG8lQf5iZ
+924BXA6Fb2e6WcXBoV5yQvFP9M0JbWYUiMCydAElAoGBAKof78r8POfTPq9fQA9I
+0zsQGnxqnSqyIu5yobM3GyXHBPOKdevlxyXxuMnGTr7upSNZrDrrA+f5Czlu7Fas
+qdt/5PmqYQjRsVoHNQFatUzHWwx2vU2Pr1jBpZFBpnjnLwn3A35+UEWn13nCjkla
+TrDniEcyId4ya5cMLDnM7Zgw
+-----END PRIVATE KEY-----
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/bin/kubectl b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/bin/kubectl
new file mode 100644
index 0000000000..d53ce5f7f2
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/bin/kubectl
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/model-loader.properties b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/model-loader.properties
new file mode 100644
index 0000000000..b2db044417
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/model-loader.properties
@@ -0,0 +1,23 @@
+# Model Loader Distribution Client Configuration

+ml.distribution.ACTIVE_SERVER_TLS_AUTH=false

+ml.distribution.ASDC_ADDRESS=c2.vm1.sdc.simpledemo.openecomp.org:8443

+ml.distribution.CONSUMER_GROUP=aai-ml-group

+ml.distribution.CONSUMER_ID=aai-ml

+ml.distribution.ENVIRONMENT_NAME=AUTO

+ml.distribution.KEYSTORE_PASSWORD=

+ml.distribution.KEYSTORE_FILE=asdc-client.jks

+ml.distribution.PASSWORD=OBF:1ks51l8d1o3i1pcc1r2r1e211r391kls1pyj1z7u1njf1lx51go21hnj1y0k1mli1sop1k8o1j651vu91mxw1vun1mze1vv11j8x1k5i1sp11mjc1y161hlr1gm41m111nkj1z781pw31kku1r4p1e391r571pbm1o741l4x1ksp

+ml.distribution.POLLING_INTERVAL=30

+ml.distribution.POLLING_TIMEOUT=20

+ml.distribution.USER=aai

+ml.distribution.ARTIFACT_TYPES=MODEL_INVENTORY_PROFILE,MODEL_QUERY_SPEC,VNF_CATALOG

+

+# Model Loader AAI REST Client Configuration

+ml.aai.BASE_URL=https://c1.vm1.aai.simpledemo.openecomp.org:8443

+ml.aai.MODEL_URL=/aai/v10/service-design-and-creation/models/model/

+ml.aai.NAMED_QUERY_URL=/aai/v10/service-design-and-creation/named-queries/named-query/

+ml.aai.VNF_IMAGE_URL=/aai/v8/service-design-and-creation/vnf-images

+ml.aai.KEYSTORE_FILE=aai-os-cert.p12

+ml.aai.KEYSTORE_PASSWORD=OBF:1i9a1u2a1unz1lr61wn51wn11lss1unz1u301i6o

+ml.aai.AUTH_USER=ModelLoader

+ml.aai.AUTH_PASSWORD=OBF:1qvu1v2h1sov1sar1wfw1j7j1wg21saj1sov1v1x1qxw

diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/aai-search-storage-write-doc.txt b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/aai-search-storage-write-doc.txt
new file mode 100644
index 0000000000..a6e084cfea
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/aai-search-storage-write-doc.txt
@@ -0,0 +1,9 @@
+{
+        "vnfId" : "testwrite",
+        "device" : "10.198.1.31",
+        "timestamp" : "2017-08-23T19:13:56Z",
+        "jdmTotalMem" : "2097152",
+        "jdmAvailableMem" : "1877272",
+        "jdmUserCpu" : "16",
+        "jdmSystemCpu" : "3"
+}
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/aai-search-storage-write-script.sh b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/aai-search-storage-write-script.sh
new file mode 100644
index 0000000000..26e13913a0
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/aai-search-storage-write-script.sh
@@ -0,0 +1,17 @@
+if curl -s -X PUT http://elasticsearch.onap-aai:9200/searchhealth/stats/testwrite -d @/consul/config/scripts/aai-search-storage-write-doc.txt | grep '\"created\":true'; then
+   if curl -s -X DELETE http://elasticsearch.onap-aai:9200/searchhealth/stats/testwrite | grep '\"failed\":0'; then
+      if curl -s -X GET http://elasticsearch.onap-aai:9200/searchhealth/stats/testwrite | grep '\"found\":false'; then
+         echo Successful PUT, DELETE, GET from Search Document Storage 2>&1
+         exit 0
+      else
+         echo Failed GET from Search Document Storage 2>&1
+         exit 1
+      fi
+   else
+      echo Failed DELETE from Search Document Storage 2>&1
+      exit 1
+   fi
+else
+   echo Failed PUT from Search Document Storage 2>&1
+   exit 1
+fi
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/data-router-script.sh b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/data-router-script.sh
new file mode 100644
index 0000000000..53cd5886f9
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/data-router-script.sh
@@ -0,0 +1,16 @@
+
+NAME=$(/consul/config/bin/kubectl -n onap-aai get pod | grep -o "data-router[^[:space:]]*")
+
+if [ -n "$NAME" ]; then
+   if /consul/config/bin/kubectl -n onap-aai exec -it $NAME -- ps -efww | grep 'java' | grep 'data-router' > /dev/null; then
+
+      echo Success. Synapse process is running. 2>&1
+      exit 0
+   else
+      echo Failed. Synapse process is not running. 2>&1
+      exit 1
+   fi
+else
+   echo Failed. Synapse container is offline. 2>&1
+   exit 1
+fi
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/gremlin-script.sh b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/gremlin-script.sh
new file mode 100644
index 0000000000..c1766f8a2a
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/gremlin-script.sh
@@ -0,0 +1,16 @@
+
+NAME=$(/consul/config/bin/kubectl -n onap-aai get pod | grep -o "gremlin[^[:space:]]*")
+
+if [ -n "$NAME" ]; then
+   if /consul/config/bin/kubectl -n onap-aai exec -it $NAME -- ps -efww | grep 'java' | grep 'gremlin-server' > /dev/null; then
+
+      echo Success. Gremlin Server process is running. 2>&1
+      exit 0
+   else
+      echo Failed. Gremlin Server process is not running. 2>&1
+      exit 1
+   fi
+else
+   echo Failed. Gremlin Server container is offline. 2>&1
+   exit 1
+fi
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/model-loader-script.sh b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/model-loader-script.sh
new file mode 100644
index 0000000000..1c93ecb38e
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/model-loader-script.sh
@@ -0,0 +1,16 @@
+
+NAME=$(/consul/config/bin/kubectl -n onap-aai get pod | grep -o "model-loader[^[:space:]]*")
+
+if [ -n "$NAME" ]; then
+   if /consul/config/bin/kubectl -n onap-aai exec -it $NAME -- ps -efww | grep 'java' | grep 'model-loader' > /dev/null; then
+
+      echo Success. Model Loader process is running. 2>&1
+      exit 0
+   else
+      echo Failed. Model Loader process is not running. 2>&1
+      exit 1
+   fi
+else
+   echo Failed. Model Loader container is offline. 2>&1
+   exit 1
+fi
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/search-data-service-availability.sh b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/search-data-service-availability.sh
new file mode 100644
index 0000000000..e5cf5cfefb
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/search-data-service-availability.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+
+SEARCH_SERVICE_NAME="search-data-service.onap-aai"
+SEARCH_SERVICE_PORT=9509
+HEALTH_CHECK_INDEX="healthcheck"
+
+# 'Document Index' REST Endpoint
+INDEX_URL="https://$SEARCH_SERVICE_NAME:$SEARCH_SERVICE_PORT/services/search-data-service/v1/search/indexes/$HEALTH_CHECK_INDEX"
+INDEX_SCHEMA="{\"fields\":[{\"name\": \"field1\", \"data-type\": \"string\"}]}"
+
+
+SEARCH_CERT_FILE="/consul/config/client-cert-onap.crt.pem"
+SEARCH_KEY_FILE="/consul/config/client-cert-onap.key.pem"
+
+
+## Try to create an index via the Search Data Service API.
+CREATE_INDEX_RESP=$(curl -s -o /dev/null -w "%{http_code}" -k --cert $SEARCH_CERT_FILE --cert-type PEM --key $SEARCH_KEY_FILE --key-type PEM -d "$INDEX_SCHEMA" --header "Content-Type: application/json" --header "X-TransactionId: ConsulHealthCheck" -X PUT $INDEX_URL) 
+
+RESULT_STRING=" "
+
+if [ $CREATE_INDEX_RESP -eq 201 ]; then
+   RESULT_STRING="Service Is Able To Communicate With Back End"
+elif [ $CREATE_INDEX_RESP -eq 400 ]; then
+   # A 400 response could mean that the index already exists (ie: we didn't
+   # clean up after ourselves on a previous check), so log the response but
+   # don't exit yet.  If we fail on the delete then we can consider the
+   # check a failure, otherwise, we are good.
+   RESULT_STRING="$RESULT_STRING Create Index [FAIL - 400 (possible index already exists)] "
+else
+   RESULT_STRING="Service API Failure - $CREATE_INDEX_RESP"
+   echo $RESULT_STRING
+   exit 1
+fi
+
+## Now, clean up after ourselves.
+DELETE_INDEX_RESP=$(curl -s -o /dev/null -w "%{http_code}" -k --cert $SEARCH_CERT_FILE --cert-type PEM --key $SEARCH_KEY_FILE --key-type PEM -d "{ }" --header "Content-Type: application/json" --header "X-TransactionId: ConsulHealthCheck" -X DELETE $INDEX_URL) 
+
+if [ $DELETE_INDEX_RESP -eq 200 ]; then
+   RESULT_STRING="Service Is Able To Communicate With Back End"
+else 
+   RESULT_STRING="Service API Failure - $DELETE_INDEX_RESP"
+   echo $RESULT_STRING 
+   exit 1
+fi
+
+echo $RESULT_STRING
+return 0
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/sparky-be-script.sh b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/sparky-be-script.sh
new file mode 100644
index 0000000000..fe265ba2b0
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/sparky-be-script.sh
@@ -0,0 +1,16 @@
+
+NAME=$(/consul/config/bin/kubectl -n onap-aai get pod | grep -o "sparky-be[^[:space:]]*")
+
+if [ -n "$NAME" ]; then
+   if /consul/config/bin/kubectl -n onap-aai exec -it $NAME -- ps -efww | grep 'java' | grep 'sparky' > /dev/null; then
+
+      echo Success. UI Backend Service process is running. 2>&1
+      exit 0
+   else
+      echo Failed. UI Backend Service process is not running. 2>&1
+      exit 1
+   fi
+else
+   echo Failed. UI Backend Service container is offline. 2>&1
+   exit 1
+fi
diff --git a/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/tabular-db-availability.sh b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/tabular-db-availability.sh
new file mode 100644
index 0000000000..da9d8a5d82
--- /dev/null
+++ b/kubernetes/config/docker/init/src/config/consul/consul-agent-config/scripts/tabular-db-availability.sh
@@ -0,0 +1,20 @@
+
+# Query the Hbase service for the cluster status.
+GET_CLUSTER_STATUS_RESPONSE=$(curl -si -X GET -H "Accept: text/xml" http://hbase.onap-aai:8080/status/cluster)
+
+if [ -z "$GET_CLUSTER_STATUS_RESPONSE" ]; then
+  echo "Tabular store is unreachable."
+  return 2 
+fi
+
+# Check the resulting status JSON to see if there is a 'DeadNodes' stanza with 
+# entries.
+DEAD_NODES=$(echo $GET_CLUSTER_STATUS_RESPONSE | grep "<DeadNodes/>")
+
+if [ -n "$DEAD_NODES" ]; then
+  echo "Tabular store is up and accessible."
+  return 0
+else
+  echo "Tabular store is up but is reporting dead nodes - cluster may be in degraded state."
+  return 1
+fi