8 files changed, 226 insertions, 36 deletions
diff --git a/kubernetes/consul/charts/consul-server/templates/NOTES.txt b/kubernetes/consul/charts/consul-server/templates/NOTES.txt
index 2465e03634..157fe92427 100644
--- a/kubernetes/consul/charts/consul-server/templates/NOTES.txt
+++ b/kubernetes/consul/charts/consul-server/templates/NOTES.txt
@@ -1,19 +1,34 @@
+{{/*
+# Copyright © 2018 Amdocs, AT&T, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
 1. Get the application URL by running these commands:
 {{- if .Values.ingress.enabled }}
 {{- range .Values.ingress.hosts }}
   http://{{ . }}
 {{- end }}
 {{- else if contains "NodePort" .Values.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ include "common.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "common.fullname" . }})
+  export NODE_PORT=$(kubectl get --namespace {{ include "common.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "common.name" . }})
   export NODE_IP=$(kubectl get nodes --namespace {{ include "common.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}")
   echo http://$NODE_IP:$NODE_PORT
 {{- else if contains "LoadBalancer" .Values.service.type }}
      NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get svc -w {{ include "common.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ include "common.namespace" . }} {{ include "common.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+           You can watch the status of by running 'kubectl get svc -w {{ include "common.name" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ include "common.namespace" . }} {{ include "common.name" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
   echo http://$SERVICE_IP:{{ .Values.service.externalPort }}
 {{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ include "common.namespace" . }} -l "app={{ template "so.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export POD_NAME=$(kubectl get pods --namespace {{ include "common.namespace" . }} -l "app={{ include "common.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
   echo "Visit http://127.0.0.1:8080 to use your application"
   kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }}
 {{- end }}
diff --git a/kubernetes/consul/charts/consul-server/templates/service.yaml b/kubernetes/consul/charts/consul-server/templates/service.yaml
index c79662b254..c0f8726826 100644
--- a/kubernetes/consul/charts/consul-server/templates/service.yaml
+++ b/kubernetes/consul/charts/consul-server/templates/service.yaml
@@ -25,14 +25,39 @@ metadata:
 spec:
   type: {{ .Values.service.type }}
   ports:
-    - port: {{ .Values.service.internalPort }}
-      targetPort: {{ .Values.service.internalPort }}
-      nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }}
-      name: {{ .Values.service.portName }}
-    - port: {{ .Values.service.internalPort2 }}
-      targetPort: {{ .Values.service.internalPort2 }}
-      nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }}
-      name: {{ .Values.service.portName2 }}
+  {{if eq .Values.service.type "NodePort" -}}
+  - port: {{ .Values.service.internalPort }}
+    nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }}
+    name: {{ .Values.service.portName }}
+  {{- else -}}
+  - port: {{ .Values.service.internalPort }}
+    name: {{ .Values.service.portName }}
+  {{- end}}
   selector:
-    app: {{ template "common.name" . }}
+    app: {{ include "common.name" . }}
+    release: {{ .Release.Name }}
+  clusterIP: None
+---
+kind: Service
+metadata:
+  name: {{ include "common.servicename" . }}-ui
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  type: {{ .Values.service.type2 }}
+  ports:
+  {{if eq .Values.service.type2 "NodePort" -}}
+  - port: {{ .Values.service.internalPort2 }}
+    nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }}
+    name: {{ .Values.service.portName2 }}
+  {{- else -}}
+  - port: {{ .Values.service.internalPort2 }}
+    name: {{ .Values.service.portName2 }}
+  {{- end}}
+  selector:
+    app: {{ include "common.name" . }}
     release: {{ .Release.Name }}
diff --git a/kubernetes/consul/charts/consul-server/templates/deployment.yaml b/kubernetes/consul/charts/consul-server/templates/statefulset.yaml
index 8272d3c01a..bce316354b 100644
--- a/kubernetes/consul/charts/consul-server/templates/deployment.yaml
+++ b/kubernetes/consul/charts/consul-server/templates/statefulset.yaml
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-apiVersion: extensions/v1beta1
-kind: Deployment
+apiVersion: apps/v1beta1
+kind: StatefulSet
 metadata:
   name: {{ include "common.fullname" . }}
   namespace: {{ include "common.namespace" . }}
@@ -23,6 +23,7 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
+  serviceName: {{ include "common.servicename" . }}
   replicas: {{ .Values.replicaCount }}
   selector:
     matchLabels:
@@ -32,12 +33,37 @@ spec:
       labels:
         app: {{ include "common.name" . }}
         release: {{ .Release.Name }}
-      name: {{ include "common.name" . }}
     spec:
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
       containers:
-      - image: "{{ include "common.repository" . }}/{{ .Values.image }}"
+      - name: {{ include "common.name" . }}
+        image: "{{ include "common.repository" . }}/{{ .Values.image }}"
         command: ["/usr/local/bin/docker-entrypoint.sh"]
-        args: ["agent","-server","-client","0.0.0.0","-enable-script-checks","-bootstrap-expect={{ .Values.replicaCount }}","-ui"]
-        name: {{ include "common.name" . }}
+        args:
+        - "agent"
+        - "-bootstrap-expect={{ .Values.replicaCount }}"
+        - "-enable-script-checks"
+{{- $fullname := include "common.fullname" . -}}
+{{- $servname := include "common.servicename" . -}}
+{{- range $i,$t := until (int .Values.replicaCount)}}
+        - "-retry-join={{ $fullname }}-{{$i}}.{{ $servname }}"
+{{- end }}
+        - "-client=0.0.0.0"
+        - "-server"
+        - "-ui"
+        ports:
+        - containerPort: {{ .Values.service.internalPort }}
+        - containerPort: {{ .Values.service.internalPort2 }}
+        {{- if eq .Values.liveness.enabled true }}
+        livenessProbe:
+          tcpSocket:
+            port: {{ .Values.service.internalPort }}
+          initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
+          periodSeconds: {{ .Values.liveness.periodSeconds }}
+        {{ end -}}
+        readinessProbe:
+          tcpSocket:
+            port: {{ .Values.service.internalPort }}
+          initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
+          periodSeconds: {{ .Values.readiness.periodSeconds }}
diff --git a/kubernetes/consul/charts/consul-server/values.yaml b/kubernetes/consul/charts/consul-server/values.yaml
index b214bd58b0..671cf07c9d 100644
--- a/kubernetes/consul/charts/consul-server/values.yaml
+++ b/kubernetes/consul/charts/consul-server/values.yaml
@@ -32,7 +32,7 @@ pullPolicy: Always
 # flag to enable debugging - application support required
 debugEnabled: false
 
-replicaCount: 1
+replicaCount: 3
 
 nodeSelector: {}
 
@@ -40,25 +40,25 @@ affinity: {}
 
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 90
-  periodSeconds: 10
+  initialDelaySeconds: 10
+  periodSeconds: 5
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
   enabled: true
 
 readiness:
-  initialDelaySeconds: 90
-  periodSeconds: 10
+  initialDelaySeconds: 10
+  periodSeconds: 5
 
 service:
-  type: NodePort
+  type: ClusterIP
   name: consul-server
-  portName: consul-ui
-  nodePort: 70
-  internalPort: 8500
-  portName2: consul-join
-  nodePort2: 71
-  internalPort2: 8301
+  portName: consul-join
+  internalPort: 8301
+  type2: NodePort
+  portName2: consul-ui
+  internalPort2: 8500
+  nodePort2: 70
 
 ingress:
   enabled: false
diff --git a/kubernetes/consul/resources/config/consul-agent-config/policy-health.json b/kubernetes/consul/resources/config/consul-agent-config/policy-health.json
new file mode 100644
index 0000000000..22d135b6dd
--- /dev/null
+++ b/kubernetes/consul/resources/config/consul-agent-config/policy-health.json
@@ -0,0 +1,111 @@
+{
+  "service": {
+  "name": "Health Check: Policy",
+  "checks": [
+  {
+    "id": "Policy-mariadb-healthcheck",
+    "name": "Policy Mariadb Health Check",
+    "script": "/consul/scripts/policy-mariadb-script.sh",
+    "interval": "10s",
+    "timeout": "1s"
+  },
+  {
+    "id": "policy-nexus-local-status",
+    "name": "Policy Nexus Local Status",
+    "http": "http://nexus:8081/nexus/service/local/status?pretty",
+    "method": "GET",
+    "header": {
+      "Authorization": ["Basic YWRtaW46YWRtaW4xMjM="],
+      "Cache-Control": ["no-cache"],
+      "Content-Type": ["application/json"],
+      "Accept": ["application/json"]
+    },
+    "tls_skip_verify": true,
+    "interval": "15s",
+    "timeout": "1s"
+  },
+  {
+    "id": "policy-nexus-internal-metrics",
+    "name": "Policy Nexus Internal Metrics",
+    "http": "http://nexus:8081/nexus/internal/metrics?pretty",
+    "method": "GET",
+    "header": {
+      "Authorization": ["Basic YWRtaW46YWRtaW4xMjM="],
+      "Cache-Control": ["no-cache"],
+      "Content-Type": ["application/json"],
+      "Accept": ["application/json"]
+    },
+    "tls_skip_verify": true,
+    "interval": "15s",
+    "timeout": "1s"
+  },
+  {
+    "id": "policy-nexus-internal-healthcheck",
+    "name": "Policy Nexus Internal Healthcheck",
+    "http": "http://nexus:8081/nexus/internal/healthcheck?pretty",
+    "method": "GET",
+    "header": {
+      "Authorization": ["Basic YWRtaW46YWRtaW4xMjM="],
+      "Cache-Control": ["no-cache"],
+      "Content-Type": ["application/json"],
+      "Accept": ["application/json"]
+    },
+    "tls_skip_verify": true,
+    "interval": "15s",
+    "timeout": "1s"
+  },
+  {
+    "id": "brmsgw-tcp",
+    "name": "BRMSGW Health Check",
+    "tcp": "brmsgw:9989",
+    "interval": "15s",
+    "timeout": "1s"
+  },
+  {
+    "id": "drools",
+    "name": "Drools Health Check",
+    "http": "http://drools:6969/healthcheck?pretty",
+    "method": "GET",
+    "header": {
+      "Authorization": ["Basic aGVhbHRoY2hlY2s6emIhWHp0RzM0"],
+      "Cache-Control": ["no-cache"],
+      "Content-Type": ["application/json"],
+      "Accept": ["application/json"]
+    },
+    "tls_skip_verify": true,
+    "interval": "15s",
+    "timeout": "1s"
+  },
+  {
+    "id": "pap",
+    "name": "PAP Health Check",
+    "http": "http://pap:9091/pap/test?pretty",
+    "method": "GET",
+    "header": {
+      "Authorization": ["Basic dGVzdHBhcDphbHBoYTEyMw=="],
+      "Cache-Control": ["no-cache"],
+      "Content-Type": ["application/json"],
+      "Accept": ["application/json"]
+    },
+    "tls_skip_verify": true,
+    "interval": "15s",
+    "timeout": "1s"
+  },
+  {
+    "id": "pdp",
+    "name": "PDP Health Check",
+    "http": "http://pdp:8081/pdp/test?pretty",
+    "method": "GET",
+    "header": {
+      "Authorization": ["Basic dGVzdHBkcDphbHBoYTEyMw=="],
+      "Cache-Control": ["no-cache"],
+      "Content-Type": ["application/json"],
+      "Accept": ["application/json"]
+    },
+    "tls_skip_verify": true,
+    "interval": "15s",
+    "timeout": "1s"
+   }
+  ]
+ }
+}
diff --git a/kubernetes/consul/resources/config/consul-agent-config/scripts/policy-mariadb-script.sh b/kubernetes/consul/resources/config/consul-agent-config/scripts/policy-mariadb-script.sh
new file mode 100644
index 0000000000..29dbe3f864
--- /dev/null
+++ b/kubernetes/consul/resources/config/consul-agent-config/scripts/policy-mariadb-script.sh
@@ -0,0 +1,14 @@
+NAME=$(/consul/bin/kubectl -n {{ include "common.namespace" . }} get pod | grep -o "[^[:space:]]*-policydb[^[:space:]]*")
+
+   if [ -n "$NAME" ]; then
+       if /consul/bin/kubectl -n {{ include "common.namespace" . }} exec -it $NAME -- bash -c 'mysqladmin status -u root -p$MYSQL_ROOT_PASSWORD' > /dev/null; then
+         echo Success. mariadb process is running. 2>&1
+         exit 0
+      else
+         echo Failed. mariadb process is not running. 2>&1
+         exit 1
+      fi
+   else
+      echo Failed. mariadb container is offline. 2>&1
+      exit 1
+   fi
diff --git a/kubernetes/consul/resources/config/consul-agent-config/scripts/tabular-db-availability.sh b/kubernetes/consul/resources/config/consul-agent-config/scripts/tabular-db-availability.sh
index fe21db504b..f2c7c587ff 100755
--- a/kubernetes/consul/resources/config/consul-agent-config/scripts/tabular-db-availability.sh
+++ b/kubernetes/consul/resources/config/consul-agent-config/scripts/tabular-db-availability.sh
@@ -1,13 +1,13 @@
 
 # Query the Hbase service for the cluster status.
-GET_CLUSTER_STATUS_RESPONSE=$(curl -si -X GET -H "Accept: text/xml" http://hbase:8080/status/cluster)
+GET_CLUSTER_STATUS_RESPONSE=$(curl -si -X GET -H "Accept: text/xml" http://aai-hbase:8080/status/cluster)
 
 if [ -z "$GET_CLUSTER_STATUS_RESPONSE" ]; then
   echo "Tabular store is unreachable."
-  return 2 
+  return 2
 fi
 
-# Check the resulting status JSON to see if there is a 'DeadNodes' stanza with 
+# Check the resulting status JSON to see if there is a 'DeadNodes' stanza with
 # entries.
 DEAD_NODES=$(echo $GET_CLUSTER_STATUS_RESPONSE | grep "<DeadNodes/>")
 
diff --git a/kubernetes/consul/templates/deployment.yaml b/kubernetes/consul/templates/deployment.yaml
index 8fd09b728e..d37521b5e6 100644
--- a/kubernetes/consul/templates/deployment.yaml
+++ b/kubernetes/consul/templates/deployment.yaml
@@ -43,7 +43,7 @@ spec:
         - "-c"
         - |
           cp /tmp/consul/config/* /consul/config
-          /usr/local/bin/docker-entrypoint.sh agent -client 0.0.0.0 -enable-script-checks -join {{ .Values.consulServer.nameOverride }}.{{ include "common.namespace" .}}
+          /usr/local/bin/docker-entrypoint.sh agent -client 0.0.0.0 -enable-script-checks -retry-join {{ .Values.consulServer.nameOverride }}
         name: {{ include "common.name" . }}
         volumeMounts:
         - mountPath: /tmp/consul/config
@@ -63,4 +63,3 @@ spec:
       - secret:
           secretName: {{ include "common.fullname" . }}-certs-secret
         name: consul-agent-certs-config
-