From b642ee553d6dd486fc375755af9da2fdd9c94885 Mon Sep 17 00:00:00 2001 From: Mohammadreza Pasandideh Date: Tue, 19 Jun 2018 15:19:53 -0400 Subject: SDN-C Multi-site High-availability - Auto-failover Change-Id: I4e028b31ce7a60154d2c04c63431c5ea996de8f8 Signed-off-by: Mohammadreza Pasandideh Issue-ID: SDNC-213 --- .../scripts/sdnc-cluster-health.sh | 58 ++++++++++++++++++++++ .../consul-agent-config/sdnc-cluster-health.json | 14 ++++++ .../sdnc-sdnctldb01-healthcheck.json | 2 +- .../sdnc-sdnctldb02-healthcheck.json | 2 +- .../config/consul-agent-config/sdnc-sdnhost.json | 2 +- kubernetes/consul/templates/deployment.yaml | 6 +++ kubernetes/consul/values.yaml | 10 ++++ 7 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 kubernetes/consul/resources/config/consul-agent-config/scripts/sdnc-cluster-health.sh create mode 100644 kubernetes/consul/resources/config/consul-agent-config/sdnc-cluster-health.json (limited to 'kubernetes/consul') diff --git a/kubernetes/consul/resources/config/consul-agent-config/scripts/sdnc-cluster-health.sh b/kubernetes/consul/resources/config/consul-agent-config/scripts/sdnc-cluster-health.sh new file mode 100644 index 0000000000..3b186547ce --- /dev/null +++ b/kubernetes/consul/resources/config/consul-agent-config/scripts/sdnc-cluster-health.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +# Copyright © 2018 Amdocs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# query ODL cluster state +USERNAME="{{.Values.odl.jolokia.username}}" +PASSWORD="{{.Values.odl.jolokia.password}}" + +count=${SDNC_ODL_COUNT:-1} +siteId=0 +if [ "$SDNC_IS_PRIMARY_CLUSTER" = "false" ];then + siteId=1 +fi + +for instance in $(seq $count);do + shard=member-$(( $siteId*$count + $instance ))-shard-default-config + mbean=Category=Shards,name=$shard,type=DistributedConfigDatastore + url=http://{{.Release.Name}}-sdnc-$(( $instance-1 )).sdnc-cluster.{{.Release.Namespace}}:8181/jolokia/read/org.opendaylight.controller:$mbean + + response=$( curl -s -u $USERNAME:$PASSWORD $url ) + rc=$? + if [ $rc -ne 0 ];then + # failed to contact SDN-C instance - try another + echo "Unable to connect to $shard [rc=$?]" + continue + fi + + status=$( echo "$response" | jq -r ".status" ) + if [ "$status" != "200" ];then + # query failed, try another instance + echo "$shard query failed [http-status=$status]" + continue + fi + + raftState=$( echo "$response" | jq -r ".value.RaftState" ) + if [ "$raftState" = "Leader" -o "$raftState" = "Follower" ];then + # cluster has a leader and is healthy + echo "$shard is healthy [RaftState=$raftState]" + exit 0 + else + echo "$shard is not healthy [RaftState=$raftState]" + fi +done + +# ODL cluster is not healthy +exit 2 diff --git a/kubernetes/consul/resources/config/consul-agent-config/sdnc-cluster-health.json b/kubernetes/consul/resources/config/consul-agent-config/sdnc-cluster-health.json new file mode 100644 index 0000000000..86a7630392 --- /dev/null +++ b/kubernetes/consul/resources/config/consul-agent-config/sdnc-cluster-health.json @@ -0,0 +1,14 @@ +{ + "service": { + "name": "Health Check: SDNC ODL Cluster", + "checks": [ + { + "id": "sdnc-odl-cluster-healthcheck", + "name": "SDNC ODL Cluster Health Check", + "script": "/consul/scripts/sdnc-cluster-health.sh", + "interval": "15s", + "timeout": "10s" + } + ] + } +} diff --git a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb01-healthcheck.json b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb01-healthcheck.json index e684c09da8..ed196bd0cf 100644 --- a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb01-healthcheck.json +++ b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb01-healthcheck.json @@ -5,7 +5,7 @@ { "id": "sdnctldb01", "name": "SDNC SDNCTLDB01 Health Check", - "tcp": "sdnc-dbhost:3306", + "tcp": "sdnc-sdnctldb01:3306", "interval": "10s", "timeout": "1s" } diff --git a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb02-healthcheck.json b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb02-healthcheck.json index ae4820a3e6..b63329f544 100644 --- a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb02-healthcheck.json +++ b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb02-healthcheck.json @@ -5,7 +5,7 @@ { "id": "sdnctldb02", "name": "SDNC SDNCTLDB02 Health Check", - "tcp": "sdnc-dbhost:3306", + "tcp": "sdnc-sdnctldb02:3306", "interval": "10s", "timeout": "1s" } diff --git a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnhost.json b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnhost.json index 604b5be901..db84164fc7 100644 --- a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnhost.json +++ b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnhost.json @@ -5,7 +5,7 @@ { "id": "sdnc-sdnhost", "name": "SDNC SDN Host Health Check", - "http": "http://sdnhost:8282/apidoc/explorer/index.html", + "http": "http://sdnc:8282/apidoc/explorer/index.html", "method": "HEAD", "header": { "Authorization": ["Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ=="], diff --git a/kubernetes/consul/templates/deployment.yaml b/kubernetes/consul/templates/deployment.yaml index d37521b5e6..2df4d2ac12 100644 --- a/kubernetes/consul/templates/deployment.yaml +++ b/kubernetes/consul/templates/deployment.yaml @@ -42,9 +42,15 @@ spec: - /bin/sh - "-c" - | + apk update && apk add jq cp /tmp/consul/config/* /consul/config /usr/local/bin/docker-entrypoint.sh agent -client 0.0.0.0 -enable-script-checks -retry-join {{ .Values.consulServer.nameOverride }} name: {{ include "common.name" . }} + env: + - name: SDNC_ODL_COUNT + value: "{{ .Values.sdnc.replicaCount }}" + - name: SDNC_IS_PRIMARY_CLUSTER + value: "{{ .Values.sdnc.config.isPrimaryCluster }}" volumeMounts: - mountPath: /tmp/consul/config name: consul-agent-config diff --git a/kubernetes/consul/values.yaml b/kubernetes/consul/values.yaml index abfc9ef0ef..37e74f38d5 100644 --- a/kubernetes/consul/values.yaml +++ b/kubernetes/consul/values.yaml @@ -61,3 +61,13 @@ ingress: enabled: false resources: {} + +odl: + jolokia: + username: admin + password: admin + +sdnc: + config: + isPrimaryCluster: true + replicaCount: 1 -- cgit 1.2.3-korg