summaryrefslogtreecommitdiffstats
path: root/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh
diff options
context:
space:
mode:
authorMohammadreza Pasandideh <mohammadreza.pasandideh@amdocs.com>2018-06-19 15:19:53 -0400
committerBorislav Glozman <Borislav.Glozman@amdocs.com>2018-07-15 10:59:27 +0000
commitb642ee553d6dd486fc375755af9da2fdd9c94885 (patch)
tree6ef3892f15ca7358c316772d388967b905040df1 /kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh
parentdfadfeb71f183943ece2b1641a2d51bcf6219ab6 (diff)
SDN-C Multi-site High-availability - Auto-failover
Change-Id: I4e028b31ce7a60154d2c04c63431c5ea996de8f8 Signed-off-by: Mohammadreza Pasandideh <mohammadreza.pasandideh@amdocs.com> Issue-ID: SDNC-213
Diffstat (limited to 'kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh')
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh105
1 files changed, 105 insertions, 0 deletions
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh b/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh
new file mode 100755
index 0000000000..fb24653129
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+debugLog(){
+ if [ "$enableDebugLogging" == true ]; then
+ if [ $# -eq 0 ]; then
+ echo "" >> $LOGFILE
+ else
+ echo $( date ) $@ >> $LOGFILE
+ fi
+ fi
+}
+
+failover(){
+ lockFile=/tmp/sdnc.failover.lock
+ # make sure that no failover is currently running
+ if [ -e ${lockFile} ] && kill -0 $(cat ${lockFile}) 2> /dev/null; then
+ debugLog "Currently running sdnc and dns failover"
+ return
+ fi
+ trap "rm -f ${lockFile}" INT TERM RETURN
+ echo $BASHPID > ${lockFile}
+
+ # perform takeover
+ debugLog "Started executing sdnc.failover for $SITE_NAME"
+ takeoverResult=$( /app/bin/sdnc.failover )
+ debugLog "Completed executing sdnc.failover. takeoverResult is: $takeoverResult"
+ if [ "success" = "$takeoverResult" ]; then
+ # update CoreDNS upon successful execution of sdnc.failover script
+ debugLog "Executing sdnc.dnsswitch"
+ /app/bin/sdnc.dnsswitch
+ rc=$?
+ debugLog "Completed executing sdnc.dnsswitch for $SITE_NAME. rc=$rc"
+ else
+ debugLog "Cluster takeover current status: $takeoverResult on $SITE_NAME."
+ rc=1
+ fi
+
+ if [ $rc -ne 0 ];then
+ takeoverResult="failure"
+ fi
+
+ data="{\
+\"type\": \"failover\",\
+\"status\": \"$takeoverResult\",\
+\"site\": \"$SITE_NAME\",\
+\"deployment\": \"{{.Values.config.deployment}}\",\
+\"timestamp\": \"$(date '+%F %T')\"\
+}"
+
+ # notifications are best-effort - ignore any failures
+ curl -H "Content-Type: application/json" -X POST --data "$data" http://$message_router/events/$topic >/dev/null 2>&1
+
+}
+
+LOGFILE="/app/geo.log"
+enableDebugLogging=true
+message_router=message-router:3904
+topic={{.Values.config.messageRouterTopic}}
+SITE_NAME="sdnc01"
+if [ "$SDNC_IS_PRIMARY_CLUSTER" = "false" ];then
+ SITE_NAME="sdnc02"
+fi
+
+debugLog
+debugLog "Executing ensureSdncActive"
+
+# query SDN-C cluster status
+debugLog "Started executing sdnc.cluster"
+clusterStatus=$( /app/bin/sdnc.cluster )
+debugLog "Completed executing sdnc.cluster. Cluster status is: $clusterStatus"
+
+if [ "active" = "$clusterStatus" ]; then
+ # peform health-check
+ debugLog "Started excuting sdnc.monitor"
+ health=$( /app/bin/sdnc.monitor )
+ debugLog "Completed executing sdnc.monitor. Cluster is: $health"
+
+ if [ "healthy" = "$health" ]; then
+ # Cluster is ACTIVE and HEALTHY
+ exit 0
+ fi
+ exit 1
+
+elif [ "standby" = "$clusterStatus" ]; then
+ # Run failover in background process and allow PROM to continue
+ ( failover & )
+ exit 0
+fi
+
+# Unknown cluster status
+exit 1