summaryrefslogtreecommitdiffstats
path: root/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover
diff options
context:
space:
mode:
authorMohammadreza Pasandideh <mohammadreza.pasandideh@amdocs.com>2018-06-19 15:19:53 -0400
committerBorislav Glozman <Borislav.Glozman@amdocs.com>2018-07-15 10:59:27 +0000
commitb642ee553d6dd486fc375755af9da2fdd9c94885 (patch)
tree6ef3892f15ca7358c316772d388967b905040df1 /kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover
parentdfadfeb71f183943ece2b1641a2d51bcf6219ab6 (diff)
SDN-C Multi-site High-availability - Auto-failover
Change-Id: I4e028b31ce7a60154d2c04c63431c5ea996de8f8 Signed-off-by: Mohammadreza Pasandideh <mohammadreza.pasandideh@amdocs.com> Issue-ID: SDNC-213
Diffstat (limited to 'kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover')
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover86
1 files changed, 86 insertions, 0 deletions
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover
new file mode 100755
index 0000000000..e78b7eeee3
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOGFILE="/app/geo.log"
+enableDebugLogging=true
+message_router=message-router:3904
+topic={{.Values.config.messageRouterTopic}}
+KEYWORD_success="success"
+KEYWORD_failure="failure"
+SITE_NAME="sdnc01"
+if [ "$SDNC_IS_PRIMARY_CLUSTER" = "false" ];then
+ SITE_NAME="sdnc02"
+fi
+
+APP_BIN=/app/bin
+
+debugLog(){
+ if [ "$enableDebugLogging" == true ]; then
+ if [ $# -eq 0 ]; then
+ echo "" >> $LOGFILE
+ else
+ echo $( date ) $@ >> $LOGFILE
+ fi
+ fi
+}
+
+EXC_SIMPLE_FAILOVER=`${APP_BIN}/switchVoting.sh`
+
+if [ "$EXC_SIMPLE_FAILOVER" == "success" ]; then
+ debugLog "Simple failover success. SDNC failover completed."
+else
+ # Simple failover failed. Trying catastrophic failover ...
+ debugLog "Simple failover failed. Trying catastrophic failover for $SITE_NAME ..."
+
+ # Notify Dmaap before executing catastrophic failover, because all connections will be reset.
+ data="{\
+ \"type\": \"Catastrophic failover\",\
+ \"reason\": \"Simple failover failed\",\
+ \"message_router\": \"$message_router\",\
+ \"topic\": \"$topic\",\
+ \"site\": \"$SITE_NAME\",\
+ \"deployment\": \"{{.Values.config.deployment}}\",\
+ \"timestamp\": \"$(date '+%F %T')\"\
+ }"
+
+ debugLog "$data"
+
+ # notifications to Dmaap
+ curl -H "Content-Type: application/json" -X POST --data "$data" http://$message_router/events/$topic >/dev/null 2>&1
+
+ # We're going to kill prom, so we need to do dnsswitch now
+
+ debugLog "Executing sdnc.dnsswitch"
+
+ /app/bin/sdnc.dnsswitch > /dev/null 2>&1
+ rc=$?
+ if [ $rc -ne 0 ];then
+ debugLog "sdnc.dnsswitch FAILED"
+ echo $KEYWORD_failure
+ exit 0
+ fi
+
+ # Now do catastrophic failure
+
+ debugLog "Catastrophic failover in progress"
+
+ ssh -o StrictHostKeyChecking=no -i /app/config/coredns/master.key root@$SDNC_LOCAL_K8S_CLUSTER_MASTER "su - ubuntu bash -c 'helm upgrade --set sdnc.config.geoEnabled=false dev local/onap --namespace onap; kubectl -n onap delete pods -l app=sdnc'" > /dev/null 2>&1
+
+ # Sleep here so prom can die without us passing control back to ensureSDNCActive
+ sleep 300
+fi
+
+echo $KEYWORD_success