Merge "SDN-C Multi-site High-availability - Auto-failover"

1 files changed, 86 insertions, 0 deletions

diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover
new file mode 100755
index 0000000000..e78b7eeee3
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOGFILE="/app/geo.log"
+enableDebugLogging=true
+message_router=message-router:3904
+topic={{.Values.config.messageRouterTopic}}
+KEYWORD_success="success"
+KEYWORD_failure="failure"
+SITE_NAME="sdnc01"
+if [ "$SDNC_IS_PRIMARY_CLUSTER" = "false" ];then
+  SITE_NAME="sdnc02"
+fi
+
+APP_BIN=/app/bin
+
+debugLog(){
+  if [ "$enableDebugLogging" == true ]; then
+     if [ $# -eq 0 ]; then
+       echo "" >> $LOGFILE
+     else
+       echo $( date ) $@ >> $LOGFILE
+    fi
+  fi
+}
+
+EXC_SIMPLE_FAILOVER=`${APP_BIN}/switchVoting.sh`
+
+if [ "$EXC_SIMPLE_FAILOVER" == "success" ]; then
+   debugLog "Simple failover success. SDNC failover completed."
+else
+   # Simple failover failed. Trying catastrophic failover ...
+   debugLog "Simple failover failed. Trying catastrophic failover for $SITE_NAME ..."
+
+   # Notify Dmaap before executing catastrophic failover, because all connections will be reset.
+   data="{\
+   \"type\": \"Catastrophic failover\",\
+   \"reason\": \"Simple failover failed\",\
+   \"message_router\": \"$message_router\",\
+   \"topic\": \"$topic\",\
+   \"site\": \"$SITE_NAME\",\
+   \"deployment\": \"{{.Values.config.deployment}}\",\
+   \"timestamp\": \"$(date '+%F %T')\"\
+   }"
+
+   debugLog "$data"
+
+   # notifications to Dmaap
+   curl -H "Content-Type: application/json" -X POST --data "$data" http://$message_router/events/$topic >/dev/null 2>&1
+
+   # We're going to kill prom, so we need to do dnsswitch now
+
+   debugLog "Executing sdnc.dnsswitch"
+
+   /app/bin/sdnc.dnsswitch > /dev/null 2>&1
+   rc=$?
+   if [ $rc -ne 0 ];then
+    debugLog "sdnc.dnsswitch FAILED"
+    echo $KEYWORD_failure
+    exit 0
+   fi
+
+   # Now do catastrophic failure
+
+   debugLog "Catastrophic failover in progress"
+
+   ssh  -o StrictHostKeyChecking=no -i /app/config/coredns/master.key root@$SDNC_LOCAL_K8S_CLUSTER_MASTER "su - ubuntu bash -c 'helm upgrade --set sdnc.config.geoEnabled=false dev local/onap --namespace onap; kubectl -n onap delete pods -l app=sdnc'" > /dev/null 2>&1
+
+   # Sleep here so prom can die without us passing control back to ensureSDNCActive
+   sleep 300
+fi
+
+echo $KEYWORD_success