Multi-site High-availability Manual Failover (PoC)

Change-Id: I0d5644790441099434322d486f4ba014fd8bc1f7
Signed-off-by: Neha Jain <neha.jain3@amdocs.com>
Issue-ID: SDNC-214

4 files changed, 163 insertions, 0 deletions

diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.cluster b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
new file mode 100755
index 0000000000..d59718fa27
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+OOM_HOME=${OOM_HOME:-$HOME}
+
+if ! [ "$(command -v jq)" ]; then
+  echo "Error: jq is not installed."
+  echo "use: sudo apt install jq"
+  exit 1
+fi
+
+IS_PRIMARY_CLUSTER=`./sdnc.isPrimaryCluster`
+
+case $IS_PRIMARY_CLUSTER in
+true)
+   MEMBER_NUMBER=1
+   ;;
+false)
+   MEMBER_NUMBER=4
+   ;;
+*)
+  echo "Error: isPrimaryODLCluster not defined in ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml."
+  exit 1
+  ;;
+esac
+
+for pod_number in {0..2}
+do
+   curl  "http://localhost:3026$((${pod_number} + 1))" > /dev/null 2>&1
+   if [ "$?" = "7" ]; then
+      continue
+   fi
+
+   VOTING_RESULT=`curl -u admin:admin -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3026$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore 2>/dev/null | jq '.value.Voting'`
+
+   case $VOTING_RESULT in
+   true)
+      echo "active"
+      exit 0
+      ;;
+   false)
+      echo "standby"
+      exit 0
+      ;;
+   *)
+     echo "Error: Voting status could not be determined."
+     exit 1
+     ;;
+   esac
+done
+
+echo "Error: Voting status could not be determined."
+exit 1
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.failover b/kubernetes/sdnc/resources/geo/bin/sdnc.failover
new file mode 100755
index 0000000000..961a5cb5cf
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.failover
@@ -0,0 +1,65 @@
+#!/usr/bin/perl -s

+use strict;

+

+my $keyWord_standby = "standby";

+my $keyWord_active = "active";

+my $keyWord_true = "true";

+my $keyWord_false = "false";

+my $keyWord_success = "success";

+my $keyWord_failure = "failure";

+my $file_cluster = "sdnc.cluster";

+my $file_switchVoting = "switchVoting.sh";

+my $file_isPrimaryCluster = "sdnc.isPrimaryCluster";

+

+if ((!(-e $file_cluster)) || (!(-e $file_switchVoting))|| (!(-e $file_isPrimaryCluster))) {

+  # file not exist.

+  print qq|$keyWord_failure\n|;

+  exit 1;

+}

+

+my $roleRes = qx("./$file_isPrimaryCluster");

+my $clusterRes = qx("./$file_cluster");

+

+if ( index ($clusterRes, $keyWord_standby) != -1) {

+	# We are at standby side

+	if ( index ($roleRes, $keyWord_false) != -1) {

+	   # We are at Secondary cluster

+	   sub_activate_secondary();

+    } elsif ( index ($roleRes, $keyWord_true) != -1) {

+       # We are at Primary cluster

+	   sub_activate_primary();

+    } else {

+      # Error.

+      print qq|$keyWord_failure\n|;

+      exit 1;

+	}

+} elsif ( index ($clusterRes, $keyWord_active) != -1) {

+    # We are at active side

+	if ( index ($roleRes, $keyWord_false) != -1) {

+	   # We are at Secondary cluster

+	   sub_activate_primary();

+    } elsif ( index ($roleRes, $keyWord_true) != -1)  {

+       # We are at Primary cluster

+	   sub_activate_secondary();

+    } else {

+      # Error.

+      print qq|$keyWord_failure\n|;

+      exit 1;

+	}

+} else {

+   # Error.

+  print qq|$keyWord_failure\n|;

+  exit 1;

+}

+

+sub sub_activate_primary {

+		#Switching voting in Primary cluster

+        system("./$file_switchVoting primary");

+	    print qq|$keyWord_success\n|;

+}

+

+sub sub_activate_secondary {

+		#Switching voting in secondary cluster

+        system("./$file_switchVoting secondary");

+	    print qq|$keyWord_success\n|;

+}

diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
new file mode 100755
index 0000000000..8e816c4153
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+OOM_HOME=${OOM_HOME:-$HOME}
+
+IS_PRIMARY_CLUSTER=`awk '/isPrimaryODLCluster/ {print $2}' ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml`
+
+if [ "$?" -eq "2" ]; then
+      echo "Make sure you are ubuntu user." >&2
+fi
+
+case $IS_PRIMARY_CLUSTER in
+true|false)
+	echo $IS_PRIMARY_CLUSTER
+	;;
+*)
+	echo "NOT CLUSTERED"
+	exit 1
+	;;
+esac
diff --git a/kubernetes/sdnc/resources/geo/bin/switchVoting.sh b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
new file mode 100755
index 0000000000..05d1e0fd68
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+function usage()
+{
+    echo usage: switchVoting.sh primary\|secondary
+    exit 1
+}
+
+if [ $# -ne 1 ]; then
+    usage
+fi
+
+partition=$1
+
+if [ "$partition" == "primary" ]; then
+   curl -u admin:Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' > switch_voting_resp.json 2>/dev/null
+   echo "" >> switch_voting_resp.json
+   exit 0
+fi
+
+if [ "$partition" == "secondary" ]; then
+   curl -u admin:Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' > switch_voting_resp.json 2>/dev/null
+   echo "" >> switch_voting_resp.json
+   exit 0
+fi
+
+usage