aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPrudence Au <prudence.au@amdocs.com>2018-03-28 11:15:32 -0400
committerPrudence Au <prudence.au@amdocs.com>2018-03-28 11:15:44 -0400
commit40977671839b780b64c2526ce6f4edadf2f44883 (patch)
treeb1c6f30613e918d0f68de22d39d917264d9d2ac3
parent8722bcb539b9ca37f0a88dde7b59dc610f0a5541 (diff)
Multi-site High-availability Manual Failover (PoC)
Change-Id: I690a52ec26acaf6c0d81a7a4f5dc9f922409be33 Signed-off-by: Prudence Au <prudence.au@amdocs.com> Issue-ID: SDNC-214 Former-commit-id: d031a8ebdf7201a4616f195762ecf7afc4c79aa7
-rwxr-xr-xinstallation/sdnc/src/main/scripts/sdnc.cluster52
-rwxr-xr-xinstallation/sdnc/src/main/scripts/sdnc.failover65
-rwxr-xr-xinstallation/sdnc/src/main/scripts/sdnc.isPrimaryCluster19
-rwxr-xr-xinstallation/sdnc/src/main/scripts/switchVoting.sh27
4 files changed, 163 insertions, 0 deletions
diff --git a/installation/sdnc/src/main/scripts/sdnc.cluster b/installation/sdnc/src/main/scripts/sdnc.cluster
new file mode 100755
index 00000000..35b60728
--- /dev/null
+++ b/installation/sdnc/src/main/scripts/sdnc.cluster
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+OOM_HOME=${OOM_HOME:-$HOME}
+
+if ! [ "$(command -v jq)" ]; then
+ echo "Error: jq is not installed."
+ echo "use: sudo apt install jq"
+ exit 1
+fi
+
+IS_PRIMARY_CLUSTER=`./sdnc.isPrimaryCluster`
+
+case $IS_PRIMARY_CLUSTER in
+true)
+ MEMBER_NUMBER=1
+ ;;
+false)
+ MEMBER_NUMBER=4
+ ;;
+*)
+ echo "Error: isPrimaryODLCluster not defined in ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml."
+ exit 1
+ ;;
+esac
+
+for pod_number in {0..2}
+do
+ curl "http://localhost:3028$((${pod_number} + 1))" > /dev/null 2>&1
+ if [ "$?" = "7" ]; then
+ continue
+ fi
+
+ VOTING_RESULT=`curl -u admin:admin -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3028$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore 2>/dev/null | jq '.value.Voting'`
+
+ case $VOTING_RESULT in
+ true)
+ echo "active"
+ exit 0
+ ;;
+ false)
+ echo "standby"
+ exit 0
+ ;;
+ *)
+ echo "Error: Voting status could not be determined."
+ exit 1
+ ;;
+ esac
+done
+
+echo "Error: Voting status could not be determined."
+exit 1
diff --git a/installation/sdnc/src/main/scripts/sdnc.failover b/installation/sdnc/src/main/scripts/sdnc.failover
new file mode 100755
index 00000000..b002ae67
--- /dev/null
+++ b/installation/sdnc/src/main/scripts/sdnc.failover
@@ -0,0 +1,65 @@
+#!/usr/bin/perl -s
+use strict;
+
+my $keyWord_standby = "standby";
+my $keyWord_active = "active";
+my $keyWord_true = "true";
+my $keyWord_false = "false";
+my $keyWord_success = "success";
+my $keyWord_failure = "failure";
+my $file_cluster = "sdnc.cluster";
+my $file_switchVoting = "switchVoting.sh";
+my $file_isPrimaryCluster = "sdnc.isPrimaryCluster";
+
+if ((!(-e $file_cluster)) || (!(-e $file_switchVoting))|| (!(-e $file_isPrimaryCluster))) {
+ # file not exist.
+ print qq|$keyWord_failure\n|;
+ exit 1;
+}
+
+my $roleRes = qx("./$file_isPrimaryCluster");
+my $clusterRes = qx("./$file_cluster");
+
+if ( index ($clusterRes, $keyWord_standby) != -1) {
+ # We are at standby side
+ if ( index ($roleRes, $keyWord_false) != -1) {
+ # We are at Secondary cluster
+ sub_activate_secondary();
+ } elsif ( index ($roleRes, $keyWord_true) != -1) {
+ # We are at Primary cluster
+ sub_activate_primary();
+ } else {
+ # Error.
+ print qq|$keyWord_failure\n|;
+ exit 1;
+ }
+} elsif ( index ($clusterRes, $keyWord_active) != -1) {
+ # We are at active side
+ if ( index ($roleRes, $keyWord_false) != -1) {
+ # We are at Secondary cluster
+ sub_activate_primary();
+ } elsif ( index ($roleRes, $keyWord_true) != -1) {
+ # We are at Primary cluster
+ sub_activate_secondary();
+ } else {
+ # Error.
+ print qq|$keyWord_failure\n|;
+ exit 1;
+ }
+} else {
+ # Error.
+ print qq|$keyWord_failure\n|;
+ exit 1;
+}
+
+sub sub_activate_primary {
+ #Switching voting in Primary cluster
+ system("./$file_switchVoting primary");
+ print qq|$keyWord_success\n|;
+}
+
+sub sub_activate_secondary {
+ #Switching voting in secondary cluster
+ system("./$file_switchVoting secondary");
+ print qq|$keyWord_success\n|;
+}
diff --git a/installation/sdnc/src/main/scripts/sdnc.isPrimaryCluster b/installation/sdnc/src/main/scripts/sdnc.isPrimaryCluster
new file mode 100755
index 00000000..c14f717a
--- /dev/null
+++ b/installation/sdnc/src/main/scripts/sdnc.isPrimaryCluster
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+OOM_HOME=${OOM_HOME:-$HOME}
+
+IS_PRIMARY_CLUSTER=`awk '/isPrimaryODLCluster/ {print $2}' ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml`
+
+if [ "$?" -eq "2" ]; then
+ echo "Make sure you are ubuntu user." >&2
+fi
+
+case $IS_PRIMARY_CLUSTER in
+true|false)
+ echo $IS_PRIMARY_CLUSTER
+ ;;
+*)
+ echo "NOT CLUSTERED"
+ exit 1
+ ;;
+esac
diff --git a/installation/sdnc/src/main/scripts/switchVoting.sh b/installation/sdnc/src/main/scripts/switchVoting.sh
new file mode 100755
index 00000000..05d1e0fd
--- /dev/null
+++ b/installation/sdnc/src/main/scripts/switchVoting.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+function usage()
+{
+ echo usage: switchVoting.sh primary\|secondary
+ exit 1
+}
+
+if [ $# -ne 1 ]; then
+ usage
+fi
+
+partition=$1
+
+if [ "$partition" == "primary" ]; then
+ curl -u admin:Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' > switch_voting_resp.json 2>/dev/null
+ echo "" >> switch_voting_resp.json
+ exit 0
+fi
+
+if [ "$partition" == "secondary" ]; then
+ curl -u admin:Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' > switch_voting_resp.json 2>/dev/null
+ echo "" >> switch_voting_resp.json
+ exit 0
+fi
+
+usage