SDN-C Multi-site High-availability - Auto-failover

Change-Id: I4e028b31ce7a60154d2c04c63431c5ea996de8f8
Signed-off-by: Mohammadreza Pasandideh <mohammadreza.pasandideh@amdocs.com>
Issue-ID: SDNC-213

7 files changed, 182 insertions, 86 deletions

diff --git a/kubernetes/sdnc/resources/env.yaml b/kubernetes/sdnc/resources/env.yaml
new file mode 100644
index 0000000000..2ad42f79a6
--- /dev/null
+++ b/kubernetes/sdnc/resources/env.yaml
@@ -0,0 +1,19 @@
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SDNC_GEO_ENABLED: "{{ .Values.config.geoEnabled }}"
+SDNC_IS_PRIMARY_CLUSTER: "{{ .Values.config.isPrimaryCluster }}"
+SDNC_ODL_COUNT: "{{ .Values.replicaCount }}"
+SDNC_LOCAL_K8S_CLUSTER_MASTER: "{{ .Values.config.myODLCluster }}"
+SDNC_REMOTE_K8S_CLUSTER_MASTER: "{{ .Values.config.peerODLCluster }}"
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.cluster b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
index d59718fa27..87cdeffe89 100755
--- a/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
@@ -1,6 +1,18 @@
 #!/bin/bash
 
-OOM_HOME=${OOM_HOME:-$HOME}
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 if ! [ "$(command -v jq)" ]; then
   echo "Error: jq is not installed."
@@ -8,7 +20,8 @@ if ! [ "$(command -v jq)" ]; then
   exit 1
 fi
 
-IS_PRIMARY_CLUSTER=`./sdnc.isPrimaryCluster`
+dir=$( dirname $0 )
+IS_PRIMARY_CLUSTER=$( $dir/sdnc.isPrimaryCluster )
 
 case $IS_PRIMARY_CLUSTER in
 true)
@@ -18,21 +31,30 @@ false)
    MEMBER_NUMBER=4
    ;;
 *)
-  echo "Error: isPrimaryODLCluster not defined in ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml."
+  echo "Error: isPrimaryCluster not defined in $dir/../../../values.yaml."
   exit 1
   ;;
 esac
 
+USERNAME=admin
+PASSWORD=admin
+
 for pod_number in {0..2}
 do
-   curl  "http://localhost:3026$((${pod_number} + 1))" > /dev/null 2>&1
-   if [ "$?" = "7" ]; then
+
+   response=`curl -s -u $USERNAME:$PASSWORD -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3026$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore`
+
+   if [ $? -ne 0 ]; then
       continue
    fi
 
-   VOTING_RESULT=`curl -u admin:admin -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3026$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore 2>/dev/null | jq '.value.Voting'`
+   status=$( echo -E "$response" | jq -r ".status" )
+   if [ "$status" != "200" ]; then
+      continue
+   fi
 
-   case $VOTING_RESULT in
+   voting=$( echo -E "$response" | jq -r ".value.Voting" )
+   case $voting in
    true)
       echo "active"
       exit 0
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.failover b/kubernetes/sdnc/resources/geo/bin/sdnc.failover
deleted file mode 100755
index 961a5cb5cf..0000000000
--- a/kubernetes/sdnc/resources/geo/bin/sdnc.failover
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/perl -s

-use strict;

-

-my $keyWord_standby = "standby";

-my $keyWord_active = "active";

-my $keyWord_true = "true";

-my $keyWord_false = "false";

-my $keyWord_success = "success";

-my $keyWord_failure = "failure";

-my $file_cluster = "sdnc.cluster";

-my $file_switchVoting = "switchVoting.sh";

-my $file_isPrimaryCluster = "sdnc.isPrimaryCluster";

-

-if ((!(-e $file_cluster)) || (!(-e $file_switchVoting))|| (!(-e $file_isPrimaryCluster))) {

-  # file not exist.

-  print qq|$keyWord_failure\n|;

-  exit 1;

-}

-

-my $roleRes = qx("./$file_isPrimaryCluster");

-my $clusterRes = qx("./$file_cluster");

-

-if ( index ($clusterRes, $keyWord_standby) != -1) {

-	# We are at standby side

-	if ( index ($roleRes, $keyWord_false) != -1) {

-	   # We are at Secondary cluster

-	   sub_activate_secondary();

-    } elsif ( index ($roleRes, $keyWord_true) != -1) {

-       # We are at Primary cluster

-	   sub_activate_primary();

-    } else {

-      # Error.

-      print qq|$keyWord_failure\n|;

-      exit 1;

-	}

-} elsif ( index ($clusterRes, $keyWord_active) != -1) {

-    # We are at active side

-	if ( index ($roleRes, $keyWord_false) != -1) {

-	   # We are at Secondary cluster

-	   sub_activate_primary();

-    } elsif ( index ($roleRes, $keyWord_true) != -1)  {

-       # We are at Primary cluster

-	   sub_activate_secondary();

-    } else {

-      # Error.

-      print qq|$keyWord_failure\n|;

-      exit 1;

-	}

-} else {

-   # Error.

-  print qq|$keyWord_failure\n|;

-  exit 1;

-}

-

-sub sub_activate_primary {

-		#Switching voting in Primary cluster

-        system("./$file_switchVoting primary");

-	    print qq|$keyWord_success\n|;

-}

-

-sub sub_activate_secondary {

-		#Switching voting in secondary cluster

-        system("./$file_switchVoting secondary");

-	    print qq|$keyWord_success\n|;

-}

diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
index 77fc65fe39..7a4f6a7dd0 100755
--- a/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
@@ -1,8 +1,22 @@
 #!/bin/bash
 
-OOM_HOME=${OOM_HOME:-$HOME}
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-IS_PRIMARY_CLUSTER=`awk '/isPrimaryCluster/ {print $2}' ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml`
+dir=$( dirname $0 )
+
+IS_PRIMARY_CLUSTER=`awk '/isPrimaryCluster/ {print $2}' $dir/../../../values.yaml`
 
 if [ "$?" -eq "2" ]; then
       echo "Make sure you are ubuntu user." >&2
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive b/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive
new file mode 100755
index 0000000000..76eca48af5
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];then
+  echo "Usage: makeactive <release> [namespace]"
+  exit 1
+fi
+
+RELEASE=$1
+NAMESPACE=onap
+if [ -n "$2" ];then
+  NAMESPACE=$2
+fi
+
+dir=$(dirname $0)
+isPrimary=$( $dir/sdnc.isPrimaryCluster)
+if [ "$isPrimary" = "true" ];then
+  SITE_NAME="sdnc01"
+elif [ "$isPrimary" = "false" ];then
+  SITE_NAME="sdnc02"
+else
+  echo "sdnc.isPrimaryCluster returned unexpected value \"$isPrimary\""
+  exit 1
+fi
+
+pod=$( kubectl -n $NAMESPACE get pods -l app=sdnc-prom,release=$RELEASE | grep Running | cut -f1 -d' ' )
+if [ -z "$pod" ];then
+  echo "prom pod not found - is prom running?"
+  exit 1
+fi
+
+kubectl -n $NAMESPACE exec $pod -- /app/promoverride.py --id $SITE_NAME --config /app/config/config.json
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.monitor b/kubernetes/sdnc/resources/geo/bin/sdnc.monitor
new file mode 100755
index 0000000000..b14bd7325d
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.monitor
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];then
+  echo "Usage: $(basename $0) [--debug] <release> [namespace]"
+  exit 1
+fi
+
+if [ "$1" = "--debug" -o "$1" = "-debug" -o "$1" = "-d" ];then
+  DEBUG="--debug"
+  shift
+fi
+
+RELEASE=$1
+NAMESPACE=onap
+if [ -n "$2" ];then
+  NAMESPACE=$2
+fi
+
+pod=$( kubectl -n $NAMESPACE get pods -l app=sdnc-prom,release=$RELEASE | grep Running | cut -f1 -d' ' )
+if [ -z "$pod" ];then
+  echo "prom pod not found - is prom running?"
+  exit 1
+fi
+
+kubectl -n $NAMESPACE exec $pod -- /app/bin/sdnc.monitor $DEBUG
diff --git a/kubernetes/sdnc/resources/geo/bin/switchVoting.sh b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
index 27e4ead99d..7a1c193492 100755
--- a/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
+++ b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 function usage()
 {
     echo usage: switchVoting.sh primary\|secondary
@@ -10,18 +24,26 @@ if [ $# -ne 1 ]; then
     usage
 fi
 
-partition=$1
+dir=$( dirname $0 )
+USERNAME=admin
+PASSWORD=`awk '/odlPassword/ {print $2}' $dir/../../../values.yaml | head -1`
 
-if [ "$partition" == "primary" ]; then
-   curl -u admin:{{.Values.config.odlPassword}} -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' > switch_voting_resp.json 2>/dev/null
-   echo "" >> switch_voting_resp.json
-   exit 0
-fi
+case "$1" in
 
-if [ "$partition" == "secondary" ]; then
-   curl -u admin:{{.Values.config.odlPassword}} -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' > switch_voting_resp.json 2>/dev/null
-   echo "" >> switch_voting_resp.json
-   exit 0
-fi
+primary)
+   status=$(curl -u $USERNAME:$PASSWORD -o /dev/null -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' -w "%{http_code}\n" $url 2> /dev/null)
+;;
 
-usage
+secondary)
+   status=$(curl -u $USERNAME:$PASSWORD -o /dev/null -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' -w "%{http_code}\n" $url 2> /dev/null)
+;;
+
+*)
+   usage
+esac
+
+if [ $status -ne 200 ];then
+  echo "failure"
+else
+  echo "success"
+fi