From b642ee553d6dd486fc375755af9da2fdd9c94885 Mon Sep 17 00:00:00 2001 From: Mohammadreza Pasandideh Date: Tue, 19 Jun 2018 15:19:53 -0400 Subject: SDN-C Multi-site High-availability - Auto-failover Change-Id: I4e028b31ce7a60154d2c04c63431c5ea996de8f8 Signed-off-by: Mohammadreza Pasandideh Issue-ID: SDNC-213 --- kubernetes/sdnc/resources/geo/bin/sdnc.cluster | 36 +++++++++--- kubernetes/sdnc/resources/geo/bin/sdnc.failover | 65 ---------------------- .../sdnc/resources/geo/bin/sdnc.isPrimaryCluster | 18 +++++- kubernetes/sdnc/resources/geo/bin/sdnc.makeActive | 45 +++++++++++++++ kubernetes/sdnc/resources/geo/bin/sdnc.monitor | 39 +++++++++++++ kubernetes/sdnc/resources/geo/bin/switchVoting.sh | 46 +++++++++++---- 6 files changed, 163 insertions(+), 86 deletions(-) delete mode 100755 kubernetes/sdnc/resources/geo/bin/sdnc.failover create mode 100755 kubernetes/sdnc/resources/geo/bin/sdnc.makeActive create mode 100755 kubernetes/sdnc/resources/geo/bin/sdnc.monitor (limited to 'kubernetes/sdnc/resources/geo/bin') diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.cluster b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster index d59718fa27..87cdeffe89 100755 --- a/kubernetes/sdnc/resources/geo/bin/sdnc.cluster +++ b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster @@ -1,6 +1,18 @@ #!/bin/bash -OOM_HOME=${OOM_HOME:-$HOME} +# Copyright © 2018 Amdocs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. if ! [ "$(command -v jq)" ]; then echo "Error: jq is not installed." @@ -8,7 +20,8 @@ if ! [ "$(command -v jq)" ]; then exit 1 fi -IS_PRIMARY_CLUSTER=`./sdnc.isPrimaryCluster` +dir=$( dirname $0 ) +IS_PRIMARY_CLUSTER=$( $dir/sdnc.isPrimaryCluster ) case $IS_PRIMARY_CLUSTER in true) @@ -18,21 +31,30 @@ false) MEMBER_NUMBER=4 ;; *) - echo "Error: isPrimaryODLCluster not defined in ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml." + echo "Error: isPrimaryCluster not defined in $dir/../../../values.yaml." exit 1 ;; esac +USERNAME=admin +PASSWORD=admin + for pod_number in {0..2} do - curl "http://localhost:3026$((${pod_number} + 1))" > /dev/null 2>&1 - if [ "$?" = "7" ]; then + + response=`curl -s -u $USERNAME:$PASSWORD -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3026$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore` + + if [ $? -ne 0 ]; then continue fi - VOTING_RESULT=`curl -u admin:admin -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3026$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore 2>/dev/null | jq '.value.Voting'` + status=$( echo -E "$response" | jq -r ".status" ) + if [ "$status" != "200" ]; then + continue + fi - case $VOTING_RESULT in + voting=$( echo -E "$response" | jq -r ".value.Voting" ) + case $voting in true) echo "active" exit 0 diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.failover b/kubernetes/sdnc/resources/geo/bin/sdnc.failover deleted file mode 100755 index 961a5cb5cf..0000000000 --- a/kubernetes/sdnc/resources/geo/bin/sdnc.failover +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/perl -s -use strict; - -my $keyWord_standby = "standby"; -my $keyWord_active = "active"; -my $keyWord_true = "true"; -my $keyWord_false = "false"; -my $keyWord_success = "success"; -my $keyWord_failure = "failure"; -my $file_cluster = "sdnc.cluster"; -my $file_switchVoting = "switchVoting.sh"; -my $file_isPrimaryCluster = "sdnc.isPrimaryCluster"; - -if ((!(-e $file_cluster)) || (!(-e $file_switchVoting))|| (!(-e $file_isPrimaryCluster))) { - # file not exist. - print qq|$keyWord_failure\n|; - exit 1; -} - -my $roleRes = qx("./$file_isPrimaryCluster"); -my $clusterRes = qx("./$file_cluster"); - -if ( index ($clusterRes, $keyWord_standby) != -1) { - # We are at standby side - if ( index ($roleRes, $keyWord_false) != -1) { - # We are at Secondary cluster - sub_activate_secondary(); - } elsif ( index ($roleRes, $keyWord_true) != -1) { - # We are at Primary cluster - sub_activate_primary(); - } else { - # Error. - print qq|$keyWord_failure\n|; - exit 1; - } -} elsif ( index ($clusterRes, $keyWord_active) != -1) { - # We are at active side - if ( index ($roleRes, $keyWord_false) != -1) { - # We are at Secondary cluster - sub_activate_primary(); - } elsif ( index ($roleRes, $keyWord_true) != -1) { - # We are at Primary cluster - sub_activate_secondary(); - } else { - # Error. - print qq|$keyWord_failure\n|; - exit 1; - } -} else { - # Error. - print qq|$keyWord_failure\n|; - exit 1; -} - -sub sub_activate_primary { - #Switching voting in Primary cluster - system("./$file_switchVoting primary"); - print qq|$keyWord_success\n|; -} - -sub sub_activate_secondary { - #Switching voting in secondary cluster - system("./$file_switchVoting secondary"); - print qq|$keyWord_success\n|; -} diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster index 77fc65fe39..7a4f6a7dd0 100755 --- a/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster +++ b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster @@ -1,8 +1,22 @@ #!/bin/bash -OOM_HOME=${OOM_HOME:-$HOME} +# Copyright © 2018 Amdocs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -IS_PRIMARY_CLUSTER=`awk '/isPrimaryCluster/ {print $2}' ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml` +dir=$( dirname $0 ) + +IS_PRIMARY_CLUSTER=`awk '/isPrimaryCluster/ {print $2}' $dir/../../../values.yaml` if [ "$?" -eq "2" ]; then echo "Make sure you are ubuntu user." >&2 diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive b/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive new file mode 100755 index 0000000000..76eca48af5 --- /dev/null +++ b/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive @@ -0,0 +1,45 @@ +#!/bin/sh + +# Copyright © 2018 Amdocs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ];then + echo "Usage: makeactive [namespace]" + exit 1 +fi + +RELEASE=$1 +NAMESPACE=onap +if [ -n "$2" ];then + NAMESPACE=$2 +fi + +dir=$(dirname $0) +isPrimary=$( $dir/sdnc.isPrimaryCluster) +if [ "$isPrimary" = "true" ];then + SITE_NAME="sdnc01" +elif [ "$isPrimary" = "false" ];then + SITE_NAME="sdnc02" +else + echo "sdnc.isPrimaryCluster returned unexpected value \"$isPrimary\"" + exit 1 +fi + +pod=$( kubectl -n $NAMESPACE get pods -l app=sdnc-prom,release=$RELEASE | grep Running | cut -f1 -d' ' ) +if [ -z "$pod" ];then + echo "prom pod not found - is prom running?" + exit 1 +fi + +kubectl -n $NAMESPACE exec $pod -- /app/promoverride.py --id $SITE_NAME --config /app/config/config.json diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.monitor b/kubernetes/sdnc/resources/geo/bin/sdnc.monitor new file mode 100755 index 0000000000..b14bd7325d --- /dev/null +++ b/kubernetes/sdnc/resources/geo/bin/sdnc.monitor @@ -0,0 +1,39 @@ +#!/bin/sh + +# Copyright © 2018 Amdocs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ];then + echo "Usage: $(basename $0) [--debug] [namespace]" + exit 1 +fi + +if [ "$1" = "--debug" -o "$1" = "-debug" -o "$1" = "-d" ];then + DEBUG="--debug" + shift +fi + +RELEASE=$1 +NAMESPACE=onap +if [ -n "$2" ];then + NAMESPACE=$2 +fi + +pod=$( kubectl -n $NAMESPACE get pods -l app=sdnc-prom,release=$RELEASE | grep Running | cut -f1 -d' ' ) +if [ -z "$pod" ];then + echo "prom pod not found - is prom running?" + exit 1 +fi + +kubectl -n $NAMESPACE exec $pod -- /app/bin/sdnc.monitor $DEBUG diff --git a/kubernetes/sdnc/resources/geo/bin/switchVoting.sh b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh index 27e4ead99d..7a1c193492 100755 --- a/kubernetes/sdnc/resources/geo/bin/switchVoting.sh +++ b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright © 2018 Amdocs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + function usage() { echo usage: switchVoting.sh primary\|secondary @@ -10,18 +24,26 @@ if [ $# -ne 1 ]; then usage fi -partition=$1 +dir=$( dirname $0 ) +USERNAME=admin +PASSWORD=`awk '/odlPassword/ {print $2}' $dir/../../../values.yaml | head -1` -if [ "$partition" == "primary" ]; then - curl -u admin:{{.Values.config.odlPassword}} -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' > switch_voting_resp.json 2>/dev/null - echo "" >> switch_voting_resp.json - exit 0 -fi +case "$1" in -if [ "$partition" == "secondary" ]; then - curl -u admin:{{.Values.config.odlPassword}} -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' > switch_voting_resp.json 2>/dev/null - echo "" >> switch_voting_resp.json - exit 0 -fi +primary) + status=$(curl -u $USERNAME:$PASSWORD -o /dev/null -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' -w "%{http_code}\n" $url 2> /dev/null) +;; -usage +secondary) + status=$(curl -u $USERNAME:$PASSWORD -o /dev/null -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' -w "%{http_code}\n" $url 2> /dev/null) +;; + +*) + usage +esac + +if [ $status -ne 200 ];then + echo "failure" +else + echo "success" +fi -- cgit 1.2.3-korg