aboutsummaryrefslogtreecommitdiffstats
path: root/kubernetes
diff options
context:
space:
mode:
authorMohammadreza Pasandideh <mohammadreza.pasandideh@amdocs.com>2018-06-19 15:19:53 -0400
committerBorislav Glozman <Borislav.Glozman@amdocs.com>2018-07-15 10:59:27 +0000
commitb642ee553d6dd486fc375755af9da2fdd9c94885 (patch)
tree6ef3892f15ca7358c316772d388967b905040df1 /kubernetes
parentdfadfeb71f183943ece2b1641a2d51bcf6219ab6 (diff)
SDN-C Multi-site High-availability - Auto-failover
Change-Id: I4e028b31ce7a60154d2c04c63431c5ea996de8f8 Signed-off-by: Mohammadreza Pasandideh <mohammadreza.pasandideh@amdocs.com> Issue-ID: SDNC-213
Diffstat (limited to 'kubernetes')
-rw-r--r--kubernetes/consul/resources/config/consul-agent-config/scripts/sdnc-cluster-health.sh58
-rw-r--r--kubernetes/consul/resources/config/consul-agent-config/sdnc-cluster-health.json14
-rw-r--r--kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb01-healthcheck.json2
-rw-r--r--kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb02-healthcheck.json2
-rw-r--r--kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnhost.json2
-rw-r--r--kubernetes/consul/templates/deployment.yaml6
-rw-r--r--kubernetes/consul/values.yaml10
-rw-r--r--kubernetes/sdnc/Makefile51
-rw-r--r--kubernetes/sdnc/requirements.yaml6
-rw-r--r--kubernetes/sdnc/resources/env.yaml19
-rwxr-xr-xkubernetes/sdnc/resources/geo/bin/sdnc.cluster36
-rwxr-xr-xkubernetes/sdnc/resources/geo/bin/sdnc.failover65
-rwxr-xr-xkubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster18
-rwxr-xr-xkubernetes/sdnc/resources/geo/bin/sdnc.makeActive45
-rwxr-xr-xkubernetes/sdnc/resources/geo/bin/sdnc.monitor39
-rwxr-xr-xkubernetes/sdnc/resources/geo/bin/switchVoting.sh46
-rw-r--r--kubernetes/sdnc/sdnc-prom/Chart.yaml18
-rw-r--r--kubernetes/sdnc/sdnc-prom/requirements.yaml18
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh105
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncStandby.sh58
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/prom.sh31
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/sdnc.cluster61
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/sdnc.dnsswitch22
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover86
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/sdnc.monitor125
-rwxr-xr-xkubernetes/sdnc/sdnc-prom/resources/bin/switchVoting.sh110
-rw-r--r--kubernetes/sdnc/sdnc-prom/resources/config/config.json19
-rw-r--r--kubernetes/sdnc/sdnc-prom/resources/config/healthchecks.json1
-rw-r--r--kubernetes/sdnc/sdnc-prom/templates/configmap.yaml29
-rw-r--r--kubernetes/sdnc/sdnc-prom/templates/deployment.yaml95
-rw-r--r--kubernetes/sdnc/sdnc-prom/values.yaml102
-rw-r--r--kubernetes/sdnc/templates/configmap.yaml8
-rw-r--r--kubernetes/sdnc/values.yaml3
33 files changed, 1218 insertions, 92 deletions
diff --git a/kubernetes/consul/resources/config/consul-agent-config/scripts/sdnc-cluster-health.sh b/kubernetes/consul/resources/config/consul-agent-config/scripts/sdnc-cluster-health.sh
new file mode 100644
index 0000000000..3b186547ce
--- /dev/null
+++ b/kubernetes/consul/resources/config/consul-agent-config/scripts/sdnc-cluster-health.sh
@@ -0,0 +1,58 @@
+#!/bin/sh
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# query ODL cluster state
+USERNAME="{{.Values.odl.jolokia.username}}"
+PASSWORD="{{.Values.odl.jolokia.password}}"
+
+count=${SDNC_ODL_COUNT:-1}
+siteId=0
+if [ "$SDNC_IS_PRIMARY_CLUSTER" = "false" ];then
+ siteId=1
+fi
+
+for instance in $(seq $count);do
+ shard=member-$(( $siteId*$count + $instance ))-shard-default-config
+ mbean=Category=Shards,name=$shard,type=DistributedConfigDatastore
+ url=http://{{.Release.Name}}-sdnc-$(( $instance-1 )).sdnc-cluster.{{.Release.Namespace}}:8181/jolokia/read/org.opendaylight.controller:$mbean
+
+ response=$( curl -s -u $USERNAME:$PASSWORD $url )
+ rc=$?
+ if [ $rc -ne 0 ];then
+ # failed to contact SDN-C instance - try another
+ echo "Unable to connect to $shard [rc=$?]"
+ continue
+ fi
+
+ status=$( echo "$response" | jq -r ".status" )
+ if [ "$status" != "200" ];then
+ # query failed, try another instance
+ echo "$shard query failed [http-status=$status]"
+ continue
+ fi
+
+ raftState=$( echo "$response" | jq -r ".value.RaftState" )
+ if [ "$raftState" = "Leader" -o "$raftState" = "Follower" ];then
+ # cluster has a leader and is healthy
+ echo "$shard is healthy [RaftState=$raftState]"
+ exit 0
+ else
+ echo "$shard is not healthy [RaftState=$raftState]"
+ fi
+done
+
+# ODL cluster is not healthy
+exit 2
diff --git a/kubernetes/consul/resources/config/consul-agent-config/sdnc-cluster-health.json b/kubernetes/consul/resources/config/consul-agent-config/sdnc-cluster-health.json
new file mode 100644
index 0000000000..86a7630392
--- /dev/null
+++ b/kubernetes/consul/resources/config/consul-agent-config/sdnc-cluster-health.json
@@ -0,0 +1,14 @@
+{
+ "service": {
+ "name": "Health Check: SDNC ODL Cluster",
+ "checks": [
+ {
+ "id": "sdnc-odl-cluster-healthcheck",
+ "name": "SDNC ODL Cluster Health Check",
+ "script": "/consul/scripts/sdnc-cluster-health.sh",
+ "interval": "15s",
+ "timeout": "10s"
+ }
+ ]
+ }
+}
diff --git a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb01-healthcheck.json b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb01-healthcheck.json
index e684c09da8..ed196bd0cf 100644
--- a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb01-healthcheck.json
+++ b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb01-healthcheck.json
@@ -5,7 +5,7 @@
{
"id": "sdnctldb01",
"name": "SDNC SDNCTLDB01 Health Check",
- "tcp": "sdnc-dbhost:3306",
+ "tcp": "sdnc-sdnctldb01:3306",
"interval": "10s",
"timeout": "1s"
}
diff --git a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb02-healthcheck.json b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb02-healthcheck.json
index ae4820a3e6..b63329f544 100644
--- a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb02-healthcheck.json
+++ b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnctldb02-healthcheck.json
@@ -5,7 +5,7 @@
{
"id": "sdnctldb02",
"name": "SDNC SDNCTLDB02 Health Check",
- "tcp": "sdnc-dbhost:3306",
+ "tcp": "sdnc-sdnctldb02:3306",
"interval": "10s",
"timeout": "1s"
}
diff --git a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnhost.json b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnhost.json
index 604b5be901..db84164fc7 100644
--- a/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnhost.json
+++ b/kubernetes/consul/resources/config/consul-agent-config/sdnc-sdnhost.json
@@ -5,7 +5,7 @@
{
"id": "sdnc-sdnhost",
"name": "SDNC SDN Host Health Check",
- "http": "http://sdnhost:8282/apidoc/explorer/index.html",
+ "http": "http://sdnc:8282/apidoc/explorer/index.html",
"method": "HEAD",
"header": {
"Authorization": ["Basic YWRtaW46S3A4Yko0U1hzek0wV1hsaGFrM2VIbGNzZTJnQXc4NHZhb0dHbUp2VXkyVQ=="],
diff --git a/kubernetes/consul/templates/deployment.yaml b/kubernetes/consul/templates/deployment.yaml
index d37521b5e6..2df4d2ac12 100644
--- a/kubernetes/consul/templates/deployment.yaml
+++ b/kubernetes/consul/templates/deployment.yaml
@@ -42,9 +42,15 @@ spec:
- /bin/sh
- "-c"
- |
+ apk update && apk add jq
cp /tmp/consul/config/* /consul/config
/usr/local/bin/docker-entrypoint.sh agent -client 0.0.0.0 -enable-script-checks -retry-join {{ .Values.consulServer.nameOverride }}
name: {{ include "common.name" . }}
+ env:
+ - name: SDNC_ODL_COUNT
+ value: "{{ .Values.sdnc.replicaCount }}"
+ - name: SDNC_IS_PRIMARY_CLUSTER
+ value: "{{ .Values.sdnc.config.isPrimaryCluster }}"
volumeMounts:
- mountPath: /tmp/consul/config
name: consul-agent-config
diff --git a/kubernetes/consul/values.yaml b/kubernetes/consul/values.yaml
index abfc9ef0ef..37e74f38d5 100644
--- a/kubernetes/consul/values.yaml
+++ b/kubernetes/consul/values.yaml
@@ -61,3 +61,13 @@ ingress:
enabled: false
resources: {}
+
+odl:
+ jolokia:
+ username: admin
+ password: admin
+
+sdnc:
+ config:
+ isPrimaryCluster: true
+ replicaCount: 1
diff --git a/kubernetes/sdnc/Makefile b/kubernetes/sdnc/Makefile
new file mode 100644
index 0000000000..d634a8c506
--- /dev/null
+++ b/kubernetes/sdnc/Makefile
@@ -0,0 +1,51 @@
+# Copyright © 2017 Amdocs, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# FIXME OOM-765
+ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+OUTPUT_DIR := $(ROOT_DIR)/../dist
+PACKAGE_DIR := $(OUTPUT_DIR)/packages
+SECRET_DIR := $(OUTPUT_DIR)/secrets
+
+EXCLUDES :=
+HELM_CHARTS := $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
+
+.PHONY: $(EXCLUDES) $(HELM_CHARTS)
+
+all: $(HELM_CHARTS)
+
+$(HELM_CHARTS):
+ @echo "\n[$@]"
+ @make package-$@
+
+make-%:
+ @if [ -f $*/Makefile ]; then make -C $*; fi
+
+dep-%: make-%
+ @if [ -f $*/requirements.yaml ]; then helm dep up $*; fi
+
+lint-%: dep-%
+ @if [ -f $*/Chart.yaml ]; then helm lint $*; fi
+
+package-%: lint-%
+ @mkdir -p $(PACKAGE_DIR)
+ @if [ -f $*/Chart.yaml ]; then helm package -d $(PACKAGE_DIR) $*; fi
+ @helm repo index $(PACKAGE_DIR)
+
+clean:
+ @rm -f */requirements.lock
+ @rm -f *tgz */charts/*tgz
+ @rm -rf $(PACKAGE_DIR)
+%:
+ @: \ No newline at end of file
diff --git a/kubernetes/sdnc/requirements.yaml b/kubernetes/sdnc/requirements.yaml
index c458755a85..09348140be 100644
--- a/kubernetes/sdnc/requirements.yaml
+++ b/kubernetes/sdnc/requirements.yaml
@@ -21,4 +21,8 @@ dependencies:
repository: '@local'
- name: dgbuilder
version: ~2.0.0
- repository: '@local' \ No newline at end of file
+ repository: '@local'
+ - name: sdnc-prom
+ version: ~2.0.0
+ repository: '@local'
+ condition: config.geoEnabled
diff --git a/kubernetes/sdnc/resources/env.yaml b/kubernetes/sdnc/resources/env.yaml
new file mode 100644
index 0000000000..2ad42f79a6
--- /dev/null
+++ b/kubernetes/sdnc/resources/env.yaml
@@ -0,0 +1,19 @@
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SDNC_GEO_ENABLED: "{{ .Values.config.geoEnabled }}"
+SDNC_IS_PRIMARY_CLUSTER: "{{ .Values.config.isPrimaryCluster }}"
+SDNC_ODL_COUNT: "{{ .Values.replicaCount }}"
+SDNC_LOCAL_K8S_CLUSTER_MASTER: "{{ .Values.config.myODLCluster }}"
+SDNC_REMOTE_K8S_CLUSTER_MASTER: "{{ .Values.config.peerODLCluster }}"
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.cluster b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
index d59718fa27..87cdeffe89 100755
--- a/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
@@ -1,6 +1,18 @@
#!/bin/bash
-OOM_HOME=${OOM_HOME:-$HOME}
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
if ! [ "$(command -v jq)" ]; then
echo "Error: jq is not installed."
@@ -8,7 +20,8 @@ if ! [ "$(command -v jq)" ]; then
exit 1
fi
-IS_PRIMARY_CLUSTER=`./sdnc.isPrimaryCluster`
+dir=$( dirname $0 )
+IS_PRIMARY_CLUSTER=$( $dir/sdnc.isPrimaryCluster )
case $IS_PRIMARY_CLUSTER in
true)
@@ -18,21 +31,30 @@ false)
MEMBER_NUMBER=4
;;
*)
- echo "Error: isPrimaryODLCluster not defined in ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml."
+ echo "Error: isPrimaryCluster not defined in $dir/../../../values.yaml."
exit 1
;;
esac
+USERNAME=admin
+PASSWORD=admin
+
for pod_number in {0..2}
do
- curl "http://localhost:3026$((${pod_number} + 1))" > /dev/null 2>&1
- if [ "$?" = "7" ]; then
+
+ response=`curl -s -u $USERNAME:$PASSWORD -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3026$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore`
+
+ if [ $? -ne 0 ]; then
continue
fi
- VOTING_RESULT=`curl -u admin:admin -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3026$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore 2>/dev/null | jq '.value.Voting'`
+ status=$( echo -E "$response" | jq -r ".status" )
+ if [ "$status" != "200" ]; then
+ continue
+ fi
- case $VOTING_RESULT in
+ voting=$( echo -E "$response" | jq -r ".value.Voting" )
+ case $voting in
true)
echo "active"
exit 0
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.failover b/kubernetes/sdnc/resources/geo/bin/sdnc.failover
deleted file mode 100755
index 961a5cb5cf..0000000000
--- a/kubernetes/sdnc/resources/geo/bin/sdnc.failover
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/perl -s
-use strict;
-
-my $keyWord_standby = "standby";
-my $keyWord_active = "active";
-my $keyWord_true = "true";
-my $keyWord_false = "false";
-my $keyWord_success = "success";
-my $keyWord_failure = "failure";
-my $file_cluster = "sdnc.cluster";
-my $file_switchVoting = "switchVoting.sh";
-my $file_isPrimaryCluster = "sdnc.isPrimaryCluster";
-
-if ((!(-e $file_cluster)) || (!(-e $file_switchVoting))|| (!(-e $file_isPrimaryCluster))) {
- # file not exist.
- print qq|$keyWord_failure\n|;
- exit 1;
-}
-
-my $roleRes = qx("./$file_isPrimaryCluster");
-my $clusterRes = qx("./$file_cluster");
-
-if ( index ($clusterRes, $keyWord_standby) != -1) {
- # We are at standby side
- if ( index ($roleRes, $keyWord_false) != -1) {
- # We are at Secondary cluster
- sub_activate_secondary();
- } elsif ( index ($roleRes, $keyWord_true) != -1) {
- # We are at Primary cluster
- sub_activate_primary();
- } else {
- # Error.
- print qq|$keyWord_failure\n|;
- exit 1;
- }
-} elsif ( index ($clusterRes, $keyWord_active) != -1) {
- # We are at active side
- if ( index ($roleRes, $keyWord_false) != -1) {
- # We are at Secondary cluster
- sub_activate_primary();
- } elsif ( index ($roleRes, $keyWord_true) != -1) {
- # We are at Primary cluster
- sub_activate_secondary();
- } else {
- # Error.
- print qq|$keyWord_failure\n|;
- exit 1;
- }
-} else {
- # Error.
- print qq|$keyWord_failure\n|;
- exit 1;
-}
-
-sub sub_activate_primary {
- #Switching voting in Primary cluster
- system("./$file_switchVoting primary");
- print qq|$keyWord_success\n|;
-}
-
-sub sub_activate_secondary {
- #Switching voting in secondary cluster
- system("./$file_switchVoting secondary");
- print qq|$keyWord_success\n|;
-}
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
index 77fc65fe39..7a4f6a7dd0 100755
--- a/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
@@ -1,8 +1,22 @@
#!/bin/bash
-OOM_HOME=${OOM_HOME:-$HOME}
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
-IS_PRIMARY_CLUSTER=`awk '/isPrimaryCluster/ {print $2}' ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml`
+dir=$( dirname $0 )
+
+IS_PRIMARY_CLUSTER=`awk '/isPrimaryCluster/ {print $2}' $dir/../../../values.yaml`
if [ "$?" -eq "2" ]; then
echo "Make sure you are ubuntu user." >&2
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive b/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive
new file mode 100755
index 0000000000..76eca48af5
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];then
+ echo "Usage: makeactive <release> [namespace]"
+ exit 1
+fi
+
+RELEASE=$1
+NAMESPACE=onap
+if [ -n "$2" ];then
+ NAMESPACE=$2
+fi
+
+dir=$(dirname $0)
+isPrimary=$( $dir/sdnc.isPrimaryCluster)
+if [ "$isPrimary" = "true" ];then
+ SITE_NAME="sdnc01"
+elif [ "$isPrimary" = "false" ];then
+ SITE_NAME="sdnc02"
+else
+ echo "sdnc.isPrimaryCluster returned unexpected value \"$isPrimary\""
+ exit 1
+fi
+
+pod=$( kubectl -n $NAMESPACE get pods -l app=sdnc-prom,release=$RELEASE | grep Running | cut -f1 -d' ' )
+if [ -z "$pod" ];then
+ echo "prom pod not found - is prom running?"
+ exit 1
+fi
+
+kubectl -n $NAMESPACE exec $pod -- /app/promoverride.py --id $SITE_NAME --config /app/config/config.json
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.monitor b/kubernetes/sdnc/resources/geo/bin/sdnc.monitor
new file mode 100755
index 0000000000..b14bd7325d
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.monitor
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];then
+ echo "Usage: $(basename $0) [--debug] <release> [namespace]"
+ exit 1
+fi
+
+if [ "$1" = "--debug" -o "$1" = "-debug" -o "$1" = "-d" ];then
+ DEBUG="--debug"
+ shift
+fi
+
+RELEASE=$1
+NAMESPACE=onap
+if [ -n "$2" ];then
+ NAMESPACE=$2
+fi
+
+pod=$( kubectl -n $NAMESPACE get pods -l app=sdnc-prom,release=$RELEASE | grep Running | cut -f1 -d' ' )
+if [ -z "$pod" ];then
+ echo "prom pod not found - is prom running?"
+ exit 1
+fi
+
+kubectl -n $NAMESPACE exec $pod -- /app/bin/sdnc.monitor $DEBUG
diff --git a/kubernetes/sdnc/resources/geo/bin/switchVoting.sh b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
index 27e4ead99d..7a1c193492 100755
--- a/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
+++ b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
@@ -1,5 +1,19 @@
#!/bin/bash
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
function usage()
{
echo usage: switchVoting.sh primary\|secondary
@@ -10,18 +24,26 @@ if [ $# -ne 1 ]; then
usage
fi
-partition=$1
+dir=$( dirname $0 )
+USERNAME=admin
+PASSWORD=`awk '/odlPassword/ {print $2}' $dir/../../../values.yaml | head -1`
-if [ "$partition" == "primary" ]; then
- curl -u admin:{{.Values.config.odlPassword}} -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' > switch_voting_resp.json 2>/dev/null
- echo "" >> switch_voting_resp.json
- exit 0
-fi
+case "$1" in
-if [ "$partition" == "secondary" ]; then
- curl -u admin:{{.Values.config.odlPassword}} -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' > switch_voting_resp.json 2>/dev/null
- echo "" >> switch_voting_resp.json
- exit 0
-fi
+primary)
+ status=$(curl -u $USERNAME:$PASSWORD -o /dev/null -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' -w "%{http_code}\n" $url 2> /dev/null)
+;;
-usage
+secondary)
+ status=$(curl -u $USERNAME:$PASSWORD -o /dev/null -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' -w "%{http_code}\n" $url 2> /dev/null)
+;;
+
+*)
+ usage
+esac
+
+if [ $status -ne 200 ];then
+ echo "failure"
+else
+ echo "success"
+fi
diff --git a/kubernetes/sdnc/sdnc-prom/Chart.yaml b/kubernetes/sdnc/sdnc-prom/Chart.yaml
new file mode 100644
index 0000000000..3c678e2b92
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/Chart.yaml
@@ -0,0 +1,18 @@
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: ONAP SDNC Policy Driven Ownership Management
+name: sdnc-prom
+version: 2.0.0
diff --git a/kubernetes/sdnc/sdnc-prom/requirements.yaml b/kubernetes/sdnc/sdnc-prom/requirements.yaml
new file mode 100644
index 0000000000..37545fe2e0
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/requirements.yaml
@@ -0,0 +1,18 @@
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+ - name: common
+ version: ~2.0.0
+ repository: '@local'
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh b/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh
new file mode 100755
index 0000000000..fb24653129
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncActive.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+debugLog(){
+ if [ "$enableDebugLogging" == true ]; then
+ if [ $# -eq 0 ]; then
+ echo "" >> $LOGFILE
+ else
+ echo $( date ) $@ >> $LOGFILE
+ fi
+ fi
+}
+
+failover(){
+ lockFile=/tmp/sdnc.failover.lock
+ # make sure that no failover is currently running
+ if [ -e ${lockFile} ] && kill -0 $(cat ${lockFile}) 2> /dev/null; then
+ debugLog "Currently running sdnc and dns failover"
+ return
+ fi
+ trap "rm -f ${lockFile}" INT TERM RETURN
+ echo $BASHPID > ${lockFile}
+
+ # perform takeover
+ debugLog "Started executing sdnc.failover for $SITE_NAME"
+ takeoverResult=$( /app/bin/sdnc.failover )
+ debugLog "Completed executing sdnc.failover. takeoverResult is: $takeoverResult"
+ if [ "success" = "$takeoverResult" ]; then
+ # update CoreDNS upon successful execution of sdnc.failover script
+ debugLog "Executing sdnc.dnsswitch"
+ /app/bin/sdnc.dnsswitch
+ rc=$?
+ debugLog "Completed executing sdnc.dnsswitch for $SITE_NAME. rc=$rc"
+ else
+ debugLog "Cluster takeover current status: $takeoverResult on $SITE_NAME."
+ rc=1
+ fi
+
+ if [ $rc -ne 0 ];then
+ takeoverResult="failure"
+ fi
+
+ data="{\
+\"type\": \"failover\",\
+\"status\": \"$takeoverResult\",\
+\"site\": \"$SITE_NAME\",\
+\"deployment\": \"{{.Values.config.deployment}}\",\
+\"timestamp\": \"$(date '+%F %T')\"\
+}"
+
+ # notifications are best-effort - ignore any failures
+ curl -H "Content-Type: application/json" -X POST --data "$data" http://$message_router/events/$topic >/dev/null 2>&1
+
+}
+
+LOGFILE="/app/geo.log"
+enableDebugLogging=true
+message_router=message-router:3904
+topic={{.Values.config.messageRouterTopic}}
+SITE_NAME="sdnc01"
+if [ "$SDNC_IS_PRIMARY_CLUSTER" = "false" ];then
+ SITE_NAME="sdnc02"
+fi
+
+debugLog
+debugLog "Executing ensureSdncActive"
+
+# query SDN-C cluster status
+debugLog "Started executing sdnc.cluster"
+clusterStatus=$( /app/bin/sdnc.cluster )
+debugLog "Completed executing sdnc.cluster. Cluster status is: $clusterStatus"
+
+if [ "active" = "$clusterStatus" ]; then
+ # peform health-check
+ debugLog "Started excuting sdnc.monitor"
+ health=$( /app/bin/sdnc.monitor )
+ debugLog "Completed executing sdnc.monitor. Cluster is: $health"
+
+ if [ "healthy" = "$health" ]; then
+ # Cluster is ACTIVE and HEALTHY
+ exit 0
+ fi
+ exit 1
+
+elif [ "standby" = "$clusterStatus" ]; then
+ # Run failover in background process and allow PROM to continue
+ ( failover & )
+ exit 0
+fi
+
+# Unknown cluster status
+exit 1
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncStandby.sh b/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncStandby.sh
new file mode 100755
index 0000000000..8dd84bd3ea
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/ensureSdncStandby.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+debugLog(){
+ if [ "$enableDebugLogging" == true ]; then
+ if [ $# -eq 0 ]; then
+ echo "" >> $LOGFILE
+ else
+ echo $( date ) $@ >> $LOGFILE
+ fi
+ fi
+}
+
+LOGFILE="/app/geo.log"
+enableDebugLogging=true
+
+debugLog
+debugLog "Executing ensureSdncStandby"
+
+# query SDN-C cluster status
+debugLog "Started executing sdnc.cluster"
+clusterStatus=$( /app/bin/sdnc.cluster )
+debugLog "Completed executing sdnc.cluster. Cluster status is: $clusterStatus"
+
+if [ "active" = "$clusterStatus" ]; then
+ # assume transient error as other side transitions to ACTIVE
+ debugLog "Cluster status: $clusterStatus. exit 0"
+ exit 0
+
+elif [ "standby" = "$clusterStatus" ]; then
+ # check that standby cluster is healthy
+ debugLog "Started executing sdnc.monitor. Cluster status is: $clusterStatus"
+ health=$( /app/bin/sdnc.monitor )
+ debugLog "Completed executing sdnc.monitor. Cluster is: $health"
+ if [ "failure" = "$health" ];then
+ # Backup site is unhealthy - can't accept traffic!
+ exit 1
+ fi
+ # Cluster is standing by
+ exit 0
+fi
+
+debugLog "Unknown cluster status: $clusterStatus"
+# Unknown cluster status
+exit 1
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/prom.sh b/kubernetes/sdnc/sdnc-prom/resources/bin/prom.sh
new file mode 100755
index 0000000000..c93ba24bd7
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/prom.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "${SDNC_IS_PRIMARY_CLUSTER:-true}" = "true" ];then
+ id=sdnc01
+else
+ id=sdnc02
+fi
+
+# should PROM start as passive?
+state=$( bin/sdnc.cluster )
+if [ "$state" == "standby" ]; then
+ echo "Starting PROM in passive mode"
+ passive="-p"
+fi
+
+# start PROM as foreground process
+java -jar prom.jar --id $id $passive --config config
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.cluster b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.cluster
new file mode 100755
index 0000000000..76603410d4
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.cluster
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# query ODL cluster state
+USERNAME="{{.Values.odl.jolokia.username}}"
+PASSWORD="{{.Values.odl.jolokia.password}}"
+
+count=${SDNC_ODL_COUNT:-1}
+memberStart=0
+if [ "${SDNC_IS_PRIMARY_CLUSTER:-true}" != "true" ];then
+ memberStart=$(( $memberStart + $count ))
+fi
+
+for instance in $(seq $count);do
+ shard=member-$(( $memberStart + $instance ))-shard-default-config
+ mbean=Category=Shards,name=$shard,type=DistributedConfigDatastore
+ url=http://{{.Release.Name}}-sdnc-$(( $instance-1 )).sdnc-cluster.{{.Release.Namespace}}:8181/jolokia/read/org.opendaylight.controller:$mbean
+
+ response=$( curl -s -u $USERNAME:$PASSWORD $url )
+ rc=$?
+ if [ $rc -ne 0 ];then
+ # failed to contact SDN-C instance - try another
+ continue
+ fi
+ status=$( echo -E "$response" | jq -r ".status" )
+ if [ "$status" != "200" ];then
+ # query failed, try another instance
+ continue
+ fi
+
+ voting=$( echo -E "$response" | jq -r ".value.Voting" )
+ case $voting in
+ true)
+ echo "active"
+ exit 0
+ ;;
+ false)
+ echo "standby"
+ exit 0
+ ;;
+ *)
+ echo "Error: Voting status could not be determined."
+ exit 1
+ ;;
+ esac
+done
+echo "Error: Voting status could not be determined."
+exit 1
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.dnsswitch b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.dnsswitch
new file mode 100755
index 0000000000..209352c4e3
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.dnsswitch
@@ -0,0 +1,22 @@
+#! /bin/bash
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+####################################################################################################
+# sdncDnsSwitchWrapper.bash: Wrapper script to invoke SDNC DNS Switch for domain: sdnc.example.com #
+####################################################################################################
+ssh -i {{.Values.coreDNS.sshKeyFile}} -o StrictHostKeyChecking=no {{.Values.coreDNS.sshUser}}@{{.Values.coreDNS.host}} "{{.Values.coreDNS.switchScript}} $SDNC_LOCAL_K8S_CLUSTER_MASTER {{.Values.config.deployment}}"
+
+exit $?
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover
new file mode 100755
index 0000000000..e78b7eeee3
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.failover
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOGFILE="/app/geo.log"
+enableDebugLogging=true
+message_router=message-router:3904
+topic={{.Values.config.messageRouterTopic}}
+KEYWORD_success="success"
+KEYWORD_failure="failure"
+SITE_NAME="sdnc01"
+if [ "$SDNC_IS_PRIMARY_CLUSTER" = "false" ];then
+ SITE_NAME="sdnc02"
+fi
+
+APP_BIN=/app/bin
+
+debugLog(){
+ if [ "$enableDebugLogging" == true ]; then
+ if [ $# -eq 0 ]; then
+ echo "" >> $LOGFILE
+ else
+ echo $( date ) $@ >> $LOGFILE
+ fi
+ fi
+}
+
+EXC_SIMPLE_FAILOVER=`${APP_BIN}/switchVoting.sh`
+
+if [ "$EXC_SIMPLE_FAILOVER" == "success" ]; then
+ debugLog "Simple failover success. SDNC failover completed."
+else
+ # Simple failover failed. Trying catastrophic failover ...
+ debugLog "Simple failover failed. Trying catastrophic failover for $SITE_NAME ..."
+
+ # Notify Dmaap before executing catastrophic failover, because all connections will be reset.
+ data="{\
+ \"type\": \"Catastrophic failover\",\
+ \"reason\": \"Simple failover failed\",\
+ \"message_router\": \"$message_router\",\
+ \"topic\": \"$topic\",\
+ \"site\": \"$SITE_NAME\",\
+ \"deployment\": \"{{.Values.config.deployment}}\",\
+ \"timestamp\": \"$(date '+%F %T')\"\
+ }"
+
+ debugLog "$data"
+
+ # notifications to Dmaap
+ curl -H "Content-Type: application/json" -X POST --data "$data" http://$message_router/events/$topic >/dev/null 2>&1
+
+ # We're going to kill prom, so we need to do dnsswitch now
+
+ debugLog "Executing sdnc.dnsswitch"
+
+ /app/bin/sdnc.dnsswitch > /dev/null 2>&1
+ rc=$?
+ if [ $rc -ne 0 ];then
+ debugLog "sdnc.dnsswitch FAILED"
+ echo $KEYWORD_failure
+ exit 0
+ fi
+
+ # Now do catastrophic failure
+
+ debugLog "Catastrophic failover in progress"
+
+ ssh -o StrictHostKeyChecking=no -i /app/config/coredns/master.key root@$SDNC_LOCAL_K8S_CLUSTER_MASTER "su - ubuntu bash -c 'helm upgrade --set sdnc.config.geoEnabled=false dev local/onap --namespace onap; kubectl -n onap delete pods -l app=sdnc'" > /dev/null 2>&1
+
+ # Sleep here so prom can die without us passing control back to ensureSDNCActive
+ sleep 300
+fi
+
+echo $KEYWORD_success
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.monitor b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.monitor
new file mode 100755
index 0000000000..0042ac368a
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/sdnc.monitor
@@ -0,0 +1,125 @@
+#!/usr/bin/env python2
+# encoding: utf-8
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+import json
+import requests
+from datetime import datetime
+
+consul_server = "consul-server:8500"
+message_router = "message-router:3904"
+topic = '{{.Values.config.messageRouterTopic}}'
+log_file='/app/monitor.log'
+status_file='/app/.health'
+logEnabled=False
+
+siteName='sdnc01'
+if os.environ.get('SDNC_IS_PRIMARY_CLUSTER', 'true') == 'false':
+ siteName='sdnc02'
+
+debug=False
+if len(sys.argv) > 1 and sys.argv[1] == '--debug':
+ debug=True
+
+def get_state(healthcheck):
+ response = requests.get("http://" + consul_server + "/v1/health/checks/" + healthcheck)
+ if response.status_code != 200:
+ raise RuntimeError("HTTP " + str(response.status_code))
+ data = response.json()
+ if len(data) == 0:
+ raise RuntimeError(healthcheck + " not found")
+ if len(data) > 1:
+ raise RuntimeError("Multiple states for " + healthcheck + " found")
+
+ return data[0]
+
+
+def log(message):
+ if logEnabled:
+ with open(log_file, 'a') as f:
+ f.write(str(datetime.now()) + " " + message + "\n")
+
+def healthcheck(checks, failFirst=True):
+ if len(checks) == 0:
+ return True
+
+ for check in checks:
+ if type(check) is list:
+ passing = healthcheck(check, False)
+ else:
+ state = get_state(check)
+ status = state['Status']
+ passing = status == "passing" or status == "warning"
+ log(check + " " + status)
+ if debug:
+ if status == "passing":
+ color = "\033[32m" # green
+ elif status == "warning":
+ color = "\033[33m" # yellow
+ else:
+ color = "\033[31m" # red
+ print check, color + status + "\033[0m"
+ if not passing:
+ print "\tCause:", state['Output']
+
+
+ if passing:
+ if not failFirst:
+ # found a passing check so can stop here
+ return True
+ else:
+ if failFirst:
+ # found a failing check so can stop here
+ return False
+
+ return failFirst
+
+
+try:
+ with open("/app/config/healthchecks.json") as f:
+ checks = json.load(f)
+
+ try:
+ with open(status_file) as f:
+ previous_result = f.read()
+ except IOError:
+ # file doesn't exist
+ previous_result = 'unknown'
+
+ if healthcheck(checks):
+ result = "healthy"
+ else:
+ result = "unhealthy"
+
+ print result
+
+ # save current result to file
+ with open(status_file, 'w') as f:
+ f.write(result)
+
+ if previous_result != 'unknown' and result != previous_result:
+ payload = { 'type' : 'health-change', 'status': result, 'site': siteName, 'deployment': '{{.Values.config.deployment}}', 'timestamp': str(datetime.now()) }
+ log("Posting event " + str(payload))
+ try:
+ requests.post("http://" + message_router + "/events/" + topic, data=json.dumps(payload), headers={ 'Content-Type' : 'application/json' } )
+ except Exception:
+ # events are best-effort
+ pass
+
+except Exception as e:
+ sys.exit(str(e))
diff --git a/kubernetes/sdnc/sdnc-prom/resources/bin/switchVoting.sh b/kubernetes/sdnc/sdnc-prom/resources/bin/switchVoting.sh
new file mode 100755
index 0000000000..f13196e7e8
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/bin/switchVoting.sh
@@ -0,0 +1,110 @@
+#/bin/sh
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+primary=${SDNC_IS_PRIMARY_CLUSTER:-true}
+
+url=http://sdnc:8282/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards
+username="${ODL_USERNAME:-{{.Values.odl.restconf.username}}}"
+password="${ODL_PASSWORD:-{{.Values.odl.restconf.password}}}"
+LOGFILE="/app/geo.log"
+enableDebugLogging=true
+
+debugLog(){
+ if [ "$enableDebugLogging" == true ]; then
+ if [ $# -eq 0 ]; then
+ echo "" >> $LOGFILE
+ else
+ echo $( date ) $@ >> $LOGFILE
+ fi
+ fi
+}
+
+
+if [ "$primary" = "true" ]; then
+ votingState='
+{
+ "input": {
+ "member-voting-state": [
+ {
+ "member-name": "member-1",
+ "voting": true
+ },
+ {
+ "member-name": "member-2",
+ "voting": true
+ },
+ {
+ "member-name": "member-3",
+ "voting": true
+ },
+ {
+ "member-name": "member-4",
+ "voting": false
+ },
+ {
+ "member-name": "member-5",
+ "voting": false
+ },
+ {
+ "member-name": "member-6",
+ "voting": false
+ }
+ ]
+ }
+}'
+else
+ votingState='
+{
+ "input": {
+ "member-voting-state": [
+ {
+ "member-name": "member-1",
+ "voting": false
+ },
+ {
+ "member-name": "member-2",
+ "voting": false
+ },
+ {
+ "member-name": "member-3",
+ "voting": false
+ },
+ {
+ "member-name": "member-4",
+ "voting": true
+ },
+ {
+ "member-name": "member-5",
+ "voting": true
+ },
+ {
+ "member-name": "member-6",
+ "voting": true
+ }
+ ]
+ }
+}'
+fi
+
+status=$(curl -s -u $username:$password -o /dev/null -H "Content-Type: application/json" -H "Accept: application/json" -X POST -d "$votingState" -w "%{http_code}\n" $url 2> /dev/null)
+if [ $status -ne 200 ];then
+ debugLog "Switch voting failed. status: $status ,username: $username ,password: $password ,votingState: $votingState ,url:$url "
+ echo "failure"
+else
+ echo "success"
+fi
+
diff --git a/kubernetes/sdnc/sdnc-prom/resources/config/config.json b/kubernetes/sdnc/sdnc-prom/resources/config/config.json
new file mode 100644
index 0000000000..54f95c140c
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/config/config.json
@@ -0,0 +1,19 @@
+{
+ "app-name": "{{.Values.config.deployment}}",
+ "aid": "{{.Values.config.aid}}",
+ "namespace": "{{.Values.config.deployment}}",
+ "userid": "{{.Values.config.deployment}}",
+ "password": "{{.Values.config.password}}",
+ "prom-timeout": "{{.Values.config.promTimeout}}",
+ "core-monitor-sleep-time": "{{.Values.config.coreMonitorSleepTime}}",
+ "no-of-retry-attempts": "{{.Values.config.noOfRetryAttempts}}",
+ "restart-backoff-time": "{{.Values.config.restartBackoffTime}}",
+ "replica-id-list": [ "sdnc01", "sdnc02" ],
+ "ensure-active-sdnc01": "/app/bin/ensureSdncActive.sh",
+ "ensure-active-sdnc02": "/app/bin/ensureSdncActive.sh",
+ "ensure-passive-sdnc01": "/app/bin/ensureSdncStandby.sh",
+ "ensure-passive-sdnc02": "/app/bin/ensureSdncStandby.sh",
+ "music-connection-timeout-ms": "{{.Values.config.musicConnectionTimeoutMs}}",
+ "music-location": {{.Values.config.musicLocation|toJson}},
+ "music-version": "2"
+}
diff --git a/kubernetes/sdnc/sdnc-prom/resources/config/healthchecks.json b/kubernetes/sdnc/sdnc-prom/resources/config/healthchecks.json
new file mode 100644
index 0000000000..ea8ceccc0c
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/resources/config/healthchecks.json
@@ -0,0 +1 @@
+{{.Values.config.healthChecks|toJson}}
diff --git a/kubernetes/sdnc/sdnc-prom/templates/configmap.yaml b/kubernetes/sdnc/sdnc-prom/templates/configmap.yaml
new file mode 100644
index 0000000000..cf4332334b
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/templates/configmap.yaml
@@ -0,0 +1,29 @@
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "common.fullname" . }}-configmap
+ namespace: {{ include "common.namespace" . }}
+data:
+{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "common.fullname" . }}-scripts
+ namespace: {{ include "common.namespace" . }}
+data:
+{{ tpl (.Files.Glob "resources/bin/*").AsConfig . | indent 2 }}
diff --git a/kubernetes/sdnc/sdnc-prom/templates/deployment.yaml b/kubernetes/sdnc/sdnc-prom/templates/deployment.yaml
new file mode 100644
index 0000000000..76f722c65f
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/templates/deployment.yaml
@@ -0,0 +1,95 @@
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: {{ include "common.fullname" . }}
+ namespace: {{ include "common.namespace" . }}
+ labels:
+ app: {{ include "common.name" . }}
+ chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ release: {{ .Release.Name }}
+ heritage: {{ .Release.Service }}
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: {{ include "common.name" . }}
+ release: {{ .Release.Name }}
+ spec:
+ initContainers:
+ - command:
+ - /root/ready.py
+ args:
+ - --container-name
+ - sdnc
+ - --container-name
+ - consul
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+ imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+ name: {{ include "common.name" . }}-readiness
+ containers:
+ - name: {{ include "common.name" . }}
+ image: "{{ include "common.repository" . }}/{{ .Values.image }}"
+ imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+ envFrom:
+ - configMapRef:
+ name: {{ .Release.Name }}-sdnc-env
+ workingDir: "/app"
+ command: [ "bin/prom.sh" ]
+ volumeMounts:
+ - name: localtime
+ mountPath: /etc/localtime
+ readOnly: true
+ - name: prom-config
+ mountPath: /app/config
+ - name: prom-scripts
+ mountPath: /app/bin
+ - name: core-dns-keyfile
+ mountPath: /app/config/coredns
+
+ resources:
+{{ toYaml .Values.resources | indent 12 }}
+ {{- if .Values.nodeSelector }}
+ nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 10 }}
+ {{- end -}}
+ {{- if .Values.affinity }}
+ affinity:
+{{ toYaml .Values.affinity | indent 10 }}
+ {{- end }}
+ volumes:
+ - name: localtime
+ hostPath:
+ path: /etc/localtime
+ - name: prom-config
+ configMap:
+ name: {{ include "common.fullname" . }}-configmap
+ - name: prom-scripts
+ configMap:
+ name: {{ include "common.fullname" . }}-scripts
+ defaultMode: 0755
+ - name: core-dns-keyfile
+ hostPath:
+ path: {{ .Values.global.persistence.mountPath | default .Values.persistence.mountPath }}/{{ .Release.Name }}/{{ .Values.persistence.mountSubPath }}
+ imagePullSecrets:
+ - name: {{ include "common.namespace" . }}-docker-registry-key"
diff --git a/kubernetes/sdnc/sdnc-prom/values.yaml b/kubernetes/sdnc/sdnc-prom/values.yaml
new file mode 100644
index 0000000000..fbee9b9d96
--- /dev/null
+++ b/kubernetes/sdnc/sdnc-prom/values.yaml
@@ -0,0 +1,102 @@
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#################################################################
+# Global configuration defaults.
+#################################################################
+global:
+ nodePortPrefix: 302
+ repository: nexus3.onap.org:10001
+ readinessRepository: oomk8s
+ readinessImage: readiness-check:2.0.0
+ loggingRepository: docker.elastic.co
+ loggingImage: beats/filebeat:5.5.0
+ persistence:
+ mountPath: /dockerdata-nfs
+
+#################################################################
+# Application configuration defaults.
+#################################################################
+# application image
+repository: nexus3.onap.org:10001
+pullPolicy: Always
+image: onap/music/prom:1.0.5-latest
+
+# application configuration
+config:
+ # generate aid from onboarding your app in MUSIC
+ aid: "aid_for_your_app"
+ deployment: "test_onap"
+ password: "onap123"
+ musicLocation:
+ - "1.2.3.4"
+ - "1.2.3.5"
+ - "1.2.3.6"
+ musicConnectionTimeoutMs: "1000"
+ promTimeout: "35000"
+ coreMonitorSleepTime: "15000"
+ noOfRetryAttempts: "2"
+ restartBackoffTime: "15000"
+ healthChecks:
+ # All top-level checks must pass
+ - "Health Check: SDNC - SDN Host"
+ - "Health Check: SDNC"
+ - "Health Check: SDNC ODL Cluster"
+ - "Health Check: SDNC Portal"
+ # Within nested lists, only one must pass
+ - - "Health Check: SDNC-SDN-CTL-DB-01"
+ - "Health Check: SDNC-SDN-CTL-DB-02"
+ messageRouterTopic: "SDNC-GEO-REDUNDANCY"
+
+odl:
+ jolokia:
+ username: "admin"
+ password: "admin"
+ restconf:
+ username: "admin"
+ password: "Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U"
+
+coreDNS:
+ host: 1.2.3.7
+ sshUser: root
+ sshKeyFile: /app/config/coredns/coredns.key
+ switchScript: /home/ubuntu/dnsSwitch.bash
+
+nodeSelector: {}
+
+affinity: {}
+
+# probe configuration parameters
+liveness:
+ initialDelaySeconds: 90
+ periodSeconds: 90
+ # necessary to disable liveness probe when setting breakpoints
+ # in debugger so K8s doesn't restart unresponsive container
+ enabled: true
+
+readiness:
+ initialDelaySeconds: 10
+ periodSeconds: 10
+
+persistence:
+ enabled: true
+ accessMode: ReadWriteOnce
+ size: 1Gi
+ mountPath: /dockerdata-nfs
+ mountSubPath: coredns
+
+ingress:
+ enabled: false
+
+resources: {}
diff --git a/kubernetes/sdnc/templates/configmap.yaml b/kubernetes/sdnc/templates/configmap.yaml
index e9498cb8dc..6f4ee2285e 100644
--- a/kubernetes/sdnc/templates/configmap.yaml
+++ b/kubernetes/sdnc/templates/configmap.yaml
@@ -43,3 +43,11 @@ metadata:
namespace: {{ include "common.namespace" . }}
data:
{{ tpl (.Files.Glob "resources/config/conf/*").AsConfig . | indent 2 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "common.fullname" . }}-env
+ namespace: {{ include "common.namespace" . }}
+data:
+{{ tpl (.Files.Get "resources/env.yaml") . | indent 2 }}
diff --git a/kubernetes/sdnc/values.yaml b/kubernetes/sdnc/values.yaml
index 0c602e50cd..c214d71a81 100644
--- a/kubernetes/sdnc/values.yaml
+++ b/kubernetes/sdnc/values.yaml
@@ -48,7 +48,7 @@ config:
# if geoEnabled is set to true the following 3 values must be set to their proper values
myODLCluster: 127.0.0.1
peerODLCluster: 127.0.0.1
- isPrimaryCluster: false
+ isPrimaryCluster: true
configDir: /opt/onap/sdnc/data/properties
dmaapTopic: SUCCESS
dmaapPort: 3904
@@ -57,7 +57,6 @@ config:
ansibleServiceName: sdnc-ansible-server
ansiblePort: 8000
-
# dependency / sub-chart configuration
dmaap-listener:
nameOverride: sdnc-dmaap-listener