summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHao Kuang <Hao.Kuang@amdocs.com>2017-12-15 20:38:03 +0000
committerHao Kuang <Hao.Kuang@amdocs.com>2017-12-20 16:43:29 +0000
commit27af4a308d56795eddb08d07cbb86d8651c4f5a2 (patch)
treea2acc5bff9251f6f5c00f859488c289c7f3211c2
parenta6eb9be0e091746c824b169a46242ef0943a2b05 (diff)
Single K8S Node Of Deploying SDNC Cluster POC
It's a reference of how to deploy sdnc cluster under single node k8s environment. Currently the config of nfs for mysql db only supports one k8s node. For mutiple nodes, because of dockernfs mount, nfs provisioner has conflict with it, which fails mount operation of SDNC pod. Also, if SDNC needs to fully utilize mysql cluster under k8s, in core project, DbLibService.java should be enhanced to support that writing to master only and reading from them. Issue-ID: SDNC-163 Change-Id: I85ad327f2fe04fe8e9a135289d7c25a917234438 Signed-off-by: Hao Kuang <Hao.Kuang@amdocs.com>
-rw-r--r--kubernetes/sdnc/templates/all-services.yaml68
-rw-r--r--kubernetes/sdnc/templates/db-deployment.yaml48
-rw-r--r--kubernetes/sdnc/templates/db-statefulset.yaml193
-rw-r--r--kubernetes/sdnc/templates/mysql-configmap.yaml20
-rw-r--r--kubernetes/sdnc/templates/nfs-provisoner-deployment.yaml55
-rw-r--r--kubernetes/sdnc/templates/sdnc-data-storageclass.yaml8
-rw-r--r--kubernetes/sdnc/templates/sdnc-pv-pvc.yaml32
-rw-r--r--kubernetes/sdnc/templates/sdnc-statefulset.yaml (renamed from kubernetes/sdnc/templates/sdnc-deployment.yaml)20
-rw-r--r--kubernetes/sdnc/values.yaml10
9 files changed, 365 insertions, 89 deletions
diff --git a/kubernetes/sdnc/templates/all-services.yaml b/kubernetes/sdnc/templates/all-services.yaml
index cfb24bf98a..9425ef4b80 100644
--- a/kubernetes/sdnc/templates/all-services.yaml
+++ b/kubernetes/sdnc/templates/all-services.yaml
@@ -13,6 +13,22 @@ spec:
app: sdnc-dbhost
clusterIP: None
---
+# Client service for connecting to any MySQL instance for reads.
+# Only master: sdnc-dbhost-0 accepts the write request.
+apiVersion: v1
+kind: Service
+metadata:
+ name: dbhost-read
+ namespace: "{{ .Values.nsPrefix }}-sdnc"
+ labels:
+ app: sdnc-dbhost
+spec:
+ ports:
+ - name: sdnc-dbhost
+ port: 3306
+ selector:
+ app: sdnc-dbhost
+---
apiVersion: v1
kind: Service
metadata:
@@ -26,7 +42,6 @@ spec:
selector:
app: sdnc-dbhost
clusterIP: None
-#{{ end }}
---
apiVersion: v1
kind: Service
@@ -41,7 +56,8 @@ spec:
selector:
app: sdnc-dbhost
clusterIP: None
-#{{ if not .Values.disableSdncSdncDgbuilder }}
+#{{ end }}
+#{{ if not .Values.disableSdncSdnc }}
---
apiVersion: v1
kind: Service
@@ -59,8 +75,6 @@ spec:
type: NodePort
selector:
app: sdnc-dgbuilder
-#{{ end }}
-#{{ if not .Values.disableSdncSdnc }}
---
apiVersion: v1
kind: Service
@@ -91,9 +105,34 @@ spec:
port: 8201
targetPort: 8101
nodePort: {{ .Values.nodePortPrefix }}08
+ - name: "sdnc-jolokia-port-8080"
+ port: 8280
+ targetPort: 8080
+ nodePort: {{ .Values.nodePortPrefix }}46
type: NodePort
selector:
app: sdnc
+---
+kind: Service
+apiVersion: v1
+metadata:
+ name: nfs-provisioner
+ namespace: "{{ .Values.nsPrefix }}-sdnc"
+ labels:
+ app: nfs-provisioner
+spec:
+ ports:
+ - name: nfs
+ port: 2049
+ - name: mountd
+ port: 20048
+ - name: rpcbind
+ port: 111
+ - name: rpcbind-udp
+ port: 111
+ protocol: UDP
+ selector:
+ app: nfs-provisioner
#{{ end }}
#{{ if not .Values.disableSdncSdncPortal }}
---
@@ -124,3 +163,24 @@ spec:
selector:
app: sdnc-portal
#{{ end }}
+#{{ if .Values.enableODLCluster }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: sdnhost-cluster
+ namespace: "{{ .Values.nsPrefix }}-sdnc"
+ labels:
+ app: sdnc
+ annotations:
+ service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+spec:
+ ports:
+ - name: "sdnc-cluster-port"
+ port: 2550
+ clusterIP: None
+ selector:
+ app: sdnc
+ sessionAffinity: None
+ type: ClusterIP
+#{{ end }}
diff --git a/kubernetes/sdnc/templates/db-deployment.yaml b/kubernetes/sdnc/templates/db-deployment.yaml
deleted file mode 100644
index e2723b5e45..0000000000
--- a/kubernetes/sdnc/templates/db-deployment.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-#{{ if not .Values.disableSdncSdncDbhost }}
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
- name: sdnc-dbhost
- namespace: "{{ .Values.nsPrefix }}-sdnc"
-spec:
- selector:
- matchLabels:
- app: sdnc-dbhost
- template:
- metadata:
- labels:
- app: sdnc-dbhost
- name: sdnc-dbhost
- spec:
- containers:
- - env:
- - name: MYSQL_ROOT_PASSWORD
- value: openECOMP1.0
- - name: MYSQL_ROOT_HOST
- value: '%'
- image: {{ .Values.image.mysqlServer }}
- imagePullPolicy: {{ .Values.pullPolicy }}
- name: sdnc-db-container
- volumeMounts:
- - mountPath: /etc/localtime
- name: localtime
- readOnly: true
- - mountPath: /var/lib/mysql
- name: sdnc-data
- ports:
- - containerPort: 3306
- readinessProbe:
- tcpSocket:
- port: 3306
- initialDelaySeconds: 5
- periodSeconds: 10
- volumes:
- - name: localtime
- hostPath:
- path: /etc/localtime
- - name: sdnc-data
- persistentVolumeClaim:
- claimName: sdnc-db
- imagePullSecrets:
- - name: "{{ .Values.nsPrefix }}-docker-registry-key"
-#{{ end }} \ No newline at end of file
diff --git a/kubernetes/sdnc/templates/db-statefulset.yaml b/kubernetes/sdnc/templates/db-statefulset.yaml
new file mode 100644
index 0000000000..f2e079d76e
--- /dev/null
+++ b/kubernetes/sdnc/templates/db-statefulset.yaml
@@ -0,0 +1,193 @@
+#{{ if not .Values.disableSdncSdncDbhost }}
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: sdnc-dbhost
+ namespace: "{{ .Values.nsPrefix }}-sdnc"
+spec:
+ serviceName: "dbhost"
+ replicas: {{ .Values.numberOfDbReplicas }}
+ selector:
+ matchLabels:
+ app: sdnc-dbhost
+ template:
+ metadata:
+ labels:
+ app: sdnc-dbhost
+ name: sdnc-dbhost
+ spec:
+ initContainers:
+ - name: init-mysql
+ image: {{ .Values.image.mysql }}
+ imagePullPolicy: {{ .Values.pullPolicy }}
+ command:
+ - bash
+ - "-c"
+ - |
+ set -ex
+ # Generate mysql server-id from pod ordinal index.
+ [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+ ordinal=${BASH_REMATCH[1]}
+ echo BASH_REMATCH=${BASH_REMATCH}
+ echo [mysqld] > /mnt/conf.d/server-id.cnf
+ # Add an offset to avoid reserved server-id=0 value.
+ echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
+ # Copy appropriate conf.d files from config-map to emptyDir.
+ if [[ $ordinal -eq 0 ]]; then
+ cp /mnt/config-map/master.cnf /mnt/conf.d/
+ else
+ cp /mnt/config-map/slave.cnf /mnt/conf.d/
+ fi
+ volumeMounts:
+ - name: conf
+ mountPath: /mnt/conf.d
+ - name: config-map
+ mountPath: /mnt/config-map
+ - name: clone-mysql
+ image: {{ .Values.image.xtrabackup }}
+ env:
+ - name: MYSQL_ROOT_PASSWORD
+ value: openECOMP1.0
+ command:
+ - bash
+ - "-c"
+ - |
+ set -ex
+ # Skip the clone if data already exists.
+ [[ -d /var/lib/mysql/mysql ]] && exit 0
+ # Skip the clone on master (ordinal index 0).
+ [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+ ordinal=${BASH_REMATCH[1]}
+ echo ${BASH_REMATCH}
+ [[ $ordinal -eq 0 ]] && exit 0
+ # Clone data from previous peer.
+ ncat --recv-only sdnc-dbhost-$(($ordinal-1)).dbhost.{{ .Values.nsPrefix }}-sdnc 3307 | xbstream -x -C /var/lib/mysql
+ # Prepare the backup.
+ xtrabackup --user=root --password=$MYSQL_ROOT_PASSWORD --prepare --target-dir=/var/lib/mysql
+ ls -l /var/lib/mysql
+ volumeMounts:
+ - name: sdnc-data
+ mountPath: /var/lib/mysql
+ subPath: mysql
+ - name: conf
+ mountPath: /etc/mysql/conf.d
+ containers:
+ - env:
+ - name: MYSQL_ROOT_PASSWORD
+ value: openECOMP1.0
+ - name: MYSQL_ROOT_HOST
+ value: '%'
+ - name: MYSQL_ALLOW_EMPTY_PASSWORD
+ value: "0"
+ image: {{ .Values.image.mysql }}
+ imagePullPolicy: {{ .Values.pullPolicy }}
+ name: sdnc-db-container
+ volumeMounts:
+ - mountPath: /var/lib/mysql
+ name: sdnc-data
+ subPath: mysql
+ - name: conf
+ mountPath: /etc/mysql/conf.d
+ ports:
+ - containerPort: 3306
+ resources:
+ requests:
+ cpu: 500m
+ memory: 1Gi
+ livenessProbe:
+ exec:
+ command: ["mysqladmin", "ping"]
+ initialDelaySeconds: 30
+ periodSeconds: 10
+ timeoutSeconds: 5
+ readinessProbe:
+ tcpSocket:
+ port: 3306
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ - name: xtrabackup
+ image: {{ .Values.image.xtrabackup }}
+ env:
+ - name: MYSQL_ROOT_PASSWORD
+ value: openECOMP1.0
+ ports:
+ - name: xtrabackup
+ containerPort: 3307
+ command:
+ - bash
+ - "-c"
+ - |
+ set -ex
+ cd /var/lib/mysql
+ ls -l
+ # Determine binlog position of cloned data, if any.
+ if [[ -f xtrabackup_slave_info ]]; then
+ echo "Inside xtrabackup_slave_info"
+ # XtraBackup already generated a partial "CHANGE MASTER TO" query
+ # because we're cloning from an existing slave.
+ mv xtrabackup_slave_info change_master_to.sql.in
+ # Ignore xtrabackup_binlog_info in this case (it's useless).
+ rm -f xtrabackup_binlog_info
+ elif [[ -f xtrabackup_binlog_info ]]; then
+ echo "Inside xtrabackup_binlog_info"
+ # We're cloning directly from master. Parse binlog position.
+ [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
+ rm xtrabackup_binlog_info
+ echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
+ MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
+ fi
+
+ # Check if we need to complete a clone by starting replication.
+ if [[ -f change_master_to.sql.in ]]; then
+ echo "Waiting for mysqld to be ready (accepting connections)"
+ [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+ ordinal=${BASH_REMATCH[1]}
+ echo $ordinal
+ until mysql --user=root --password=$MYSQL_ROOT_PASSWORD -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
+
+ echo "Initializing replication from clone position"
+ # In case of container restart, attempt this at-most-once.
+ mv change_master_to.sql.in change_master_to.sql.orig
+ mysql --user=root --password=$MYSQL_ROOT_PASSWORD -h 127.0.0.1 <<EOF
+ $(<change_master_to.sql.orig),
+ MASTER_HOST="sdnc-dbhost-0.dbhost.{{ .Values.nsPrefix }}-sdnc",
+ MASTER_USER="root",
+ MASTER_PASSWORD="$MYSQL_ROOT_PASSWORD",
+ MASTER_CONNECT_RETRY=10;
+ START SLAVE;
+ EOF
+ fi
+
+ # Start a server to send backups when requested by peers.
+ exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
+ "xtrabackup --user=root --password=$MYSQL_ROOT_PASSWORD --backup --slave-info --stream=xbstream --host=127.0.0.1"
+ volumeMounts:
+ - name: sdnc-data
+ mountPath: /var/lib/mysql
+ subPath: mysql
+ - name: conf
+ mountPath: /etc/mysql/conf.d
+ resources:
+ requests:
+ cpu: 100m
+ memory: 100Mi
+ volumes:
+ - name: conf
+ emptyDir: {}
+ - name: config-map
+ configMap:
+ name: mysql
+ - name: localtime
+ hostPath:
+ path: /etc/localtime
+ volumeClaimTemplates:
+ - metadata:
+ name: sdnc-data
+ annotations:
+ volume.beta.kubernetes.io/storage-class: "{{ .Values.nsPrefix }}-sdnc-data"
+ spec:
+ accessModes: ["ReadWriteMany"]
+ resources:
+ requests:
+ storage: 1Gi
+#{{ end }}
diff --git a/kubernetes/sdnc/templates/mysql-configmap.yaml b/kubernetes/sdnc/templates/mysql-configmap.yaml
new file mode 100644
index 0000000000..deaa997f02
--- /dev/null
+++ b/kubernetes/sdnc/templates/mysql-configmap.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: mysql
+ namespace: "{{ .Values.nsPrefix }}-sdnc"
+ labels:
+ app: mysql
+data:
+ master.cnf: |
+ # Apply this config only on the master.
+ [mysqld]
+ log-bin
+ [localpathprefix]
+ master
+ slave.cnf: |
+ # Apply this config only on slaves.
+ [mysqld]
+ super-read-only
+ [localpathprefix]
+ slave
diff --git a/kubernetes/sdnc/templates/nfs-provisoner-deployment.yaml b/kubernetes/sdnc/templates/nfs-provisoner-deployment.yaml
new file mode 100644
index 0000000000..9d3feeff9c
--- /dev/null
+++ b/kubernetes/sdnc/templates/nfs-provisoner-deployment.yaml
@@ -0,0 +1,55 @@
+#{{ if not .Values.disableSdncSdncDbhost }}
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+ name: nfs-provisioner
+ namespace: "{{ .Values.nsPrefix }}-sdnc"
+spec:
+ replicas: 1
+ strategy:
+ type: Recreate
+ template:
+ metadata:
+ labels:
+ app: nfs-provisioner
+ spec:
+ containers:
+ - name: nfs-provisioner
+ image: quay.io/kubernetes_incubator/nfs-provisioner:v1.0.8
+ ports:
+ - name: nfs
+ containerPort: 2049
+ - name: mountd
+ containerPort: 20048
+ - name: rpcbind
+ containerPort: 111
+ - name: rpcbind-udp
+ containerPort: 111
+ protocol: UDP
+ securityContext:
+ capabilities:
+ add:
+ - DAC_READ_SEARCH
+ - SYS_RESOURCE
+ args:
+ - "-provisioner=sdnc/nfs"
+ env:
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: SERVICE_NAME
+ value: nfs-provisioner
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ imagePullPolicy: "IfNotPresent"
+ volumeMounts:
+ - name: export-volume
+ mountPath: /export
+ volumes:
+ - name: export-volume
+ hostPath:
+ path: /dockerdata-nfs/{{ .Values.nsPrefix }}/sdnc/data
+#{{ end }}
diff --git a/kubernetes/sdnc/templates/sdnc-data-storageclass.yaml b/kubernetes/sdnc/templates/sdnc-data-storageclass.yaml
new file mode 100644
index 0000000000..7c46cf4a68
--- /dev/null
+++ b/kubernetes/sdnc/templates/sdnc-data-storageclass.yaml
@@ -0,0 +1,8 @@
+#{{ if not .Values.disableSdncSdncDbhost }}
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+ name: "{{ .Values.nsPrefix }}-sdnc-data"
+ namespace: "{{ .Values.nsPrefix }}-sdnc"
+provisioner: sdnc/nfs
+#{{ end }}
diff --git a/kubernetes/sdnc/templates/sdnc-pv-pvc.yaml b/kubernetes/sdnc/templates/sdnc-pv-pvc.yaml
deleted file mode 100644
index f45d8f817d..0000000000
--- a/kubernetes/sdnc/templates/sdnc-pv-pvc.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-#{{ if not .Values.disableSdncSdncDbhost }}
-apiVersion: v1
-kind: PersistentVolume
-metadata:
- name: "{{ .Values.nsPrefix }}-sdnc-db"
- namespace: "{{ .Values.nsPrefix }}-sdnc"
- labels:
- name: "{{ .Values.nsPrefix }}-sdnc-db"
-spec:
- capacity:
- storage: 2Gi
- accessModes:
- - ReadWriteMany
- persistentVolumeReclaimPolicy: Retain
- hostPath:
- path: /dockerdata-nfs/{{ .Values.nsPrefix }}/sdnc/data
----
-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
- name: sdnc-db
- namespace: "{{ .Values.nsPrefix }}-sdnc"
-spec:
- accessModes:
- - ReadWriteMany
- resources:
- requests:
- storage: 2Gi
- selector:
- matchLabels:
- name: "{{ .Values.nsPrefix }}-sdnc-db"
-#{{ end }} \ No newline at end of file
diff --git a/kubernetes/sdnc/templates/sdnc-deployment.yaml b/kubernetes/sdnc/templates/sdnc-statefulset.yaml
index 1c88fffea9..3ba37af197 100644
--- a/kubernetes/sdnc/templates/sdnc-deployment.yaml
+++ b/kubernetes/sdnc/templates/sdnc-statefulset.yaml
@@ -1,10 +1,13 @@
#{{ if not .Values.disableSdncSdnc }}
-apiVersion: extensions/v1beta1
-kind: Deployment
+apiVersion: apps/v1beta1
+kind: StatefulSet
metadata:
name: sdnc
namespace: "{{ .Values.nsPrefix }}-sdnc"
spec:
+ serviceName: "sdnhost-cluster"
+ replicas: {{ .Values.numberOfODLReplicas }}
+ podManagementPolicy: Parallel
selector:
matchLabels:
app: sdnc
@@ -31,12 +34,21 @@ spec:
name: sdnc-readiness
containers:
- command:
- - /opt/onap/sdnc/bin/startODL.sh
+ - bash
+ - "-c"
+ - |
+ sed -i 's/dbhost/sdnc-dbhost-0.dbhost.onap-sdnc.svc.cluster.local/g' /opt/onap/sdnc/data/properties/svclogic.properties
+ sed -i 's/^\(org.onap.ccsdk.sli.jdbc.hosts=\).*/\1sdnc-dbhost-0.dbhost.onap-sdnc.svc.cluster.local/' /opt/onap/sdnc/data/properties/dblib.properties
+ /opt/onap/sdnc/bin/startODL.sh
env:
- name: MYSQL_ROOT_PASSWORD
value: openECOMP1.0
- name: SDNC_CONFIG_DIR
value: /opt/onap/sdnc/data/properties
+ - name: ENABLE_ODL_CLUSTER
+ value: "{{ .Values.enableODLCluster }}"
+ - name: SDNC_REPLICAS
+ value: "{{ .Values.numberOfODLReplicas }}"
image: {{ .Values.image.sdnc }}
imagePullPolicy: {{ .Values.pullPolicy }}
name: sdnc-controller-container
@@ -55,6 +67,8 @@ spec:
ports:
- containerPort: 8181
- containerPort: 8101
+ - containerPort: 2550
+ - containerPort: 8080
readinessProbe:
tcpSocket:
port: 8181
diff --git a/kubernetes/sdnc/values.yaml b/kubernetes/sdnc/values.yaml
index 407d047850..d04ff84cbd 100644
--- a/kubernetes/sdnc/values.yaml
+++ b/kubernetes/sdnc/values.yaml
@@ -3,13 +3,19 @@ pullPolicy: Always
nodePortPrefix: 302
image:
readiness: oomk8s/readiness-check:1.0.0
- mysqlServer: mysql/mysql-server:5.6
+ mysql: mysql:5.7
+ xtrabackup: gcr.io/google-samples/xtrabackup:1.0
dgbuilderSdnc: nexus3.onap.org:10001/onap/ccsdk-dgbuilder-image:v0.1.0
sdnc: nexus3.onap.org:10001/onap/sdnc-image:v1.2.1
admportalSdnc: nexus3.onap.org:10001/onap/admportal-sdnc-image:v1.2.1
filebeat: docker.elastic.co/beats/filebeat:5.5.0
+enableODLCluster: false
+numberOfODLReplicas: 1
+numberOfDbReplicas: 1
+disableSdncSdncDgbuilder: false
+disableSdncSdncPortal: false
# FIXME
# Missing the following deps
# nexus3.onap.org:10001/onap/sdnc-dmaap-listener-image:v1.2.1
-# nexus3.onap.org:10001/onap/sdnc-ueb-listener-image:v1.2.1 \ No newline at end of file
+# nexus3.onap.org:10001/onap/sdnc-ueb-listener-image:v1.2.1