summaryrefslogtreecommitdiffstats
path: root/ms/blueprintsprocessor/application/src/main
diff options
context:
space:
mode:
authorBrinda Santh <bs2796@att.com>2019-12-16 20:59:41 -0500
committerBrinda Santh <bs2796@att.com>2019-12-16 20:59:41 -0500
commit383235b495c32a1762511f1837bc9e98af6226eb (patch)
treeafe0e093f4756b2f82679038a3029c515082c9e2 /ms/blueprintsprocessor/application/src/main
parent73a37ecd64accefc0e4b8a9db2cb9e0127d94408 (diff)
Cluster distributed data store
Add experimental cluster co-ordination service using Atomic framework. Included distributed data store creation utilities. Sample docker compose data cluster between cds controller and resource-resolution instances. Issue-ID: CCSDK-2000 Signed-off-by: Brinda Santh <bs2796@att.com> Change-Id: I4de00e773a996e08fd1d260fc27ed18832433883
Diffstat (limited to 'ms/blueprintsprocessor/application/src/main')
-rw-r--r--ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml85
-rwxr-xr-xms/blueprintsprocessor/application/src/main/dc/docker-compose.yaml14
-rw-r--r--ms/blueprintsprocessor/application/src/main/docker/startService.sh8
-rw-r--r--ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt103
-rw-r--r--ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf35
-rw-r--r--ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf40
-rw-r--r--ms/blueprintsprocessor/application/src/main/resources/logback.xml1
7 files changed, 284 insertions, 2 deletions
diff --git a/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml b/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml
new file mode 100644
index 000000000..f4b4b7995
--- /dev/null
+++ b/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml
@@ -0,0 +1,85 @@
+version: '3.7'
+
+services:
+ db:
+ image: mariadb:latest
+ container_name: ccsdk-mariadb
+ networks:
+ - cds-network
+ ports:
+ - "3306:3306"
+ volumes:
+ - ~/vm_mysql:/var/lib/mysql
+ restart: always
+ environment:
+ MYSQL_ROOT_PASSWORD: sdnctl
+ MYSQL_DATABASE: sdnctl
+ MYSQL_USER: sdnctl
+ MYSQL_PASSWORD: sdnctl
+ cds-controller-1:
+ depends_on:
+ - db
+ image: onap/ccsdk-blueprintsprocessor:latest
+ container_name: cds-controller-1
+ hostname: cds-controller-1
+ networks:
+ - cds-network
+ ports:
+ - "8000:8080"
+ - "9111:9111"
+ restart: always
+ volumes:
+ - target: /opt/app/onap/blueprints/deploy
+ type: volume
+ source: blueprints-deploy
+ - target: /opt/app/onap/config
+ type: bind
+ source: ./config
+ environment:
+ # Same as hostname and container name
+ CLUSTER_ID: cds-cluster
+ CLUSTER_NODE_ID: cds-controller-1
+ CLUSTER_MEMBERS: cds-controller-1,resource-resolution-1
+ CLUSTER_STORAGE_PATH: /opt/app/onap/config/cluster
+ #CLUSTER_CONFIG_FILE: /opt/app/onap/config/atomix/atomix-multicast.conf
+ APPLICATIONNAME: cds-controller
+ BUNDLEVERSION: 1.0.0
+ APP_CONFIG_HOME: /opt/app/onap/config
+ STICKYSELECTORKEY:
+ ENVCONTEXT: dev
+ resource-resolution-1:
+ depends_on:
+ - db
+ image: onap/ccsdk-blueprintsprocessor:latest
+ container_name: resource-resolution-1
+ hostname: resource-resolution-1
+ networks:
+ - cds-network
+ ports:
+ - "8001:8080"
+ - "9112:9111"
+ restart: always
+ volumes:
+ - target: /opt/app/onap/blueprints/deploy
+ type: volume
+ source: blueprints-deploy
+ - target: /opt/app/onap/config
+ type: bind
+ source: ./config
+ environment:
+ CLUSTER_ID: cds-cluster
+ CLUSTER_NODE_ID: resource-resolution-1
+ CLUSTER_MEMBERS: cds-controller-1,resource-resolution-1
+ CLUSTER_STORAGE_PATH: /opt/app/onap/config/cluster
+ #CLUSTER_CONFIG_FILE: /opt/app/onap/config/atomix/atomix-multicast.conf
+ APPLICATIONNAME: resource-resolution
+ BUNDLEVERSION: 1.0.0
+ APP_CONFIG_HOME: /opt/app/onap/config
+ STICKYSELECTORKEY:
+ ENVCONTEXT: dev
+volumes:
+ blueprints-deploy:
+
+networks:
+ cds-network:
+ driver: bridge
diff --git a/ms/blueprintsprocessor/application/src/main/dc/docker-compose.yaml b/ms/blueprintsprocessor/application/src/main/dc/docker-compose.yaml
index 0ff04bf3a..d87770286 100755
--- a/ms/blueprintsprocessor/application/src/main/dc/docker-compose.yaml
+++ b/ms/blueprintsprocessor/application/src/main/dc/docker-compose.yaml
@@ -1,9 +1,11 @@
-version: '3.3'
+version: '3.7'
services:
db:
image: mariadb:latest
container_name: ccsdk-mariadb
+ networks:
+ - cds-network
ports:
- "3306:3306"
volumes:
@@ -20,6 +22,8 @@ services:
image: onap/ccsdk-blueprintsprocessor:latest
container_name: cds-controller-default
hostname: cds-controller-default
+ networks:
+ - cds-network
ports:
- "8000:8080"
- "9111:9111"
@@ -37,6 +41,8 @@ services:
- db
image: onap/ccsdk-commandexecutor:latest
container_name: bp-command-executor
+ networks:
+ - cds-network
ports:
- "50051:50051"
restart: always
@@ -48,6 +54,8 @@ services:
image: onap/ccsdk-py-executor
container_name: py-executor-default
hostname: py-executor-default
+ networks:
+ - cds-network
ports:
- "50052:50052"
restart: always
@@ -65,3 +73,7 @@ services:
volumes:
blueprints-deploy:
+
+networks:
+ cds-network:
+ driver: bridge
diff --git a/ms/blueprintsprocessor/application/src/main/docker/startService.sh b/ms/blueprintsprocessor/application/src/main/docker/startService.sh
index 7dcb5ff16..f5967dcb4 100644
--- a/ms/blueprintsprocessor/application/src/main/docker/startService.sh
+++ b/ms/blueprintsprocessor/application/src/main/docker/startService.sh
@@ -2,7 +2,7 @@
nodeName=BlueprintsProcessor_1.0.0_$(cat /proc/self/cgroup | grep docker | sed s/\\//\\n/g | tail -1)
-echo "APP Config HOME : ${APP_CONFIG_HOME}"
+echo "${CLUSTER_ID}:${CLUSTER_NODE_ID} APP Config HOME : ${APP_CONFIG_HOME}"
export APP_HOME=/opt/app/onap
keytool -import -noprompt -trustcacerts -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit -alias ONAP -import -file $APP_CONFIG_HOME/ONAP_RootCA.cer
@@ -18,4 +18,10 @@ exec java -classpath "/etc:${APP_HOME}/lib/*:/lib/*:/src:/schema:/generated-sour
-Djava.security.egd=file:/dev/./urandom \
-DAPPNAME=${APPLICATIONNAME} -DAPPENV=${APP_ENV} -DAPPVERSION=${APP_VERSION} -DNAMESPACE=${NAMESPACE} \
-Dspring.config.location=${APP_CONFIG_HOME}/ \
+-DCLUSTER_ID=${CLUSTER_ID} \
+-DCLUSTER_NODE_ID=${CLUSTER_NODE_ID} \
+-DCLUSTER_NODE_ADDRESS=${CLUSTER_NODE_ID} \
+-DCLUSTER_MEMBERS=${CLUSTER_MEMBERS} \
+-DCLUSTER_STORAGE_PATH=${CLUSTER_STORAGE_PATH} \
+-DCLUSTER_CONFIG_FILE=${CLUSTER_CONFIG_FILE} \
org.onap.ccsdk.cds.blueprintsprocessor.BlueprintProcessorApplicationKt
diff --git a/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt b/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt
new file mode 100644
index 000000000..b78ebf68b
--- /dev/null
+++ b/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt
@@ -0,0 +1,103 @@
+/*
+ * Copyright © 2018-2019 AT&T Intellectual Property.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onap.ccsdk.cds.blueprintsprocessor
+
+import kotlinx.coroutines.runBlocking
+import org.onap.ccsdk.cds.blueprintsprocessor.core.service.BluePrintClusterService
+import org.onap.ccsdk.cds.blueprintsprocessor.core.service.ClusterInfo
+import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
+import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
+import org.onap.ccsdk.cds.controllerblueprints.core.logger
+import org.springframework.boot.context.event.ApplicationReadyEvent
+import org.springframework.context.event.EventListener
+import org.springframework.stereotype.Component
+import java.time.Duration
+import javax.annotation.PreDestroy
+
+/**
+ * To Start the cluster, minimum 2 Instances/ Replicas od CDS needed.
+ * All instance such as Blueprintprocessor, ResourceResolution, MessagePrioritization should be in
+ * same cluster and should have same cluster name.
+ *
+ * Data can be shared only between the clusters, outside the cluster data can't be shared.
+ * If cds-controller-x instance wants to share data with resource-resolution-x instance, then it should be in the
+ * same cluster.(cds-cluster) and same network (cds-network)
+ *
+ * Assumptions:
+ * 1. Container, Pod and Host names are same.
+ * 2. Container names should end with sequence number.
+ * Blueprintprocessor example be : cds-controller-1, cds-controller-2, cds-controller-3
+ * ResourceResolution example be : resource-resolution-1, resource-resolution-2, resource-resolution-3
+ * 3. Each contained, should have environment properties CLUSTER_ID, CLUSTER_NODE_ID, CLUSTER_NODE_ADDRESS,
+ * CLUSTER_MEMBERS, CLUSTER_STORAGE_PATH
+ * Example values :
+ * CLUSTER_ID: cds-cluster
+ * CLUSTER_NODE_ID: cds-controller-2
+ * CLUSTER_NODE_ADDRESS: cds-controller-2
+ * CLUSTER_MEMBERS: cds-controller-1,cds-controller-2,cds-controller-3,resource-resolution-1,resource-resolution-2,resource-resolution-3
+ * CLUSTER_STORAGE_PATH: /opt/app/onap/config/cluster
+ * CLUSTER_CONFIG_FILE: /opt/app/onap/config/atomix/atomix-multicast.conf
+ * 4. Cluster will be enabled only all the above properties present in the environments.
+ * if CLUSTER_ID is present, then it will try to create cluster.
+ */
+@Component
+open class BluePrintProcessorCluster(private val bluePrintClusterService: BluePrintClusterService) {
+
+ private val log = logger(BluePrintProcessorCluster::class)
+
+ @EventListener(ApplicationReadyEvent::class)
+ fun startAndJoinCluster() = runBlocking {
+ val clusterId = System.getProperty(BluePrintConstants.PROPERTY_CLUSTER_ID)
+
+ if (!clusterId.isNullOrEmpty()) {
+
+ val nodeId = System.getProperty(BluePrintConstants.PROPERTY_CLUSTER_NODE_ID)
+ ?: throw BluePrintProcessorException("couldn't get environment variable ${BluePrintConstants.PROPERTY_CLUSTER_NODE_ID}")
+
+ val nodeAddress = System.getProperty(BluePrintConstants.PROPERTY_CLUSTER_NODE_ADDRESS)
+ ?: throw BluePrintProcessorException("couldn't get environment variable ${BluePrintConstants.PROPERTY_CLUSTER_NODE_ADDRESS}")
+
+ val clusterMembers = System.getProperty(BluePrintConstants.PROPERTY_CLUSTER_MEMBERS)
+ ?: throw BluePrintProcessorException("couldn't get environment variable ${BluePrintConstants.PROPERTY_CLUSTER_MEMBERS}")
+
+ val clusterMemberList = clusterMembers.split(",").map { it.trim() }.toList()
+
+ val clusterStorage = System.getProperty(BluePrintConstants.PROPERTY_CLUSTER_STORAGE_PATH)
+ ?: throw BluePrintProcessorException("couldn't get environment variable ${BluePrintConstants.PROPERTY_CLUSTER_STORAGE_PATH}")
+
+ val clusterConfigFile = System.getProperty(BluePrintConstants.PROPERTY_CLUSTER_CONFIG_FILE)
+
+ val clusterInfo = ClusterInfo(
+ id = clusterId, nodeId = nodeId,
+ clusterMembers = clusterMemberList, nodeAddress = nodeAddress,
+ storagePath = clusterStorage,
+ configFile = clusterConfigFile
+ )
+ bluePrintClusterService.startCluster(clusterInfo)
+ } else {
+ log.info(
+ "Cluster is disabled, to enable cluster set the environment " +
+ "properties[CLUSTER_ID,CLUSTER_NODE_ID, CLUSTER_NODE_ADDRESS, CLUSTER_MEMBERS,CLUSTER_CONFIG_FILE]"
+ )
+ }
+ }
+
+ @PreDestroy
+ fun shutDown() = runBlocking {
+ bluePrintClusterService.shutDown(Duration.ofSeconds(1))
+ }
+}
diff --git a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf b/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf
new file mode 100644
index 000000000..0fc31e00f
--- /dev/null
+++ b/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf
@@ -0,0 +1,35 @@
+cluster {
+ # Configure the cluster node information.
+ node {
+ id: ${CLUSTER_NODE_ID}
+ address: ${CLUSTER_NODE_ADDRESS}
+ }
+ # Configure the node discovery protocol.
+ discovery {
+ type: bootstrap
+ nodes.1 {
+ id: cds-controller-1
+ address: "cds-controller-1:5679"
+ }
+ nodes.2 {
+ id: resource-reolution-1
+ address: "resource-reolution-1:5679"
+ }
+ }
+}
+# Configure the system management group.
+managementGroup {
+ type: raft
+ name: system
+ partitions: 1
+ members: [${CLUSTER_MEMBERS}]
+ storage {
+ directory: ${CLUSTER_STORAGE_PATH}/data-${CLUSTER_NODE_ID}
+ level: DISK
+ }
+}
+# Configure a Raft partition group.
+partitionGroups.data {
+ type: primary-backup
+ partitions: 7
+}
diff --git a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf b/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf
new file mode 100644
index 000000000..fd161879c
--- /dev/null
+++ b/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf
@@ -0,0 +1,40 @@
+cluster {
+ # Configure the cluster node information.
+ node {
+ id: ${CLUSTER_NODE_ID}
+ address: ${CLUSTER_NODE_ADDRESS}
+ }
+ # Configure the node discovery protocol.
+ discovery {
+ type: multicast
+ }
+ multicast: {
+ enabled: true
+ port: 54321
+ }
+ # Configure the SWIM membership protocol.
+ protocol {
+ type: swim
+ broadcastUpdates: true
+ gossipInterval: 500ms
+ probeInterval: 2s
+ suspectProbes: 2
+ }
+}
+# Configure the system management group.
+managementGroup {
+ type: raft
+ name: system
+ partitions: 1
+ members: [${CLUSTER_MEMBERS}]
+ storage {
+ directory: ${CLUSTER_STORAGE_PATH}/data-${CLUSTER_NODE_ID}
+ level: DISK
+ }
+}
+
+# Configure a Raft partition group.
+partitionGroups.data {
+ type: primary-backup
+ partitions: 7
+}
diff --git a/ms/blueprintsprocessor/application/src/main/resources/logback.xml b/ms/blueprintsprocessor/application/src/main/resources/logback.xml
index e1389a66f..d58be8ac7 100644
--- a/ms/blueprintsprocessor/application/src/main/resources/logback.xml
+++ b/ms/blueprintsprocessor/application/src/main/resources/logback.xml
@@ -39,6 +39,7 @@
<logger name="org.springframework" level="info"/>
<logger name="org.springframework.web" level="info"/>
<logger name="org.hibernate" level="error"/>
+ <logger name="io.atomix" level="warn"/>
<logger name="org.onap.ccsdk.cds" level="info"/>
<root level="info">