summaryrefslogtreecommitdiffstats
path: root/ms/blueprintsprocessor/application/src
diff options
context:
space:
mode:
authorBrinda Santh <bs2796@att.com>2020-02-05 15:51:03 -0500
committerBrinda Santh <bs2796@att.com>2020-02-12 14:16:28 -0500
commit65bb9d0d83762e8fa8e3ab568c801908eafa0686 (patch)
treead8ae7fafc1954b44ddd9b72d1fceb482f042431 /ms/blueprintsprocessor/application/src
parent723cb0b0f4fca052561f21bb8312bf7c6e8cd524 (diff)
Cluster co-ordination with Hazelcast.
Remove Atomix implementation, due to Kubernetes clustering issues. Cluster environment property changes. Issue-ID: CCSDK-2011 Signed-off-by: Brinda Santh <bs2796@att.com> Change-Id: I23f40c92c0adc6b3ab8690871385f78525c76433
Diffstat (limited to 'ms/blueprintsprocessor/application/src')
-rw-r--r--ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml71
-rwxr-xr-xms/blueprintsprocessor/application/src/main/docker/distribution.xml5
-rw-r--r--ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt32
-rw-r--r--ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf35
-rw-r--r--ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf40
-rw-r--r--ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast-client.yaml13
-rw-r--r--ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast.yaml18
-rw-r--r--ms/blueprintsprocessor/application/src/main/resources/logback.xml2
8 files changed, 99 insertions, 117 deletions
diff --git a/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml b/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml
index d53b85f98..451f35c3a 100644
--- a/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml
+++ b/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml
@@ -44,7 +44,7 @@ services:
- cds-network
ports:
- "8000:8080"
- - "9111:9111"
+ - "9110:9111"
restart: always
volumes:
- target: /opt/app/onap/blueprints/deploy
@@ -58,26 +58,58 @@ services:
CLUSTER_ENABLED: "true"
CLUSTER_ID: cds-cluster
CLUSTER_NODE_ID: cds-controller-0
- CLUSTER_MEMBERS: cds-controller-0,resource-resolution-0
- CLUSTER_STORAGE_PATH: /opt/app/onap/config/cluster
- #CLUSTER_CONFIG_FILE: /opt/app/onap/config/atomix/atomix-multicast.conf
+ CLUSTER_JOIN_AS_CLIENT: "false"
+ CLUSTER_CONFIG_FILE: /opt/app/onap/config/hazelcast/hazelcast.yaml
NATS_CLUSTER_ID: cds-cluster
APP_NAME: cds-controller
BUNDLEVERSION: 1.0.0
APP_CONFIG_HOME: /opt/app/onap/config
STICKYSELECTORKEY:
ENVCONTEXT: dev
- resource-resolution-0:
+ cds-controller-1:
depends_on:
- db
- nats
image: onap/ccsdk-blueprintsprocessor:latest
- container_name: resource-resolution-0
- hostname: resource-resolution-0
+ container_name: cds-controller-1
+ hostname: cds-controller-1
networks:
- cds-network
ports:
- "8001:8080"
+ - "9111:9111"
+ restart: always
+ volumes:
+ - target: /opt/app/onap/blueprints/deploy
+ type: volume
+ source: blueprints-deploy
+ - target: /opt/app/onap/config
+ type: volume
+ source: controller-config
+ environment:
+ # Same as hostname and container name
+ CLUSTER_ENABLED: "true"
+ CLUSTER_ID: cds-cluster
+ CLUSTER_NODE_ID: cds-controller-1
+ CLUSTER_JOIN_AS_CLIENT: "false"
+ CLUSTER_CONFIG_FILE: /opt/app/onap/config/hazelcast/hazelcast.yaml
+ NATS_CLUSTER_ID: cds-cluster
+ APP_NAME: cds-controller
+ BUNDLEVERSION: 1.0.0
+ APP_CONFIG_HOME: /opt/app/onap/config
+ STICKYSELECTORKEY:
+ ENVCONTEXT: dev
+ cds-controller-2:
+ depends_on:
+ - db
+ - nats
+ image: onap/ccsdk-blueprintsprocessor:latest
+ container_name: cds-controller-2
+ hostname: cds-controller-2
+ networks:
+ - cds-network
+ ports:
+ - "8002:8080"
- "9112:9111"
restart: always
volumes:
@@ -86,16 +118,16 @@ services:
source: blueprints-deploy
- target: /opt/app/onap/config
type: volume
- source: resource-resolution-config
+ source: controller-config
environment:
+ # Same as hostname and container name
CLUSTER_ENABLED: "true"
CLUSTER_ID: cds-cluster
- CLUSTER_NODE_ID: resource-resolution-0
- CLUSTER_MEMBERS: cds-controller-0,resource-resolution-0
- CLUSTER_STORAGE_PATH: /opt/app/onap/config/cluster
- #CLUSTER_CONFIG_FILE: /opt/app/onap/config/atomix/atomix-multicast.conf
+ CLUSTER_NODE_ID: cds-controller-2
+ CLUSTER_JOIN_AS_CLIENT: "false"
+ CLUSTER_CONFIG_FILE: /opt/app/onap/config/hazelcast/hazelcast.yaml
NATS_CLUSTER_ID: cds-cluster
- APP_NAME: resource-resolution
+ APP_NAME: cds-controller
BUNDLEVERSION: 1.0.0
APP_CONFIG_HOME: /opt/app/onap/config
STICKYSELECTORKEY:
@@ -119,7 +151,8 @@ services:
environment:
CLUSTER_ID: cds-cluster
CLUSTER_NODE_ID: py-executor-0
- CLUSTER_MEMBERS: cds-controller-0,resource-resolution-0,py-executor-0
+ CLUSTER_JOIN_AS_CLIENT: "false"
+ CLUSTER_CONFIG_FILE: /opt/app/onap/config/hazelcast/hazelcast.yaml
NATS_CLUSTER_ID: cds-cluster
APP_NAME: py-executor
BUNDLEVERSION: 1.0.0
@@ -140,7 +173,7 @@ volumes:
driver: local
driver_opts:
type: none
- device: /opt/app/cds/nats/nats-0/store
+ device: /opt/app/cds/nats/store
o: bind
blueprints-deploy:
driver: local
@@ -152,13 +185,7 @@ volumes:
driver: local
driver_opts:
type: none
- device: /opt/app/cds/cds-controller/cds-controller-0/config
- o: bind
- resource-resolution-config:
- driver: local
- driver_opts:
- type: none
- device: /opt/app/cds/resource-resolution/resource-resolution-0/config
+ device: /opt/app/cds/cds-controller/config
o: bind
networks:
diff --git a/ms/blueprintsprocessor/application/src/main/docker/distribution.xml b/ms/blueprintsprocessor/application/src/main/docker/distribution.xml
index 291dca09b..9a079ac55 100755
--- a/ms/blueprintsprocessor/application/src/main/docker/distribution.xml
+++ b/ms/blueprintsprocessor/application/src/main/docker/distribution.xml
@@ -45,6 +45,11 @@
<useDefaultExcludes>true</useDefaultExcludes>
</fileSet>
<fileSet>
+ <directory>${project.basedir}/src/main/resources/hazelcast</directory>
+ <outputDirectory>opt/app/onap/config/hazelcast</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ </fileSet>
+ <fileSet>
<directory>${project.basedir}/src/main/resources/certs</directory>
<outputDirectory>opt/app/onap/config/certs</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt b/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt
index 4c9314ec2..16cb5d6e2 100644
--- a/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt
+++ b/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt
@@ -20,14 +20,13 @@ import kotlinx.coroutines.runBlocking
import org.onap.ccsdk.cds.blueprintsprocessor.core.service.BluePrintClusterService
import org.onap.ccsdk.cds.blueprintsprocessor.core.service.ClusterInfo
import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
-import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
import org.onap.ccsdk.cds.controllerblueprints.core.logger
-import org.onap.ccsdk.cds.controllerblueprints.core.splitCommaAsList
import org.onap.ccsdk.cds.controllerblueprints.core.utils.ClusterUtils
import org.springframework.boot.context.event.ApplicationReadyEvent
import org.springframework.context.event.EventListener
import org.springframework.stereotype.Component
import java.time.Duration
+import java.util.Properties
import javax.annotation.PreDestroy
/**
@@ -44,15 +43,13 @@ import javax.annotation.PreDestroy
* 2. Container names should end with sequence number.
* Blueprintprocessor example be : cds-controller-1, cds-controller-2, cds-controller-3
* ResourceResolution example be : resource-resolution-1, resource-resolution-2, resource-resolution-3
- * 3. Each contained, should have environment properties CLUSTER_ID, CLUSTER_NODE_ID, CLUSTER_NODE_ADDRESS,
- * CLUSTER_MEMBERS, CLUSTER_STORAGE_PATH
+ * 3. Each contained, should have environment properties CLUSTER_ID, CLUSTER_NODE_ID, CLUSTER_JOIN_AS_CLIENT,
+ * CLUSTER_CONFIG_FILE
* Example values :
* CLUSTER_ID: cds-cluster
* CLUSTER_NODE_ID: cds-controller-2
- * CLUSTER_NODE_ADDRESS: cds-controller-2
- * CLUSTER_MEMBERS: cds-controller-1,cds-controller-2,cds-controller-3,resource-resolution-1,resource-resolution-2,resource-resolution-3
- * CLUSTER_STORAGE_PATH: /opt/app/onap/config/cluster
- * CLUSTER_CONFIG_FILE: /opt/app/onap/config/atomix/atomix-multicast.conf
+ * CLUSTER_JOIN_AS_CLIENT: "true" or "false"
+ * CLUSTER_CONFIG_FILE: <Config location>
* 4. Cluster will be enabled only all the above properties present in the environments.
* if CLUSTER_ENABLED is present, then it will try to create cluster.
*/
@@ -68,23 +65,20 @@ open class BluePrintProcessorCluster(private val bluePrintClusterService: BluePr
val clusterId = ClusterUtils.clusterId()
val nodeId = ClusterUtils.clusterNodeId()
- val nodeAddress = ClusterUtils.clusterNodeAddress()
- val clusterMembers = System.getenv(BluePrintConstants.PROPERTY_CLUSTER_MEMBERS)
- ?: throw BluePrintProcessorException("couldn't get environment variable ${BluePrintConstants.PROPERTY_CLUSTER_MEMBERS}")
-
- val clusterMemberList = clusterMembers.splitCommaAsList()
-
- val clusterStorage = System.getenv(BluePrintConstants.PROPERTY_CLUSTER_STORAGE_PATH)
- ?: throw BluePrintProcessorException("couldn't get environment variable ${BluePrintConstants.PROPERTY_CLUSTER_STORAGE_PATH}")
+ val joinAsClient =
+ (System.getenv(BluePrintConstants.PROPERTY_CLUSTER_JOIN_AS_CLIENT) ?: "false").toBoolean()
val clusterConfigFile = System.getenv(BluePrintConstants.PROPERTY_CLUSTER_CONFIG_FILE)
+ val properties = Properties()
+ properties["hazelcast.logging.type"] = "slf4j"
+
val clusterInfo = ClusterInfo(
id = clusterId, nodeId = nodeId,
- clusterMembers = clusterMemberList, nodeAddress = nodeAddress,
- storagePath = clusterStorage,
- configFile = clusterConfigFile
+ joinAsClient = joinAsClient,
+ configFile = clusterConfigFile,
+ properties = properties
)
bluePrintClusterService.startCluster(clusterInfo)
} else {
diff --git a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf b/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf
deleted file mode 100644
index 0fc31e00f..000000000
--- a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf
+++ /dev/null
@@ -1,35 +0,0 @@
-cluster {
- # Configure the cluster node information.
- node {
- id: ${CLUSTER_NODE_ID}
- address: ${CLUSTER_NODE_ADDRESS}
- }
- # Configure the node discovery protocol.
- discovery {
- type: bootstrap
- nodes.1 {
- id: cds-controller-1
- address: "cds-controller-1:5679"
- }
- nodes.2 {
- id: resource-reolution-1
- address: "resource-reolution-1:5679"
- }
- }
-}
-# Configure the system management group.
-managementGroup {
- type: raft
- name: system
- partitions: 1
- members: [${CLUSTER_MEMBERS}]
- storage {
- directory: ${CLUSTER_STORAGE_PATH}/data-${CLUSTER_NODE_ID}
- level: DISK
- }
-}
-# Configure a Raft partition group.
-partitionGroups.data {
- type: primary-backup
- partitions: 7
-}
diff --git a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf b/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf
deleted file mode 100644
index fd161879c..000000000
--- a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf
+++ /dev/null
@@ -1,40 +0,0 @@
-cluster {
- # Configure the cluster node information.
- node {
- id: ${CLUSTER_NODE_ID}
- address: ${CLUSTER_NODE_ADDRESS}
- }
- # Configure the node discovery protocol.
- discovery {
- type: multicast
- }
- multicast: {
- enabled: true
- port: 54321
- }
- # Configure the SWIM membership protocol.
- protocol {
- type: swim
- broadcastUpdates: true
- gossipInterval: 500ms
- probeInterval: 2s
- suspectProbes: 2
- }
-}
-# Configure the system management group.
-managementGroup {
- type: raft
- name: system
- partitions: 1
- members: [${CLUSTER_MEMBERS}]
- storage {
- directory: ${CLUSTER_STORAGE_PATH}/data-${CLUSTER_NODE_ID}
- level: DISK
- }
-}
-
-# Configure a Raft partition group.
-partitionGroups.data {
- type: primary-backup
- partitions: 7
-}
diff --git a/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast-client.yaml b/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast-client.yaml
new file mode 100644
index 000000000..e60b5dfc4
--- /dev/null
+++ b/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast-client.yaml
@@ -0,0 +1,13 @@
+hazelcast-client:
+ cluster-name: ${CLUSTER_ID}
+ instance-name: ${CLUSTER_NODE_ID}
+
+ network:
+ cluster-members:
+ - 127.0.0.1:5701
+# kubernetes:
+# enabled: true
+# namespace: MY-KUBERNETES-NAMESPACE
+# service-name: MY-SERVICE-NAME
+# service-label-name: MY-SERVICE-LABEL-NAME
+# service-label-value: MY-SERVICE-LABEL-VALUE
diff --git a/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast.yaml b/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast.yaml
new file mode 100644
index 000000000..bacbe2a45
--- /dev/null
+++ b/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast.yaml
@@ -0,0 +1,18 @@
+hazelcast:
+ cluster-name: ${CLUSTER_ID}
+ instance-name: ${CLUSTER_NODE_ID}
+ lite-member:
+ enabled: false
+ cp-subsystem:
+ cp-member-count: 3
+ group-size: 3
+# network:
+# join:
+# multicast:
+# enabled: false
+# kubernetes:
+# enabled: true
+# namespace: MY-KUBERNETES-NAMESPACE
+# service-name: MY-SERVICE-NAME
+# service-label-name: MY-SERVICE-LABEL-NAME
+# service-label-value: MY-SERVICE-LABEL-VALUE \ No newline at end of file
diff --git a/ms/blueprintsprocessor/application/src/main/resources/logback.xml b/ms/blueprintsprocessor/application/src/main/resources/logback.xml
index 63ede28b4..aceea4327 100644
--- a/ms/blueprintsprocessor/application/src/main/resources/logback.xml
+++ b/ms/blueprintsprocessor/application/src/main/resources/logback.xml
@@ -28,7 +28,7 @@
<logger name="org.springframework" level="info"/>
<logger name="org.springframework.web" level="info"/>
<logger name="org.hibernate" level="error"/>
- <logger name="io.atomix" level="warn"/>
+ <logger name="com.hazelcast" level="error"/>
<logger name="org.onap.ccsdk.cds" level="info"/>
<root level="info">