Cluster co-ordination with Hazelcast.

Remove Atomix implementation, due to Kubernetes clustering issues.

Cluster environment property changes.

Issue-ID: CCSDK-2011
Signed-off-by: Brinda Santh <bs2796@att.com>
Change-Id: I23f40c92c0adc6b3ab8690871385f78525c76433

8 files changed, 99 insertions, 117 deletions

diff --git a/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml b/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml
index d53b85f98..451f35c3a 100644
--- a/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml
+++ b/ms/blueprintsprocessor/application/src/main/dc/docker-compose-cluster.yaml
@@ -44,7 +44,7 @@ services:
       - cds-network
     ports:
       - "8000:8080"
-      - "9111:9111"
+      - "9110:9111"
     restart: always
     volumes:
       - target: /opt/app/onap/blueprints/deploy
@@ -58,26 +58,58 @@ services:
       CLUSTER_ENABLED: "true"
       CLUSTER_ID: cds-cluster
       CLUSTER_NODE_ID: cds-controller-0
-      CLUSTER_MEMBERS: cds-controller-0,resource-resolution-0
-      CLUSTER_STORAGE_PATH: /opt/app/onap/config/cluster
-      #CLUSTER_CONFIG_FILE: /opt/app/onap/config/atomix/atomix-multicast.conf
+      CLUSTER_JOIN_AS_CLIENT: "false"
+      CLUSTER_CONFIG_FILE: /opt/app/onap/config/hazelcast/hazelcast.yaml
       NATS_CLUSTER_ID: cds-cluster
       APP_NAME: cds-controller
       BUNDLEVERSION: 1.0.0
       APP_CONFIG_HOME: /opt/app/onap/config
       STICKYSELECTORKEY:
       ENVCONTEXT: dev
-  resource-resolution-0:
+  cds-controller-1:
     depends_on:
       - db
       - nats
     image: onap/ccsdk-blueprintsprocessor:latest
-    container_name: resource-resolution-0
-    hostname: resource-resolution-0
+    container_name: cds-controller-1
+    hostname: cds-controller-1
     networks:
       - cds-network
     ports:
       - "8001:8080"
+      - "9111:9111"
+    restart: always
+    volumes:
+      - target: /opt/app/onap/blueprints/deploy
+        type: volume
+        source: blueprints-deploy
+      - target: /opt/app/onap/config
+        type: volume
+        source: controller-config
+    environment:
+      # Same as hostname and container name
+      CLUSTER_ENABLED: "true"
+      CLUSTER_ID: cds-cluster
+      CLUSTER_NODE_ID: cds-controller-1
+      CLUSTER_JOIN_AS_CLIENT: "false"
+      CLUSTER_CONFIG_FILE: /opt/app/onap/config/hazelcast/hazelcast.yaml
+      NATS_CLUSTER_ID: cds-cluster
+      APP_NAME: cds-controller
+      BUNDLEVERSION: 1.0.0
+      APP_CONFIG_HOME: /opt/app/onap/config
+      STICKYSELECTORKEY:
+      ENVCONTEXT: dev
+  cds-controller-2:
+    depends_on:
+      - db
+      - nats
+    image: onap/ccsdk-blueprintsprocessor:latest
+    container_name: cds-controller-2
+    hostname: cds-controller-2
+    networks:
+      - cds-network
+    ports:
+      - "8002:8080"
       - "9112:9111"
     restart: always
     volumes:
@@ -86,16 +118,16 @@ services:
         source: blueprints-deploy
       - target: /opt/app/onap/config
         type: volume
-        source: resource-resolution-config
+        source: controller-config
     environment:
+      # Same as hostname and container name
       CLUSTER_ENABLED: "true"
       CLUSTER_ID: cds-cluster
-      CLUSTER_NODE_ID: resource-resolution-0
-      CLUSTER_MEMBERS: cds-controller-0,resource-resolution-0
-      CLUSTER_STORAGE_PATH: /opt/app/onap/config/cluster
-      #CLUSTER_CONFIG_FILE: /opt/app/onap/config/atomix/atomix-multicast.conf
+      CLUSTER_NODE_ID: cds-controller-2
+      CLUSTER_JOIN_AS_CLIENT: "false"
+      CLUSTER_CONFIG_FILE: /opt/app/onap/config/hazelcast/hazelcast.yaml
       NATS_CLUSTER_ID: cds-cluster
-      APP_NAME: resource-resolution
+      APP_NAME: cds-controller
       BUNDLEVERSION: 1.0.0
       APP_CONFIG_HOME: /opt/app/onap/config
       STICKYSELECTORKEY:
@@ -119,7 +151,8 @@ services:
     environment:
       CLUSTER_ID: cds-cluster
       CLUSTER_NODE_ID: py-executor-0
-      CLUSTER_MEMBERS: cds-controller-0,resource-resolution-0,py-executor-0
+      CLUSTER_JOIN_AS_CLIENT: "false"
+      CLUSTER_CONFIG_FILE: /opt/app/onap/config/hazelcast/hazelcast.yaml
       NATS_CLUSTER_ID: cds-cluster
       APP_NAME: py-executor
       BUNDLEVERSION: 1.0.0
@@ -140,7 +173,7 @@ volumes:
     driver: local
     driver_opts:
       type: none
-      device: /opt/app/cds/nats/nats-0/store
+      device: /opt/app/cds/nats/store
       o: bind
   blueprints-deploy:
     driver: local
@@ -152,13 +185,7 @@ volumes:
     driver: local
     driver_opts:
       type: none
-      device: /opt/app/cds/cds-controller/cds-controller-0/config
-      o: bind
-  resource-resolution-config:
-    driver: local
-    driver_opts:
-      type: none
-      device: /opt/app/cds/resource-resolution/resource-resolution-0/config
+      device: /opt/app/cds/cds-controller/config
       o: bind
 
 networks:
diff --git a/ms/blueprintsprocessor/application/src/main/docker/distribution.xml b/ms/blueprintsprocessor/application/src/main/docker/distribution.xml
index 291dca09b..9a079ac55 100755
--- a/ms/blueprintsprocessor/application/src/main/docker/distribution.xml
+++ b/ms/blueprintsprocessor/application/src/main/docker/distribution.xml
@@ -45,6 +45,11 @@
             <useDefaultExcludes>true</useDefaultExcludes>
         </fileSet>
         <fileSet>
+            <directory>${project.basedir}/src/main/resources/hazelcast</directory>
+            <outputDirectory>opt/app/onap/config/hazelcast</outputDirectory>
+            <useDefaultExcludes>true</useDefaultExcludes>
+        </fileSet>
+        <fileSet>
             <directory>${project.basedir}/src/main/resources/certs</directory>
             <outputDirectory>opt/app/onap/config/certs</outputDirectory>
             <useDefaultExcludes>true</useDefaultExcludes>
diff --git a/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt b/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt
index 4c9314ec2..16cb5d6e2 100644
--- a/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt
+++ b/ms/blueprintsprocessor/application/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/BluePrintProcessorCluster.kt
@@ -20,14 +20,13 @@ import kotlinx.coroutines.runBlocking
 import org.onap.ccsdk.cds.blueprintsprocessor.core.service.BluePrintClusterService
 import org.onap.ccsdk.cds.blueprintsprocessor.core.service.ClusterInfo
 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
-import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
 import org.onap.ccsdk.cds.controllerblueprints.core.logger
-import org.onap.ccsdk.cds.controllerblueprints.core.splitCommaAsList
 import org.onap.ccsdk.cds.controllerblueprints.core.utils.ClusterUtils
 import org.springframework.boot.context.event.ApplicationReadyEvent
 import org.springframework.context.event.EventListener
 import org.springframework.stereotype.Component
 import java.time.Duration
+import java.util.Properties
 import javax.annotation.PreDestroy
 
 /**
@@ -44,15 +43,13 @@ import javax.annotation.PreDestroy
  * 2. Container names should end with sequence number.
  *      Blueprintprocessor example be : cds-controller-1, cds-controller-2, cds-controller-3
  *      ResourceResolution example be : resource-resolution-1, resource-resolution-2,  resource-resolution-3
- * 3. Each contained, should have environment properties CLUSTER_ID, CLUSTER_NODE_ID, CLUSTER_NODE_ADDRESS,
- * CLUSTER_MEMBERS, CLUSTER_STORAGE_PATH
+ * 3. Each contained, should have environment properties CLUSTER_ID, CLUSTER_NODE_ID, CLUSTER_JOIN_AS_CLIENT,
+ * CLUSTER_CONFIG_FILE
  *     Example values :
  *      CLUSTER_ID: cds-cluster
  *      CLUSTER_NODE_ID: cds-controller-2
- *      CLUSTER_NODE_ADDRESS: cds-controller-2
- *      CLUSTER_MEMBERS: cds-controller-1,cds-controller-2,cds-controller-3,resource-resolution-1,resource-resolution-2,resource-resolution-3
- *      CLUSTER_STORAGE_PATH: /opt/app/onap/config/cluster
- *      CLUSTER_CONFIG_FILE:  /opt/app/onap/config/atomix/atomix-multicast.conf
+ *      CLUSTER_JOIN_AS_CLIENT: "true" or "false"
+ *      CLUSTER_CONFIG_FILE:  <Config location>
  * 4. Cluster will be enabled only all the above properties present in the environments.
  * if CLUSTER_ENABLED is present, then it will try to create cluster.
  */
@@ -68,23 +65,20 @@ open class BluePrintProcessorCluster(private val bluePrintClusterService: BluePr
 
             val clusterId = ClusterUtils.clusterId()
             val nodeId = ClusterUtils.clusterNodeId()
-            val nodeAddress = ClusterUtils.clusterNodeAddress()
 
-            val clusterMembers = System.getenv(BluePrintConstants.PROPERTY_CLUSTER_MEMBERS)
-                ?: throw BluePrintProcessorException("couldn't get environment variable ${BluePrintConstants.PROPERTY_CLUSTER_MEMBERS}")
-
-            val clusterMemberList = clusterMembers.splitCommaAsList()
-
-            val clusterStorage = System.getenv(BluePrintConstants.PROPERTY_CLUSTER_STORAGE_PATH)
-                ?: throw BluePrintProcessorException("couldn't get environment variable ${BluePrintConstants.PROPERTY_CLUSTER_STORAGE_PATH}")
+            val joinAsClient =
+                (System.getenv(BluePrintConstants.PROPERTY_CLUSTER_JOIN_AS_CLIENT) ?: "false").toBoolean()
 
             val clusterConfigFile = System.getenv(BluePrintConstants.PROPERTY_CLUSTER_CONFIG_FILE)
 
+            val properties = Properties()
+            properties["hazelcast.logging.type"] = "slf4j"
+
             val clusterInfo = ClusterInfo(
                 id = clusterId, nodeId = nodeId,
-                clusterMembers = clusterMemberList, nodeAddress = nodeAddress,
-                storagePath = clusterStorage,
-                configFile = clusterConfigFile
+                joinAsClient = joinAsClient,
+                configFile = clusterConfigFile,
+                properties = properties
             )
             bluePrintClusterService.startCluster(clusterInfo)
         } else {
diff --git a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf b/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf
deleted file mode 100644
index 0fc31e00f..000000000
--- a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-bootstrap.conf
+++ /dev/null
@@ -1,35 +0,0 @@
-cluster {
-  # Configure the cluster node information.
-  node {
-    id: ${CLUSTER_NODE_ID}
-    address: ${CLUSTER_NODE_ADDRESS}
-  }
-  # Configure the node discovery protocol.
-  discovery {
-    type: bootstrap
-    nodes.1 {
-      id: cds-controller-1
-      address: "cds-controller-1:5679"
-    }
-    nodes.2 {
-      id: resource-reolution-1
-      address: "resource-reolution-1:5679"
-    }
-  }
-}
-# Configure the system management group.
-managementGroup {
-  type: raft
-  name: system
-  partitions: 1
-  members: [${CLUSTER_MEMBERS}]
-  storage {
-    directory: ${CLUSTER_STORAGE_PATH}/data-${CLUSTER_NODE_ID}
-    level: DISK
-  }
-}
-# Configure a Raft partition group.
-partitionGroups.data {
-  type: primary-backup
-  partitions: 7
-}
diff --git a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf b/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf
deleted file mode 100644
index fd161879c..000000000
--- a/ms/blueprintsprocessor/application/src/main/resources/atomix/atomix-multicast.conf
+++ /dev/null
@@ -1,40 +0,0 @@
-cluster {
-  # Configure the cluster node information.
-  node {
-    id: ${CLUSTER_NODE_ID}
-    address: ${CLUSTER_NODE_ADDRESS}
-  }
-  # Configure the node discovery protocol.
-  discovery {
-    type: multicast
-  }
-  multicast: {
-    enabled: true
-    port: 54321
-  }
-  # Configure the SWIM membership protocol.
-  protocol {
-    type: swim
-    broadcastUpdates: true
-    gossipInterval: 500ms
-    probeInterval: 2s
-    suspectProbes: 2
-  }
-}
-# Configure the system management group.
-managementGroup {
-  type: raft
-  name: system
-  partitions: 1
-  members: [${CLUSTER_MEMBERS}]
-  storage {
-    directory: ${CLUSTER_STORAGE_PATH}/data-${CLUSTER_NODE_ID}
-    level: DISK
-  }
-}
-
-# Configure a Raft partition group.
-partitionGroups.data {
-  type: primary-backup
-  partitions: 7
-}
diff --git a/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast-client.yaml b/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast-client.yaml
new file mode 100644
index 000000000..e60b5dfc4
--- /dev/null
+++ b/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast-client.yaml
@@ -0,0 +1,13 @@
+hazelcast-client:
+  cluster-name: ${CLUSTER_ID}
+  instance-name: ${CLUSTER_NODE_ID}
+
+  network:
+    cluster-members:
+      - 127.0.0.1:5701
+#    kubernetes:
+#      enabled: true
+#      namespace: MY-KUBERNETES-NAMESPACE
+#      service-name: MY-SERVICE-NAME
+#      service-label-name: MY-SERVICE-LABEL-NAME
+#      service-label-value: MY-SERVICE-LABEL-VALUE
diff --git a/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast.yaml b/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast.yaml
new file mode 100644
index 000000000..bacbe2a45
--- /dev/null
+++ b/ms/blueprintsprocessor/application/src/main/resources/hazelcast/hazelcast.yaml
@@ -0,0 +1,18 @@
+hazelcast:
+  cluster-name: ${CLUSTER_ID}
+  instance-name: ${CLUSTER_NODE_ID}
+  lite-member:
+    enabled: false
+  cp-subsystem:
+    cp-member-count: 3
+    group-size: 3
+#  network:
+#    join:
+#      multicast:
+#        enabled: false
+#      kubernetes:
+#        enabled: true
+#        namespace: MY-KUBERNETES-NAMESPACE
+#        service-name: MY-SERVICE-NAME
+#        service-label-name: MY-SERVICE-LABEL-NAME
+#        service-label-value: MY-SERVICE-LABEL-VALUE
\ No newline at end of file
diff --git a/ms/blueprintsprocessor/application/src/main/resources/logback.xml b/ms/blueprintsprocessor/application/src/main/resources/logback.xml
index 63ede28b4..aceea4327 100644
--- a/ms/blueprintsprocessor/application/src/main/resources/logback.xml
+++ b/ms/blueprintsprocessor/application/src/main/resources/logback.xml
@@ -28,7 +28,7 @@
     <logger name="org.springframework" level="info"/>
     <logger name="org.springframework.web" level="info"/>
     <logger name="org.hibernate" level="error"/>
-    <logger name="io.atomix" level="warn"/>
+    <logger name="com.hazelcast" level="error"/>
     <logger name="org.onap.ccsdk.cds" level="info"/>
 
     <root level="info">