[CCSDK-1985]GR Toolkit Refactor

Refactor of the GR Toolkit provider module to allow for more
flexibility for different architectures. Health checking and
failover logic has been extracted into an abstract HealthResolver
class and several implementations for Single, Three, and Six
node controller architectures. Resolvers can be specified in
the gr-toolkit.properties file, or a fallback resolver can be
used if one is not specified.

Signed-off-by: Haddox, Anthony (ah0647) <ah0647@att.com>
Issue-ID: CCSDK-1985
Change-Id: I262407e9d8830d91c39c4e75134a9f1cb1d259fe

22 files changed, 667 insertions, 8 deletions

diff --git a/grToolkit/provider/src/test/resources/akka.conf b/grToolkit/provider/src/test/resources/akka.conf
index cbb73d548..ce46748ec 100644
--- a/grToolkit/provider/src/test/resources/akka.conf
+++ b/grToolkit/provider/src/test/resources/akka.conf
@@ -1,3 +1,21 @@
+# ============LICENSE_START=======================================================
+# openECOMP : SDN-C
+# ================================================================================
+# Copyright (C) 2019 AT&T Intellectual Property. All rights
+# 			reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
 
 odl-cluster-data {
   akka {
@@ -20,7 +38,7 @@ odl-cluster-data {
 
     cluster {
       # Remove ".tcp" when using artery.
-      seed-nodes = ["akka.tcp://opendaylight-cluster-data@127.0.0.1:2550"]
+      seed-nodes = ["akka.tcp://opendaylight-cluster-data@127.0.1.1:2550"]
 
       roles = [
         "member-1"
diff --git a/grToolkit/provider/src/test/resources/gr-toolkit.properties b/grToolkit/provider/src/test/resources/gr-toolkit.properties
index d9bc66dcd..52b19bf51 100755
--- a/grToolkit/provider/src/test/resources/gr-toolkit.properties
+++ b/grToolkit/provider/src/test/resources/gr-toolkit.properties
@@ -17,18 +17,19 @@
 # limitations under the License.
 # ============LICENSE_END=========================================================
 
+resolver=org.onap.ccsdk.sli.plugins.grtoolkit.resolver.SingleNodeHealthResolver
 akka.conf.location=src/test/resources/akka.conf
 adm.useSsl=false
-adm.fqdn=wiki.onap.org
-adm.healthcheck=
-adm.port.http=80
-adm.port.ssl=443
+adm.fqdn=localhost
+adm.healthcheck=/adm/healthcheck
+adm.port.http=9999
+adm.port.ssl=19999
 controller.credentials=admin:admin
 controller.useSsl=false
-controller.port.http=8181
-controller.port.ssl=8443
+controller.port.http=9999
+controller.port.ssl=19999
 controller.port.akka=2550
 mbean.cluster=/jolokia/read/akka:type=Cluster
 mbean.shardManager=/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
 mbean.shard.config=/jolokia/read/org.opendaylight.controller:Category=Shards,name=%s,type=DistributedConfigDatastore
-site.identifier=TestODL
+#site.identifier=TestODL
diff --git a/grToolkit/provider/src/test/resources/single/akka.conf b/grToolkit/provider/src/test/resources/single/akka.conf
new file mode 100644
index 000000000..ce46748ec
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/single/akka.conf
@@ -0,0 +1,67 @@
+# ============LICENSE_START=======================================================
+# openECOMP : SDN-C
+# ================================================================================
+# Copyright (C) 2019 AT&T Intellectual Property. All rights
+# 			reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+odl-cluster-data {
+  akka {
+    remote {
+      artery {
+        enabled = off
+        canonical.hostname = "127.0.0.1"
+        canonical.port = 2550
+      }
+      netty.tcp {
+        hostname = "127.0.0.1"
+        port = 2550
+      }
+      # when under load we might trip a false positive on the failure detector
+      # transport-failure-detector {
+        # heartbeat-interval = 4 s
+        # acceptable-heartbeat-pause = 16s
+      # }
+    }
+
+    cluster {
+      # Remove ".tcp" when using artery.
+      seed-nodes = ["akka.tcp://opendaylight-cluster-data@127.0.1.1:2550"]
+
+      roles = [
+        "member-1"
+      ]
+
+    }
+
+    persistence {
+      # By default the snapshots/journal directories live in KARAF_HOME. You can choose to put it somewhere else by
+      # modifying the following two properties. The directory location specified may be a relative or absolute path.
+      # The relative path is always relative to KARAF_HOME.
+
+      # snapshot-store.local.dir = "target/snapshots"
+      # journal.leveldb.dir = "target/journal"
+
+      journal {
+        leveldb {
+          # Set native = off to use a Java-only implementation of leveldb.
+          # Note that the Java-only version is not currently considered by Akka to be production quality.
+
+          # native = off
+        }
+      }
+    }
+  }
+}
diff --git a/grToolkit/provider/src/test/resources/single/cluster.json b/grToolkit/provider/src/test/resources/single/cluster.json
new file mode 100644
index 000000000..93360845f
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/single/cluster.json
@@ -0,0 +1,17 @@
+{
+  "request": {
+    "mbean": "akka:type=Cluster",
+    "type": "read"
+  },
+  "value": {
+    "Leader": "akka.tcp://opendaylight-cluster-data@localhost:2550",
+    "Unreachable": "",
+    "Singleton": true,
+    "Available": true,
+    "MemberStatus": "Up",
+    "ClusterStatus": "{\n  \"members\": [\n    {\n      \"address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n      \"roles\": [\n        \"dc-default\",\n        \"member-1\"\n      ],\n      \"status\": \"Up\"\n    }\n  ],\n  \"self-address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n  \"unreachable\": []\n}\n",
+    "Members": "akka.tcp://opendaylight-cluster-data@localhost:2550"
+  },
+  "timestamp": 1575393881,
+  "status": 200
+}
diff --git a/grToolkit/provider/src/test/resources/single/default-config.json b/grToolkit/provider/src/test/resources/single/default-config.json
new file mode 100644
index 000000000..d75998e5d
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/single/default-config.json
@@ -0,0 +1,46 @@
+{
+  "request": {
+    "mbean": "org.opendaylight.controller:Category=Shards,name=member-1-shard-default-config,type=DistributedConfigDatastore",
+    "type": "read"
+  },
+  "value": {
+    "ReadWriteTransactionCount": 0,
+    "SnapshotIndex": 22,
+    "InMemoryJournalLogSize": 1,
+    "ReplicatedToAllIndex": -1,
+    "Leader": "member-1-shard-default-config",
+    "LastIndex": 23,
+    "RaftState": "Leader",
+    "LastCommittedTransactionTime": "2019-12-03 16:36:39.413",
+    "LastApplied": 23,
+    "PeerAddresses": "",
+    "LastLogIndex": 23,
+    "LastLeadershipChangeTime": "2019-12-03 16:36:33.460",
+    "WriteOnlyTransactionCount": 0,
+    "FollowerInitialSyncStatus": false,
+    "FollowerInfo": [],
+    "FailedReadTransactionsCount": 0,
+    "Voting": true,
+    "StatRetrievalTime": "454.8 μs",
+    "CurrentTerm": 4,
+    "LastTerm": 4,
+    "FailedTransactionsCount": 0,
+    "PendingTxCommitQueueSize": 0,
+    "VotedFor": "member-1-shard-default-config",
+    "SnapshotCaptureInitiated": false,
+    "CommittedTransactionsCount": 5,
+    "TxCohortCacheSize": 0,
+    "PeerVotingStates": "",
+    "LastLogTerm": 4,
+    "StatRetrievalError": null,
+    "CommitIndex": 23,
+    "SnapshotTerm": 4,
+    "AbortTransactionsCount": 0,
+    "ReadOnlyTransactionCount": 0,
+    "ShardName": "member-1-shard-default-config",
+    "LeadershipChangeCount": 1,
+    "InMemoryJournalDataSize": 37
+  },
+  "timestamp": 1575393787,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/single/default-operational.json b/grToolkit/provider/src/test/resources/single/default-operational.json
new file mode 100644
index 000000000..88a5d2c96
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/single/default-operational.json
@@ -0,0 +1,46 @@
+{
+  "request": {
+    "mbean": "org.opendaylight.controller:Category=Shards,name=member-1-shard-default-operational,type=DistributedOperationalDatastore",
+    "type": "read"
+  },
+  "value": {
+    "ReadWriteTransactionCount": 0,
+    "SnapshotIndex": 22,
+    "InMemoryJournalLogSize": 1,
+    "ReplicatedToAllIndex": -1,
+    "Leader": "member-1-shard-default-operational",
+    "LastIndex": 23,
+    "RaftState": "Leader",
+    "LastCommittedTransactionTime": "2019-12-03 16:36:39.413",
+    "LastApplied": 23,
+    "PeerAddresses": "",
+    "LastLogIndex": 23,
+    "LastLeadershipChangeTime": "2019-12-03 16:36:33.460",
+    "WriteOnlyTransactionCount": 0,
+    "FollowerInitialSyncStatus": false,
+    "FollowerInfo": [],
+    "FailedReadTransactionsCount": 0,
+    "Voting": true,
+    "StatRetrievalTime": "454.8 μs",
+    "CurrentTerm": 4,
+    "LastTerm": 4,
+    "FailedTransactionsCount": 0,
+    "PendingTxCommitQueueSize": 0,
+    "VotedFor": "member-1-shard-default-operational",
+    "SnapshotCaptureInitiated": false,
+    "CommittedTransactionsCount": 5,
+    "TxCohortCacheSize": 0,
+    "PeerVotingStates": "",
+    "LastLogTerm": 4,
+    "StatRetrievalError": null,
+    "CommitIndex": 23,
+    "SnapshotTerm": 4,
+    "AbortTransactionsCount": 0,
+    "ReadOnlyTransactionCount": 0,
+    "ShardName": "member-1-shard-default-operational",
+    "LeadershipChangeCount": 1,
+    "InMemoryJournalDataSize": 37
+  },
+  "timestamp": 1575393787,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/single/gr-toolkit.properties b/grToolkit/provider/src/test/resources/single/gr-toolkit.properties
new file mode 100755
index 000000000..cc7820e90
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/single/gr-toolkit.properties
@@ -0,0 +1,34 @@
+# ============LICENSE_START=======================================================
+# openECOMP : SDN-C
+# ================================================================================
+# Copyright (C) 2019 AT&T Intellectual Property. All rights
+# 			reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+akka.conf.location=src/test/resources/single/akka.conf
+adm.useSsl=false
+adm.fqdn=localhost
+adm.healthcheck=/adm/healthcheck
+adm.port.http=9999
+adm.port.ssl=19999
+controller.credentials=admin:admin
+controller.useSsl=false
+controller.port.http=9999
+controller.port.ssl=19999
+controller.port.akka=2550
+mbean.cluster=/jolokia/read/akka:type=Cluster
+mbean.shardManager=/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
+mbean.shard.config=/jolokia/read/org.opendaylight.controller:Category=Shards,name=%s,type=DistributedConfigDatastore
+site.identifier=TestODL
diff --git a/grToolkit/provider/src/test/resources/single/shard-manager.json b/grToolkit/provider/src/test/resources/single/shard-manager.json
new file mode 100644
index 000000000..301184ef8
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/single/shard-manager.json
@@ -0,0 +1,15 @@
+{
+  "request": {
+    "mbean": "org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore",
+    "type": "read"
+  },
+  "value": {
+    "LocalShards": [
+      "member-1-shard-default-config"
+    ],
+    "SyncStatus": true,
+    "MemberName": "member-1"
+  },
+  "timestamp": 1575393918,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/akka6.conf b/grToolkit/provider/src/test/resources/six/akka.conf
index 358218d9c..9bda35d68 100644
--- a/grToolkit/provider/src/test/resources/akka6.conf
+++ b/grToolkit/provider/src/test/resources/six/akka.conf
@@ -1,3 +1,21 @@
+# ============LICENSE_START=======================================================
+# openECOMP : SDN-C
+# ================================================================================
+# Copyright (C) 2019 AT&T Intellectual Property. All rights
+# 			reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
 
 odl-cluster-data {
   akka {
diff --git a/grToolkit/provider/src/test/resources/six/cluster.json b/grToolkit/provider/src/test/resources/six/cluster.json
new file mode 100644
index 000000000..d6a54c8c8
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/six/cluster.json
@@ -0,0 +1,17 @@
+{
+  "request": {
+    "mbean": "akka:type=Cluster",
+    "type": "read"
+  },
+  "value": {
+    "Leader": "akka.tcp://opendaylight-cluster-data@localhost:2550",
+    "Unreachable": "",
+    "Singleton": true,
+    "Available": true,
+    "MemberStatus": "Up",
+    "ClusterStatus": "{\n  \"members\": [\n    {\n      \"address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n      \"roles\": [\n        \"dc-default\",\n        \"member-1\"\n      ],\n      \"status\": \"Up\"\n    },\n    {\n      \"address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n      \"roles\": [\n        \"dc-default\",\n        \"member-2\"\n      ],\n      \"status\": \"Up\"\n    },\n    {\n      \"address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n      \"roles\": [\n        \"dc-default\",\n        \"member-3\"\n      ],\n      \"status\": \"Up\"\n    }\n  ],\n  \"self-address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n  \"unreachable\": []\n}\n",
+    "Members": "akka.tcp://opendaylight-cluster-data@localhost:2550"
+  },
+  "timestamp": 1575393881,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/six/component-health.json b/grToolkit/provider/src/test/resources/six/component-health.json
new file mode 100644
index 000000000..1ca01da42
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/six/component-health.json
@@ -0,0 +1,7 @@
+{
+  "output": {
+    "status": "200",
+    "served-by": "member-2",
+    "health": "HEALTHY"
+  }
+}
diff --git a/grToolkit/provider/src/test/resources/six/default-config.json b/grToolkit/provider/src/test/resources/six/default-config.json
new file mode 100644
index 000000000..d75998e5d
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/six/default-config.json
@@ -0,0 +1,46 @@
+{
+  "request": {
+    "mbean": "org.opendaylight.controller:Category=Shards,name=member-1-shard-default-config,type=DistributedConfigDatastore",
+    "type": "read"
+  },
+  "value": {
+    "ReadWriteTransactionCount": 0,
+    "SnapshotIndex": 22,
+    "InMemoryJournalLogSize": 1,
+    "ReplicatedToAllIndex": -1,
+    "Leader": "member-1-shard-default-config",
+    "LastIndex": 23,
+    "RaftState": "Leader",
+    "LastCommittedTransactionTime": "2019-12-03 16:36:39.413",
+    "LastApplied": 23,
+    "PeerAddresses": "",
+    "LastLogIndex": 23,
+    "LastLeadershipChangeTime": "2019-12-03 16:36:33.460",
+    "WriteOnlyTransactionCount": 0,
+    "FollowerInitialSyncStatus": false,
+    "FollowerInfo": [],
+    "FailedReadTransactionsCount": 0,
+    "Voting": true,
+    "StatRetrievalTime": "454.8 μs",
+    "CurrentTerm": 4,
+    "LastTerm": 4,
+    "FailedTransactionsCount": 0,
+    "PendingTxCommitQueueSize": 0,
+    "VotedFor": "member-1-shard-default-config",
+    "SnapshotCaptureInitiated": false,
+    "CommittedTransactionsCount": 5,
+    "TxCohortCacheSize": 0,
+    "PeerVotingStates": "",
+    "LastLogTerm": 4,
+    "StatRetrievalError": null,
+    "CommitIndex": 23,
+    "SnapshotTerm": 4,
+    "AbortTransactionsCount": 0,
+    "ReadOnlyTransactionCount": 0,
+    "ShardName": "member-1-shard-default-config",
+    "LeadershipChangeCount": 1,
+    "InMemoryJournalDataSize": 37
+  },
+  "timestamp": 1575393787,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/six/default-operational.json b/grToolkit/provider/src/test/resources/six/default-operational.json
new file mode 100644
index 000000000..88a5d2c96
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/six/default-operational.json
@@ -0,0 +1,46 @@
+{
+  "request": {
+    "mbean": "org.opendaylight.controller:Category=Shards,name=member-1-shard-default-operational,type=DistributedOperationalDatastore",
+    "type": "read"
+  },
+  "value": {
+    "ReadWriteTransactionCount": 0,
+    "SnapshotIndex": 22,
+    "InMemoryJournalLogSize": 1,
+    "ReplicatedToAllIndex": -1,
+    "Leader": "member-1-shard-default-operational",
+    "LastIndex": 23,
+    "RaftState": "Leader",
+    "LastCommittedTransactionTime": "2019-12-03 16:36:39.413",
+    "LastApplied": 23,
+    "PeerAddresses": "",
+    "LastLogIndex": 23,
+    "LastLeadershipChangeTime": "2019-12-03 16:36:33.460",
+    "WriteOnlyTransactionCount": 0,
+    "FollowerInitialSyncStatus": false,
+    "FollowerInfo": [],
+    "FailedReadTransactionsCount": 0,
+    "Voting": true,
+    "StatRetrievalTime": "454.8 μs",
+    "CurrentTerm": 4,
+    "LastTerm": 4,
+    "FailedTransactionsCount": 0,
+    "PendingTxCommitQueueSize": 0,
+    "VotedFor": "member-1-shard-default-operational",
+    "SnapshotCaptureInitiated": false,
+    "CommittedTransactionsCount": 5,
+    "TxCohortCacheSize": 0,
+    "PeerVotingStates": "",
+    "LastLogTerm": 4,
+    "StatRetrievalError": null,
+    "CommitIndex": 23,
+    "SnapshotTerm": 4,
+    "AbortTransactionsCount": 0,
+    "ReadOnlyTransactionCount": 0,
+    "ShardName": "member-1-shard-default-operational",
+    "LeadershipChangeCount": 1,
+    "InMemoryJournalDataSize": 37
+  },
+  "timestamp": 1575393787,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/six/gr-toolkit.properties b/grToolkit/provider/src/test/resources/six/gr-toolkit.properties
new file mode 100755
index 000000000..54c9af742
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/six/gr-toolkit.properties
@@ -0,0 +1,34 @@
+# ============LICENSE_START=======================================================
+# openECOMP : SDN-C
+# ================================================================================
+# Copyright (C) 2019 AT&T Intellectual Property. All rights
+# 			reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+akka.conf.location=src/test/resources/six/akka.conf
+adm.useSsl=false
+adm.fqdn=localhost
+adm.healthcheck=/adm/healthcheck
+adm.port.http=9999
+adm.port.ssl=19999
+controller.credentials=admin:admin
+controller.useSsl=false
+controller.port.http=9999
+controller.port.ssl=19999
+controller.port.akka=2550
+mbean.cluster=/jolokia/read/akka:type=Cluster
+mbean.shardManager=/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
+mbean.shard.config=/jolokia/read/org.opendaylight.controller:Category=Shards,name=%s,type=DistributedConfigDatastore
+site.identifier=TestODL
diff --git a/grToolkit/provider/src/test/resources/six/shard-manager.json b/grToolkit/provider/src/test/resources/six/shard-manager.json
new file mode 100644
index 000000000..301184ef8
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/six/shard-manager.json
@@ -0,0 +1,15 @@
+{
+  "request": {
+    "mbean": "org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore",
+    "type": "read"
+  },
+  "value": {
+    "LocalShards": [
+      "member-1-shard-default-config"
+    ],
+    "SyncStatus": true,
+    "MemberName": "member-1"
+  },
+  "timestamp": 1575393918,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/six/site-identifier.json b/grToolkit/provider/src/test/resources/six/site-identifier.json
new file mode 100644
index 000000000..7599ceb15
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/six/site-identifier.json
@@ -0,0 +1,7 @@
+{
+  "output": {
+    "status": "200",
+    "id": "test-site",
+    "served-by": "member-1"
+  }
+}
diff --git a/grToolkit/provider/src/test/resources/three/akka.conf b/grToolkit/provider/src/test/resources/three/akka.conf
new file mode 100644
index 000000000..a6c151dd1
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/three/akka.conf
@@ -0,0 +1,67 @@
+# ============LICENSE_START=======================================================
+# openECOMP : SDN-C
+# ================================================================================
+# Copyright (C) 2019 AT&T Intellectual Property. All rights
+# 			reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+odl-cluster-data {
+  akka {
+    remote {
+      artery {
+        enabled = off
+        canonical.hostname = "127.0.0.1"
+        canonical.port = 2550
+      }
+      netty.tcp {
+        hostname = "127.0.0.1"
+        port = 2550
+      }
+      # when under load we might trip a false positive on the failure detector
+      # transport-failure-detector {
+        # heartbeat-interval = 4 s
+        # acceptable-heartbeat-pause = 16s
+      # }
+    }
+
+    cluster {
+      # Remove ".tcp" when using artery.
+      seed-nodes = ["akka.tcp://opendaylight-cluster-data@127.0.0.1:2550", "akka.tcp://opendaylight-cluster-data@127.0.0.2:2550", "akka.tcp://opendaylight-cluster-data@127.0.0.3:2550"]
+
+      roles = [
+        "member-1"
+      ]
+
+    }
+
+    persistence {
+      # By default the snapshots/journal directories live in KARAF_HOME. You can choose to put it somewhere else by
+      # modifying the following two properties. The directory location specified may be a relative or absolute path.
+      # The relative path is always relative to KARAF_HOME.
+
+      # snapshot-store.local.dir = "target/snapshots"
+      # journal.leveldb.dir = "target/journal"
+
+      journal {
+        leveldb {
+          # Set native = off to use a Java-only implementation of leveldb.
+          # Note that the Java-only version is not currently considered by Akka to be production quality.
+
+          # native = off
+        }
+      }
+    }
+  }
+}
diff --git a/grToolkit/provider/src/test/resources/three/cluster.json b/grToolkit/provider/src/test/resources/three/cluster.json
new file mode 100644
index 000000000..d6a54c8c8
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/three/cluster.json
@@ -0,0 +1,17 @@
+{
+  "request": {
+    "mbean": "akka:type=Cluster",
+    "type": "read"
+  },
+  "value": {
+    "Leader": "akka.tcp://opendaylight-cluster-data@localhost:2550",
+    "Unreachable": "",
+    "Singleton": true,
+    "Available": true,
+    "MemberStatus": "Up",
+    "ClusterStatus": "{\n  \"members\": [\n    {\n      \"address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n      \"roles\": [\n        \"dc-default\",\n        \"member-1\"\n      ],\n      \"status\": \"Up\"\n    },\n    {\n      \"address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n      \"roles\": [\n        \"dc-default\",\n        \"member-2\"\n      ],\n      \"status\": \"Up\"\n    },\n    {\n      \"address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n      \"roles\": [\n        \"dc-default\",\n        \"member-3\"\n      ],\n      \"status\": \"Up\"\n    }\n  ],\n  \"self-address\": \"akka.tcp://opendaylight-cluster-data@localhost:2550\",\n  \"unreachable\": []\n}\n",
+    "Members": "akka.tcp://opendaylight-cluster-data@localhost:2550"
+  },
+  "timestamp": 1575393881,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/three/default-config.json b/grToolkit/provider/src/test/resources/three/default-config.json
new file mode 100644
index 000000000..d75998e5d
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/three/default-config.json
@@ -0,0 +1,46 @@
+{
+  "request": {
+    "mbean": "org.opendaylight.controller:Category=Shards,name=member-1-shard-default-config,type=DistributedConfigDatastore",
+    "type": "read"
+  },
+  "value": {
+    "ReadWriteTransactionCount": 0,
+    "SnapshotIndex": 22,
+    "InMemoryJournalLogSize": 1,
+    "ReplicatedToAllIndex": -1,
+    "Leader": "member-1-shard-default-config",
+    "LastIndex": 23,
+    "RaftState": "Leader",
+    "LastCommittedTransactionTime": "2019-12-03 16:36:39.413",
+    "LastApplied": 23,
+    "PeerAddresses": "",
+    "LastLogIndex": 23,
+    "LastLeadershipChangeTime": "2019-12-03 16:36:33.460",
+    "WriteOnlyTransactionCount": 0,
+    "FollowerInitialSyncStatus": false,
+    "FollowerInfo": [],
+    "FailedReadTransactionsCount": 0,
+    "Voting": true,
+    "StatRetrievalTime": "454.8 μs",
+    "CurrentTerm": 4,
+    "LastTerm": 4,
+    "FailedTransactionsCount": 0,
+    "PendingTxCommitQueueSize": 0,
+    "VotedFor": "member-1-shard-default-config",
+    "SnapshotCaptureInitiated": false,
+    "CommittedTransactionsCount": 5,
+    "TxCohortCacheSize": 0,
+    "PeerVotingStates": "",
+    "LastLogTerm": 4,
+    "StatRetrievalError": null,
+    "CommitIndex": 23,
+    "SnapshotTerm": 4,
+    "AbortTransactionsCount": 0,
+    "ReadOnlyTransactionCount": 0,
+    "ShardName": "member-1-shard-default-config",
+    "LeadershipChangeCount": 1,
+    "InMemoryJournalDataSize": 37
+  },
+  "timestamp": 1575393787,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/three/default-operational.json b/grToolkit/provider/src/test/resources/three/default-operational.json
new file mode 100644
index 000000000..88a5d2c96
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/three/default-operational.json
@@ -0,0 +1,46 @@
+{
+  "request": {
+    "mbean": "org.opendaylight.controller:Category=Shards,name=member-1-shard-default-operational,type=DistributedOperationalDatastore",
+    "type": "read"
+  },
+  "value": {
+    "ReadWriteTransactionCount": 0,
+    "SnapshotIndex": 22,
+    "InMemoryJournalLogSize": 1,
+    "ReplicatedToAllIndex": -1,
+    "Leader": "member-1-shard-default-operational",
+    "LastIndex": 23,
+    "RaftState": "Leader",
+    "LastCommittedTransactionTime": "2019-12-03 16:36:39.413",
+    "LastApplied": 23,
+    "PeerAddresses": "",
+    "LastLogIndex": 23,
+    "LastLeadershipChangeTime": "2019-12-03 16:36:33.460",
+    "WriteOnlyTransactionCount": 0,
+    "FollowerInitialSyncStatus": false,
+    "FollowerInfo": [],
+    "FailedReadTransactionsCount": 0,
+    "Voting": true,
+    "StatRetrievalTime": "454.8 μs",
+    "CurrentTerm": 4,
+    "LastTerm": 4,
+    "FailedTransactionsCount": 0,
+    "PendingTxCommitQueueSize": 0,
+    "VotedFor": "member-1-shard-default-operational",
+    "SnapshotCaptureInitiated": false,
+    "CommittedTransactionsCount": 5,
+    "TxCohortCacheSize": 0,
+    "PeerVotingStates": "",
+    "LastLogTerm": 4,
+    "StatRetrievalError": null,
+    "CommitIndex": 23,
+    "SnapshotTerm": 4,
+    "AbortTransactionsCount": 0,
+    "ReadOnlyTransactionCount": 0,
+    "ShardName": "member-1-shard-default-operational",
+    "LeadershipChangeCount": 1,
+    "InMemoryJournalDataSize": 37
+  },
+  "timestamp": 1575393787,
+  "status": 200
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/resources/three/gr-toolkit.properties b/grToolkit/provider/src/test/resources/three/gr-toolkit.properties
new file mode 100755
index 000000000..cc7820e90
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/three/gr-toolkit.properties
@@ -0,0 +1,34 @@
+# ============LICENSE_START=======================================================
+# openECOMP : SDN-C
+# ================================================================================
+# Copyright (C) 2019 AT&T Intellectual Property. All rights
+# 			reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+akka.conf.location=src/test/resources/single/akka.conf
+adm.useSsl=false
+adm.fqdn=localhost
+adm.healthcheck=/adm/healthcheck
+adm.port.http=9999
+adm.port.ssl=19999
+controller.credentials=admin:admin
+controller.useSsl=false
+controller.port.http=9999
+controller.port.ssl=19999
+controller.port.akka=2550
+mbean.cluster=/jolokia/read/akka:type=Cluster
+mbean.shardManager=/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
+mbean.shard.config=/jolokia/read/org.opendaylight.controller:Category=Shards,name=%s,type=DistributedConfigDatastore
+site.identifier=TestODL
diff --git a/grToolkit/provider/src/test/resources/three/shard-manager.json b/grToolkit/provider/src/test/resources/three/shard-manager.json
new file mode 100644
index 000000000..301184ef8
--- /dev/null
+++ b/grToolkit/provider/src/test/resources/three/shard-manager.json
@@ -0,0 +1,15 @@
+{
+  "request": {
+    "mbean": "org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore",
+    "type": "read"
+  },
+  "value": {
+    "LocalShards": [
+      "member-1-shard-default-config"
+    ],
+    "SyncStatus": true,
+    "MemberName": "member-1"
+  },
+  "timestamp": 1575393918,
+  "status": 200
+}
\ No newline at end of file