[CCSDK-1985]GR Toolkit Refactor

Refactor of the GR Toolkit provider module to allow for more
flexibility for different architectures. Health checking and
failover logic has been extracted into an abstract HealthResolver
class and several implementations for Single, Three, and Six
node controller architectures. Resolvers can be specified in
the gr-toolkit.properties file, or a fallback resolver can be
used if one is not specified.

Signed-off-by: Haddox, Anthony (ah0647) <ah0647@att.com>
Issue-ID: CCSDK-1985
Change-Id: I262407e9d8830d91c39c4e75134a9f1cb1d259fe

13 files changed, 1455 insertions, 85 deletions

diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProviderTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProviderTest.java
index dabdf2065..3be159598 100644
--- a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProviderTest.java
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProviderTest.java
@@ -20,14 +20,19 @@
  */
 
 package org.onap.ccsdk.sli.plugins.grtoolkit;
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+
 import com.google.common.util.concurrent.ListenableFuture;
+
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.contrib.java.lang.system.EnvironmentVariables;
+
 import org.onap.ccsdk.sli.core.dblib.DBLibConnection;
 import org.onap.ccsdk.sli.core.dblib.DbLibService;
 import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor;
+
 import org.opendaylight.controller.cluster.access.concepts.MemberName;
 import org.opendaylight.controller.cluster.datastore.DistributedDataStoreInterface;
 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
@@ -38,22 +43,34 @@ import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180
 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthOutput;
 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthOutput;
 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverOutput;
-import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverOutputBuilder;
+import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficInputBuilder;
+import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficOutput;
+import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficInputBuilder;
+import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficOutput;
 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthOutput;
 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierOutput;
 import org.opendaylight.yangtools.yang.common.RpcResult;
 
-import java.lang.reflect.Field;
+import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Properties;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
+
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
@@ -67,13 +84,16 @@ public class GrToolkitProviderTest {
     DistributedDataStoreInterface configDatastore;
     DbLibService dbLibService;
     DBLibConnection connection;
+    Properties properties;
 
     @Rule
     public final EnvironmentVariables environmentVariables = new EnvironmentVariables();
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(9999);
 
     @Before
     public void setup() {
-        environmentVariables.set("SDNC_CONFIG_DIR","src/test/resources/");
+        environmentVariables.set("SDNC_CONFIG_DIR","src/test/resources");
         dataBroker = mock(DataBroker.class);
         notificationProviderService = mock(NotificationPublishService.class);
         rpcProviderRegistry = mock(RpcProviderRegistry.class);
@@ -99,14 +119,14 @@ public class GrToolkitProviderTest {
         provider = new GrToolkitProvider(dataBroker, notificationProviderService,
                 rpcProviderRegistry, configDatastore, dbLibService);
         providerSpy = spy(provider);
+        stubController();
     }
 
     @Test
     public void closeTest() {
         try {
             provider.close();
-        }
-        catch(Exception e) {
+        } catch(Exception e) {
             // Exception expected
         }
     }
@@ -117,63 +137,54 @@ public class GrToolkitProviderTest {
         // onDataTreeChanged is an empty stub
     }
 
-    @Test
-    public void clusterHealthTest() {
-        ListenableFuture<RpcResult<ClusterHealthOutput>> result = provider.clusterHealth(null);
-        try {
-            assertEquals("200", result.get().getResult().getStatus());
-        } catch(InterruptedException | ExecutionException e) {
+    private void stubController() {
+        String clusterBody = null;
+        String shardManagerBody = null;
+        String shardDefaultBody = null;
+        String shardOperationalBody = null;
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/three/cluster.json"))) {
+            clusterBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/three/shard-manager.json"))) {
+            shardManagerBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/three/default-config.json"))) {
+            shardDefaultBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/three/default-operational.json"))) {
+            shardOperationalBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
             fail();
         }
-    }
 
-    @Test
-    public void siteHealthTest() {
-        ListenableFuture<RpcResult<SiteHealthOutput>> result = provider.siteHealth(null);
-        try {
-            assertEquals("200", result.get().getResult().getStatus());
-        } catch(InterruptedException | ExecutionException e) {
+        if(clusterBody == null || shardManagerBody == null || shardDefaultBody == null || shardOperationalBody == null) {
             fail();
         }
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        stubFor(get(urlEqualTo("/jolokia/read/akka:type=Cluster")).willReturn(aResponse().withStatus(200).withBody(clusterBody)));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore")).inScenario("testing").willReturn(aResponse().withStatus(200).withBody(shardManagerBody)).willSetStateTo("next"));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-config,type=DistributedConfigDatastore")).inScenario("testing").willReturn(aResponse().withStatus(200).withBody(shardDefaultBody)).willSetStateTo("next"));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-operational,type=DistributedOperationalDatastore")).willReturn(aResponse().withStatus(200).withBody(shardOperationalBody)));
     }
 
     @Test
-    public void siteHealth6NodeTest() {
-        Map<String, ClusterActor> memberMap = new HashMap<>();
-        ClusterActor actor;
-        for(int ndx = 0; ndx < 6; ndx++) {
-            actor = new ClusterActor();
-            actor.setNode("member-" + (ndx + 1));
-            actor.setUp(true);
-            actor.setUnreachable(false);
-
-            memberMap.put(actor.getNode(),  actor);
-        }
-
+    public void clusterHealthTest() {
+        ListenableFuture<RpcResult<ClusterHealthOutput>> result = provider.clusterHealth(null);
         try {
-            Field field = provider.getClass().getDeclaredField("siteConfiguration");
-            field.setAccessible(true);
-            field.set(provider, GrToolkitProvider.SiteConfiguration.GEO);
-
-            field = provider.getClass().getDeclaredField("memberMap");
-            field.setAccessible(true);
-            field.set(provider, memberMap);
-
-
-            actor = new ClusterActor();
-            actor.setNode("member-1");
-            field = provider.getClass().getDeclaredField("self");
-            field.setAccessible(true);
-            field.set(provider, actor);
-
-            field = provider.getClass().getDeclaredField("member");
-            field.setAccessible(true);
-            field.set(provider, actor.getNode());
-        }
-        catch(IllegalAccessException | NoSuchFieldException e) {
+            assertEquals("0", result.get().getResult().getStatus());
+        } catch(InterruptedException | ExecutionException e) {
             fail();
         }
+    }
 
+    @Test
+    public void siteHealthTest() {
         ListenableFuture<RpcResult<SiteHealthOutput>> result = provider.siteHealth(null);
         try {
             assertEquals("200", result.get().getResult().getStatus());
@@ -201,7 +212,7 @@ public class GrToolkitProviderTest {
         }
         ListenableFuture<RpcResult<DatabaseHealthOutput>> result = provider.databaseHealth(null);
         try {
-            assertEquals("200", result.get().getResult().getStatus());
+            assertEquals("500", result.get().getResult().getStatus());
         } catch(InterruptedException | ExecutionException e) {
             fail();
         }
@@ -216,7 +227,7 @@ public class GrToolkitProviderTest {
         }
         ListenableFuture<RpcResult<DatabaseHealthOutput>> result = provider.databaseHealth(null);
         try {
-            assertEquals("200", result.get().getResult().getStatus());
+            assertEquals("500", result.get().getResult().getStatus());
         } catch(InterruptedException | ExecutionException e) {
             fail();
         }
@@ -253,13 +264,36 @@ public class GrToolkitProviderTest {
     }
 
     @Test
+    public void haltTrafficTest() {
+        HaltAkkaTrafficInputBuilder builder = new HaltAkkaTrafficInputBuilder();
+        builder.setNodeInfo(new ArrayList<>());
+        ListenableFuture<RpcResult<HaltAkkaTrafficOutput>> result = provider.haltAkkaTraffic(builder.build());
+        try {
+            assertEquals("200", result.get().getResult().getStatus());
+        } catch(InterruptedException | ExecutionException e) {
+            fail();
+        }
+    }
+
+    @Test
+    public void resumeTrafficTest() {
+        ResumeAkkaTrafficInputBuilder builder = new ResumeAkkaTrafficInputBuilder();
+        builder.setNodeInfo(new ArrayList<>());
+        ListenableFuture<RpcResult<ResumeAkkaTrafficOutput>> result = provider.resumeAkkaTraffic(builder.build());
+        try {
+            assertEquals("200", result.get().getResult().getStatus());
+        } catch(InterruptedException | ExecutionException e) {
+            fail();
+        }
+    }
+
+    @Test
     public void executeCommandTest() {
         try {
             Method method = provider.getClass().getDeclaredMethod("executeCommand", String.class);
             method.setAccessible(true);
             method.invoke(provider, "ls");
-        }
-        catch(NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
+        } catch(NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
             fail();
         }
     }
@@ -277,8 +311,7 @@ public class GrToolkitProviderTest {
             Method method = provider.getClass().getDeclaredMethod("isolateSiteFromCluster", ArrayList.class, ArrayList.class, String.class);
             method.setAccessible(true);
             method.invoke(provider, activeList, standbyList, "80");
-        }
-        catch(NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
+        } catch(NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
             fail();
         }
     }
@@ -296,30 +329,7 @@ public class GrToolkitProviderTest {
             Method method = provider.getClass().getDeclaredMethod("downUnreachableNodes", ArrayList.class, ArrayList.class, String.class);
             method.setAccessible(true);
             method.invoke(provider, activeList, standbyList, "80");
-        }
-        catch(NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
-            fail();
-        }
-    }
-
-    @Test
-    public void changeClusterVotingTest() {
-        try {
-            ClusterActor actor = new ClusterActor();
-            actor.setMember("some-member");
-            actor.setNode("some-Node");
-            ArrayList<ClusterActor> activeList = new ArrayList<>();
-            activeList.add(actor);
-            ArrayList<ClusterActor> standbyList = new ArrayList<>();
-            standbyList.add(actor);
-            Field field = provider.getClass().getDeclaredField("self");
-            field.setAccessible(true);
-            field.set(provider, actor);
-            Method method = provider.getClass().getDeclaredMethod("changeClusterVoting", FailoverOutputBuilder.class, ArrayList.class, ArrayList.class, String.class);
-            method.setAccessible(true);
-            method.invoke(provider, new FailoverOutputBuilder(), activeList, standbyList, "80");
-        }
-        catch(NoSuchMethodException | IllegalAccessException | InvocationTargetException | NoSuchFieldException e) {
+        } catch(NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
             fail();
         }
     }
@@ -335,10 +345,8 @@ public class GrToolkitProviderTest {
             Method method = provider.getClass().getDeclaredMethod("backupMdSal", ArrayList.class, String.class);
             method.setAccessible(true);
             method.invoke(provider, activeList, "80");
-        }
-        catch(NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
+        } catch(NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
             fail();
         }
     }
-
 }
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionManagerTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionManagerTest.java
new file mode 100644
index 000000000..f749688d9
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionManagerTest.java
@@ -0,0 +1,65 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.connection;
+
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.*;
+
+public class ConnectionManagerTest {
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(9999);
+
+    @Test
+    public void getConnectionResponseWithInput() throws Exception {
+        stubFor(post(urlEqualTo("/post"))
+                        .willReturn(aResponse().withStatus(200)));
+        ConnectionResponse response = ConnectionManager.getConnectionResponse("http://localhost:9999/post", ConnectionManager.HttpMethod.POST, "", "creds:creds");
+        assertNotNull(response);
+        assertEquals(200, response.statusCode);
+    }
+
+    @Test
+    public void getConnectionResponseWithCredentials() throws Exception {
+        stubFor(post(urlEqualTo("/post"))
+                        .willReturn(aResponse().withStatus(200)));
+        ConnectionResponse response = ConnectionManager.getConnectionResponse("http://localhost:9999/post", ConnectionManager.HttpMethod.POST, "", "creds:creds");
+        assertNotNull(response);
+        assertEquals(200, response.statusCode);
+    }
+
+    @Test
+    public void getConnectionResponse() throws Exception {
+        stubFor(get(urlEqualTo("/get"))
+                        .willReturn(aResponse().withStatus(200)
+                        .withBody("Multi\nLine\nResponse")));
+        ConnectionResponse response = ConnectionManager.getConnectionResponse("http://localhost:9999/get", ConnectionManager.HttpMethod.GET, null, null);
+        assertNotNull(response);
+        assertEquals(200, response.statusCode);
+        assertEquals("MultiLineResponse", response.content);
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionResponseTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionResponseTest.java
new file mode 100644
index 000000000..a9f0edc30
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionResponseTest.java
@@ -0,0 +1,42 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.connection;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class ConnectionResponseTest {
+    @Test
+    public void constructorTest() {
+        ConnectionResponse response = new ConnectionResponse();
+        assertNotNull(response);
+        assertEquals(0, response.statusCode);
+        assertNull(response.content);
+        assertTrue(response.toString().length() > 0);
+    }
+    @Test
+    public void withStatusCode() {
+        ConnectionResponse response = new ConnectionResponse().withStatusCode(123);
+        assertEquals(123, response.statusCode);
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/AdminHealthTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/AdminHealthTest.java
new file mode 100644
index 000000000..fa56a4d49
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/AdminHealthTest.java
@@ -0,0 +1,59 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.data;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class AdminHealthTest {
+    @Test
+    public void constructorTest() {
+        AdminHealth health = new AdminHealth(Health.HEALTHY);
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    @Test
+    public void constructor2Test() {
+        AdminHealth health = new AdminHealth(Health.FAULTY, 500);
+        assertEquals(Health.FAULTY, health.getHealth());
+        assertEquals(500, health.getStatusCode());
+    }
+
+    @Test
+    public void setHealth() {
+        AdminHealth health = new AdminHealth(Health.HEALTHY, 201);
+        assertEquals(Health.HEALTHY, health.getHealth());
+        assertEquals(201, health.getStatusCode());
+        health.setHealth(Health.FAULTY);
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void setStatusCode() {
+        AdminHealth health = new AdminHealth(Health.HEALTHY, 200);
+        assertEquals(Health.HEALTHY, health.getHealth());
+        assertEquals(200, health.getStatusCode());
+        health.setStatusCode(409);
+        assertEquals(409, health.getStatusCode());
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/ClusterHealthTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/ClusterHealthTest.java
new file mode 100644
index 000000000..2e2ab3fd7
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/ClusterHealthTest.java
@@ -0,0 +1,47 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.data;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class ClusterHealthTest {
+    @Test
+    public void constructorTest() {
+        ClusterHealth health = new ClusterHealth();
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void withHealth() {
+        ClusterHealth health = new ClusterHealth().withHealth(Health.HEALTHY);
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    @Test
+    public void setHealth() {
+        ClusterHealth health = new ClusterHealth();
+        health.setHealth(Health.HEALTHY);
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/DatabaseHealthTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/DatabaseHealthTest.java
new file mode 100644
index 000000000..05621a503
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/DatabaseHealthTest.java
@@ -0,0 +1,42 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.data;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class DatabaseHealthTest {
+    @Test
+    public void constructorTest() {
+        DatabaseHealth health = new DatabaseHealth(Health.FAULTY);
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void setHealth() {
+        DatabaseHealth health = new DatabaseHealth(Health.FAULTY);
+        assertEquals(Health.FAULTY, health.getHealth());
+        health.setHealth(Health.HEALTHY);
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/FailoverStatusTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/FailoverStatusTest.java
new file mode 100644
index 000000000..b5b3d00d2
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/FailoverStatusTest.java
@@ -0,0 +1,60 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.data;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class FailoverStatusTest {
+    @Test
+    public void constructorTest() {
+        FailoverStatus status = new FailoverStatus();
+        assertEquals(200, status.getStatusCode());
+        assertEquals("Failover complete.", status.getMessage());
+    }
+    @Test
+    public void withStatusCode() {
+        FailoverStatus status = new FailoverStatus().withStatusCode(500);
+        assertEquals(500, status.getStatusCode());
+    }
+
+    @Test
+    public void withMessage() {
+        FailoverStatus status = new FailoverStatus().withMessage("Test");
+        assertEquals("Test", status.getMessage());
+    }
+
+    @Test
+    public void setStatusCode() {
+        FailoverStatus status = new FailoverStatus();
+        status.setStatusCode(500);
+        assertEquals(500, status.getStatusCode());
+    }
+
+    @Test
+    public void setMessage() {
+        FailoverStatus status = new FailoverStatus();
+        status.setMessage("Test");
+        assertEquals("Test", status.getMessage());
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/HealthTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/HealthTest.java
new file mode 100644
index 000000000..1115d2084
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/HealthTest.java
@@ -0,0 +1,34 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.data;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class HealthTest {
+    @Test
+    public void getHealth() {
+        assertEquals("HEALTHY", Health.HEALTHY.getHealth());
+        assertEquals("FAULTY", Health.FAULTY.getHealth());
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/MemberBuilderTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/MemberBuilderTest.java
index 4b657cf0a..7ed0135a6 100644
--- a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/MemberBuilderTest.java
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/MemberBuilderTest.java
@@ -1,3 +1,24 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2018 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
 package org.onap.ccsdk.sli.plugins.grtoolkit.data;
 
 import org.junit.Before;
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/SiteHealthTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/SiteHealthTest.java
new file mode 100644
index 000000000..7b74991e7
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/SiteHealthTest.java
@@ -0,0 +1,108 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.data;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class SiteHealthTest {
+    @Test
+    public void constructorTest() {
+        SiteHealth health = new SiteHealth();
+        assertNotNull(health.getAdminHealth());
+        assertNotNull(health.getDatabaseHealth());
+        assertNotNull(health.getClusterHealth());
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+    @Test
+    public void withAdminHealth() {
+        SiteHealth health = new SiteHealth().withAdminHealth(new AdminHealth(Health.HEALTHY));
+        assertEquals(Health.HEALTHY, health.getAdminHealth().get(0).getHealth());
+    }
+
+    @Test
+    public void withDatabaseHealth() {
+        SiteHealth health = new SiteHealth().withDatabaseHealth(new DatabaseHealth(Health.HEALTHY));
+        assertEquals(Health.HEALTHY, health.getDatabaseHealth().get(0).getHealth());
+    }
+
+    @Test
+    public void withClusterHealth() {
+        SiteHealth health = new SiteHealth().withClusterHealth(new ClusterHealth());
+        assertEquals(Health.FAULTY, health.getClusterHealth().get(0).getHealth());
+    }
+
+    @Test
+    public void withId() {
+        SiteHealth health = new SiteHealth().withId("My_ID");
+        assertEquals("My_ID", health.getId());
+    }
+
+    @Test
+    public void withRole() {
+        SiteHealth health = new SiteHealth().withRole("My_role");
+        assertEquals("My_role", health.getRole());
+    }
+
+    @Test
+    public void setHealth() {
+        SiteHealth health = new SiteHealth();
+        health.setHealth(Health.HEALTHY);
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    @Test
+    public void setAdminHealth() {
+        SiteHealth health = new SiteHealth().withAdminHealth(new AdminHealth(Health.HEALTHY));
+        health.setAdminHealth(null);
+        assertNull(health.getAdminHealth());
+    }
+
+    @Test
+    public void setDatabaseHealth() {
+        SiteHealth health = new SiteHealth().withDatabaseHealth(new DatabaseHealth(Health.HEALTHY));
+        health.setDatabaseHealth(null);
+        assertNull(health.getDatabaseHealth());
+    }
+
+    @Test
+    public void setClusterHealth() {
+        SiteHealth health = new SiteHealth().withClusterHealth(new ClusterHealth());
+        health.setClusterHealth(null);
+        assertNull(health.getClusterHealth());
+    }
+
+    @Test
+    public void setId() {
+        SiteHealth health = new SiteHealth().withId("My_ID");
+        health.setId("My_new_ID");
+        assertEquals("My_new_ID", health.getId());
+    }
+
+    @Test
+    public void setRole() {
+        SiteHealth health = new SiteHealth().withRole("My_role");
+        health.setRole("My_new_role");
+        assertEquals("My_new_role", health.getRole());
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SingleNodeHealthResolverTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SingleNodeHealthResolverTest.java
new file mode 100644
index 000000000..2827b4055
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SingleNodeHealthResolverTest.java
@@ -0,0 +1,235 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.resolver;
+
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import org.onap.ccsdk.sli.core.dblib.DBLibConnection;
+import org.onap.ccsdk.sli.core.dblib.DbLibService;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.AdminHealth;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterHealth;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.DatabaseHealth;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.FailoverStatus;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.Health;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.SiteHealth;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+
+import static org.junit.Assert.*;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SingleNodeHealthResolverTest {
+    private Map<String, ClusterActor> memberMap;
+    private DbLibService dbLibService;
+    private DBLibConnection connection;
+    private SingleNodeHealthResolver resolver;
+
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(9999);
+
+    @Before
+    public void setUp() {
+        memberMap = generateMemberMap(1);
+        Properties properties = new Properties();
+        try(FileInputStream fileInputStream = new FileInputStream("src/test/resources/single/gr-toolkit.properties")) {
+            properties.load(fileInputStream);
+        } catch(IOException e) {
+            fail();
+        }
+
+        dbLibService = mock(DbLibService.class);
+        connection = mock(DBLibConnection.class);
+        resolver = new SingleNodeHealthResolver(memberMap, properties, dbLibService);
+    }
+
+    private Map<String, ClusterActor> generateMemberMap(int memberCount) {
+        Map<String, ClusterActor> map = new HashMap<>();
+        ClusterActor actor;
+        for(int ndx = 0; ndx < memberCount; ndx++) {
+            actor = new ClusterActor();
+            actor.setNode("localhost");
+            actor.setAkkaPort("2550");
+            actor.setMember("member-" + (ndx + 1));
+            actor.setUp(true);
+            actor.setUnreachable(false);
+
+            map.put(actor.getNode(),  actor);
+        }
+        return map;
+    }
+
+    @Test
+    public void getAdminHealthFaulty() {
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(500)));
+        AdminHealth health = resolver.getAdminHealth();
+        assertNotNull(health);
+        assertEquals(500, health.getStatusCode());
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void getAdminHealthHealthy() {
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        AdminHealth health = resolver.getAdminHealth();
+        assertNotNull(health);
+        assertEquals(200, health.getStatusCode());
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    @Test
+    public void getDatabaseHealth() {
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        DatabaseHealth health = resolver.getDatabaseHealth();
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    @Test
+    public void getDatabaseHealthFaulty() {
+        try {
+            when(connection.isReadOnly()).thenReturn(true);
+            when(connection.isClosed()).thenReturn(true);
+            when(dbLibService.isActive()).thenReturn(false);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        DatabaseHealth health = resolver.getDatabaseHealth();
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void getDatabaseHealthException() {
+        try {
+            when(connection.isReadOnly()).thenThrow(new SQLException());
+            when(connection.isClosed()).thenReturn(true);
+            when(dbLibService.isActive()).thenReturn(false);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        DatabaseHealth health = resolver.getDatabaseHealth();
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void siteIdentifier() {
+        assertEquals("TestODL", resolver.getSiteIdentifier());
+        resolver.setSiteIdentifier("NewTestODL");
+        assertEquals("NewTestODL", resolver.getSiteIdentifier());
+    }
+
+    @Test
+    public void getClusterHealth() {
+        stubController();
+        ClusterHealth health = resolver.getClusterHealth();
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    private void stubController() {
+        String clusterBody = null;
+        String shardManagerBody = null;
+        String shardDefaultBody = null;
+        String shardOperationalBody = null;
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/single/cluster.json"))) {
+            clusterBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/single/shard-manager.json"))) {
+            shardManagerBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/single/default-config.json"))) {
+            shardDefaultBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/single/default-operational.json"))) {
+            shardOperationalBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+
+        if(clusterBody == null || shardManagerBody == null || shardDefaultBody == null || shardOperationalBody == null) {
+            fail();
+        }
+        stubFor(get(urlEqualTo("/jolokia/read/akka:type=Cluster")).willReturn(aResponse().withStatus(200).withBody(clusterBody)));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore")).willReturn(aResponse().withStatus(200).withBody(shardManagerBody)));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-config,type=DistributedConfigDatastore")).willReturn(aResponse().withStatus(200).withBody(shardDefaultBody)));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-operational,type=DistributedOperationalDatastore")).willReturn(aResponse().withStatus(200).withBody(shardOperationalBody)));
+    }
+
+    @Test
+    public void getSiteHealth() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(1, health.size());
+        assertEquals(Health.HEALTHY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void tryFailover() {
+        FailoverStatus status = resolver.tryFailover(null);
+        assertEquals(400, status.getStatusCode());
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SixNodeHealthResolverTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SixNodeHealthResolverTest.java
new file mode 100644
index 000000000..cbab450e1
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SixNodeHealthResolverTest.java
@@ -0,0 +1,335 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.resolver;
+
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import org.onap.ccsdk.sli.core.dblib.DBLibConnection;
+import org.onap.ccsdk.sli.core.dblib.DbLibService;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.AdminHealth;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterHealth;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.DatabaseHealth;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.FailoverStatus;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.Health;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.SiteHealth;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+
+import static org.junit.Assert.*;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SixNodeHealthResolverTest {
+    private Map<String, ClusterActor> memberMap;
+    private DbLibService dbLibService;
+    private DBLibConnection connection;
+    private SixNodeHealthResolver resolver;
+
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(9999);
+
+    @Before
+    public void setUp() {
+        memberMap = generateMemberMap(6);
+        Properties properties = new Properties();
+        try(FileInputStream fileInputStream = new FileInputStream("src/test/resources/six/gr-toolkit.properties")) {
+            properties.load(fileInputStream);
+        } catch(IOException e) {
+            fail();
+        }
+
+        dbLibService = mock(DbLibService.class);
+        connection = mock(DBLibConnection.class);
+        resolver = new SixNodeHealthResolver(memberMap, properties, dbLibService);
+    }
+
+    private Map<String, ClusterActor> generateMemberMap(int memberCount) {
+        Map<String, ClusterActor> map = new HashMap<>();
+        ClusterActor actor;
+        for(int ndx = 0; ndx < memberCount; ndx++) {
+            actor = new ClusterActor();
+            actor.setNode("127.0.1." + (ndx + 1));
+            actor.setAkkaPort("2550");
+            actor.setMember("member-" + (ndx + 1));
+            actor.setUp(true);
+            actor.setUnreachable(false);
+
+            map.put(actor.getNode(),  actor);
+        }
+        return map;
+    }
+
+    @Test
+    public void getAdminHealthFaulty() {
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(500)));
+        AdminHealth health = resolver.getAdminHealth();
+        assertNotNull(health);
+        assertEquals(500, health.getStatusCode());
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void getAdminHealthHealthy() {
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        AdminHealth health = resolver.getAdminHealth();
+        assertNotNull(health);
+        assertEquals(200, health.getStatusCode());
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    @Test
+    public void getDatabaseHealth() {
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        DatabaseHealth health = resolver.getDatabaseHealth();
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    @Test
+    public void getDatabaseHealthFaulty() {
+        try {
+            when(connection.isReadOnly()).thenReturn(true);
+            when(connection.isClosed()).thenReturn(true);
+            when(dbLibService.isActive()).thenReturn(false);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        DatabaseHealth health = resolver.getDatabaseHealth();
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void getDatabaseHealthException() {
+        try {
+            when(connection.isReadOnly()).thenThrow(new SQLException());
+            when(connection.isClosed()).thenReturn(true);
+            when(dbLibService.isActive()).thenReturn(false);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        DatabaseHealth health = resolver.getDatabaseHealth();
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void siteIdentifier() {
+        assertEquals("TestODL", resolver.getSiteIdentifier());
+        resolver.setSiteIdentifier("NewTestODL");
+        assertEquals("NewTestODL", resolver.getSiteIdentifier());
+    }
+
+    @Test
+    public void getClusterHealth() {
+        stubController();
+        ClusterHealth health = resolver.getClusterHealth();
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    private void stubController() {
+        String clusterBody = null;
+        String shardManagerBody = null;
+        String shardDefaultBody = null;
+        String shardOperationalBody = null;
+        String componentBody = null;
+        String identifierBody = null;
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/six/cluster.json"))) {
+            clusterBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/six/shard-manager.json"))) {
+            shardManagerBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/six/default-config.json"))) {
+            shardDefaultBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/six/default-operational.json"))) {
+            shardOperationalBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/six/component-health.json"))) {
+            componentBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/six/site-identifier.json"))) {
+            identifierBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+
+        if(clusterBody == null || shardManagerBody == null || shardDefaultBody == null || shardOperationalBody == null
+            || componentBody == null || identifierBody == null) {
+            fail();
+        }
+        stubFor(get(urlEqualTo("/jolokia/read/akka:type=Cluster")).willReturn(aResponse().withStatus(200).withBody(clusterBody)));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore")).inScenario("testing").willReturn(aResponse().withStatus(200).withBody(shardManagerBody)).willSetStateTo("next"));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-config,type=DistributedConfigDatastore")).inScenario("testing").willReturn(aResponse().withStatus(200).withBody(shardDefaultBody)).willSetStateTo("next"));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-operational,type=DistributedOperationalDatastore")).willReturn(aResponse().withStatus(200).withBody(shardOperationalBody)));
+        stubFor(post(urlEqualTo("/restconf/operations/gr-toolkit:site-identifier")).willReturn(aResponse().withStatus(200).withBody(identifierBody)));
+        stubFor(post(urlEqualTo("/restconf/operations/gr-toolkit:admin-health")).inScenario("testing").willReturn(aResponse().withStatus(200).withBody(componentBody)).willSetStateTo("next"));
+        stubFor(post(urlEqualTo("/restconf/operations/gr-toolkit:database-health")).willReturn(aResponse().withStatus(200).withBody(componentBody)));
+    }
+
+    @Test
+    public void getSiteHealth() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(2, health.size());
+        assertEquals(Health.HEALTHY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void getSiteHealthFaulty() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore")).inScenario("testing").whenScenarioStateIs("next").willReturn(aResponse().withBodyFile("nonexistent")));
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(2, health.size());
+        assertEquals(Health.FAULTY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void getSiteHealthFaultyShard() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-config,type=DistributedConfigDatastore")).inScenario("testing").willReturn(aResponse().withBodyFile("nonexistent")).willSetStateTo("next"));
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(2, health.size());
+        assertEquals(Health.FAULTY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void getSiteHealthFaultyCluster() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        stubFor(get(urlEqualTo("/jolokia/read/akka:type=Cluster")).willReturn(aResponse().withStatus(200).withBodyFile("nonexistent")));
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(2, health.size());
+        assertEquals(Health.FAULTY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void getSiteHealthFaultyAdmin() {
+        stubController();
+        stubFor(post(urlEqualTo("/restconf/operations/gr-toolkit:admin-health")).inScenario("testing").willReturn(aResponse().withBodyFile("nonexistent")).willSetStateTo("next"));
+        stubFor(get(urlEqualTo("/restconf/operations/gr-toolkit:admin-health")).inScenario("testing").whenScenarioStateIs("next").willReturn(aResponse().withBodyFile("nonexistent")));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(2, health.size());
+        assertEquals(Health.FAULTY, health.get(0).getHealth());
+        assertEquals(Health.FAULTY, health.get(1).getHealth());
+    }
+
+    @Test
+    public void tryFailover() {
+        stubController();
+        stubFor(get(urlEqualTo("/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards")).willReturn(aResponse().withStatus(200)));
+        FailoverStatus status = resolver.tryFailover(null);
+        assertEquals(500, status.getStatusCode());
+    }
+}
\ No newline at end of file
diff --git a/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/ThreeNodeHealthResolverTest.java b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/ThreeNodeHealthResolverTest.java
new file mode 100644
index 000000000..4ea07be43
--- /dev/null
+++ b/grToolkit/provider/src/test/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/ThreeNodeHealthResolverTest.java
@@ -0,0 +1,314 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * openECOMP : SDN-C
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * 			reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.ccsdk.sli.plugins.grtoolkit.resolver;
+
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import org.onap.ccsdk.sli.core.dblib.DBLibConnection;
+import org.onap.ccsdk.sli.core.dblib.DbLibService;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.AdminHealth;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterHealth;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.DatabaseHealth;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.FailoverStatus;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.Health;
+import org.onap.ccsdk.sli.plugins.grtoolkit.data.SiteHealth;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+
+import static org.junit.Assert.*;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ThreeNodeHealthResolverTest {
+    private Map<String, ClusterActor> memberMap;
+    private DbLibService dbLibService;
+    private DBLibConnection connection;
+    private ThreeNodeHealthResolver resolver;
+
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(9999);
+
+    @Before
+    public void setUp() {
+        memberMap = generateMemberMap(3);
+        Properties properties = new Properties();
+        try(FileInputStream fileInputStream = new FileInputStream("src/test/resources/three/gr-toolkit.properties")) {
+            properties.load(fileInputStream);
+        } catch(IOException e) {
+            fail();
+        }
+
+        dbLibService = mock(DbLibService.class);
+        connection = mock(DBLibConnection.class);
+        resolver = new ThreeNodeHealthResolver(memberMap, properties, dbLibService);
+    }
+
+    private Map<String, ClusterActor> generateMemberMap(int memberCount) {
+        Map<String, ClusterActor> map = new HashMap<>();
+        ClusterActor actor;
+        for(int ndx = 0; ndx < memberCount; ndx++) {
+            actor = new ClusterActor();
+            actor.setNode("127.0.1." + (ndx + 1));
+            actor.setAkkaPort("2550");
+            actor.setMember("member-" + (ndx + 1));
+            actor.setUp(true);
+            actor.setUnreachable(false);
+
+            map.put(actor.getNode(),  actor);
+        }
+        return map;
+    }
+
+    @Test
+    public void getAdminHealthFaulty() {
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(500)));
+        AdminHealth health = resolver.getAdminHealth();
+        assertNotNull(health);
+        assertEquals(500, health.getStatusCode());
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void getAdminHealthHealthy() {
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        AdminHealth health = resolver.getAdminHealth();
+        assertNotNull(health);
+        assertEquals(200, health.getStatusCode());
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    @Test
+    public void getDatabaseHealth() {
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        DatabaseHealth health = resolver.getDatabaseHealth();
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    @Test
+    public void getDatabaseHealthFaulty() {
+        try {
+            when(connection.isReadOnly()).thenReturn(true);
+            when(connection.isClosed()).thenReturn(true);
+            when(dbLibService.isActive()).thenReturn(false);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        DatabaseHealth health = resolver.getDatabaseHealth();
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void getDatabaseHealthException() {
+        try {
+            when(connection.isReadOnly()).thenThrow(new SQLException());
+            when(connection.isClosed()).thenReturn(true);
+            when(dbLibService.isActive()).thenReturn(false);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        DatabaseHealth health = resolver.getDatabaseHealth();
+        assertEquals(Health.FAULTY, health.getHealth());
+    }
+
+    @Test
+    public void siteIdentifier() {
+        assertEquals("TestODL", resolver.getSiteIdentifier());
+        resolver.setSiteIdentifier("NewTestODL");
+        assertEquals("NewTestODL", resolver.getSiteIdentifier());
+    }
+
+    @Test
+    public void getClusterHealth() {
+        stubController();
+        ClusterHealth health = resolver.getClusterHealth();
+        assertEquals(Health.HEALTHY, health.getHealth());
+    }
+
+    private void stubController() {
+        String clusterBody = null;
+        String shardManagerBody = null;
+        String shardDefaultBody = null;
+        String shardOperationalBody = null;
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/three/cluster.json"))) {
+            clusterBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/three/shard-manager.json"))) {
+            shardManagerBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/three/default-config.json"))) {
+            shardDefaultBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+        try(Stream<String> stream = Files.lines(Paths.get("src/test/resources/three/default-operational.json"))) {
+            shardOperationalBody = stream.collect(Collectors.joining());
+        } catch(IOException e) {
+            fail();
+        }
+
+        if(clusterBody == null || shardManagerBody == null || shardDefaultBody == null || shardOperationalBody == null) {
+            fail();
+        }
+        stubFor(get(urlEqualTo("/jolokia/read/akka:type=Cluster")).willReturn(aResponse().withStatus(200).withBody(clusterBody)));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore")).inScenario("testing").willReturn(aResponse().withStatus(200).withBody(shardManagerBody)).willSetStateTo("next"));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-config,type=DistributedConfigDatastore")).inScenario("testing").willReturn(aResponse().withStatus(200).withBody(shardDefaultBody)).willSetStateTo("next"));
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-operational,type=DistributedOperationalDatastore")).willReturn(aResponse().withStatus(200).withBody(shardOperationalBody)));
+    }
+
+    @Test
+    public void getSiteHealth() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(1, health.size());
+        assertEquals(Health.HEALTHY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void getSiteHealthFaulty() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore")).inScenario("testing").whenScenarioStateIs("next").willReturn(aResponse().withBodyFile("nonexistent")));
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(1, health.size());
+        assertEquals(Health.FAULTY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void getSiteHealthFaultyShard() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        stubFor(get(urlEqualTo("/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-1-shard-default-config,type=DistributedConfigDatastore")).inScenario("testing").willReturn(aResponse().withBodyFile("nonexistent")).willSetStateTo("next"));
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(1, health.size());
+        assertEquals(Health.HEALTHY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void getSiteHealthFaultyCluster() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(200)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        stubFor(get(urlEqualTo("/jolokia/read/akka:type=Cluster")).willReturn(aResponse().withStatus(200).withBodyFile("nonexistent")));
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(1, health.size());
+        assertEquals(Health.FAULTY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void getSiteHealthFaultyAdmin() {
+        stubController();
+        stubFor(get(urlEqualTo("/adm/healthcheck")).willReturn(aResponse().withStatus(400)));
+        try {
+            when(connection.isReadOnly()).thenReturn(false);
+            when(connection.isClosed()).thenReturn(false);
+            when(dbLibService.isActive()).thenReturn(true);
+            when(dbLibService.getConnection()).thenReturn(connection);
+        } catch(SQLException e) {
+            fail();
+        }
+        List<SiteHealth> health = resolver.getSiteHealth();
+        assertNotNull(health);
+        assertNotEquals(0, health.size());
+        assertEquals(1, health.size());
+        assertEquals(Health.FAULTY, health.get(0).getHealth());
+    }
+
+    @Test
+    public void tryFailover() {
+        FailoverStatus status = resolver.tryFailover(null);
+        assertEquals(400, status.getStatusCode());
+    }
+}
\ No newline at end of file