Merge "Add controllers and remote servers healthchecks"

9 files changed, 1072 insertions, 402 deletions

diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckFeatureTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckFeatureTest.java
index 4d93af23..0567595f 100644
--- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckFeatureTest.java
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckFeatureTest.java
@@ -1,8 +1,8 @@
 /*-
  * ============LICENSE_START=======================================================
- * feature-healthcheck
+ * ONAP
  * ================================================================================
- * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2019,2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,137 +23,147 @@ package org.onap.policy.drools.healthcheck;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
-import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Properties;
+import org.eclipse.jetty.http.HttpStatus;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.onap.policy.common.endpoints.properties.PolicyEndPointProperties;
+import org.kie.api.builder.ReleaseId;
+import org.mockito.AdditionalAnswers;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactoryInstance;
+import org.onap.policy.common.endpoints.http.server.HttpServletServerFactoryInstance;
+import org.onap.policy.common.utils.logging.LoggerUtils;
 import org.onap.policy.common.utils.network.NetworkUtil;
-import org.onap.policy.drools.healthcheck.HealthCheck.Report;
 import org.onap.policy.drools.healthcheck.HealthCheck.Reports;
 import org.onap.policy.drools.persistence.SystemPersistenceConstants;
+import org.onap.policy.drools.properties.DroolsPropertyConstants;
+import org.onap.policy.drools.system.PolicyController;
+import org.onap.policy.drools.system.PolicyControllerConstants;
 import org.onap.policy.drools.system.PolicyEngineConstants;
+import org.onap.policy.drools.util.KieUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class HealthCheckFeatureTest {
 
-    /**
-     * Healthcheck Configuration File.
-     */
-    private static final String HEALTH_CHECK_PROPERTIES_FILE = "feature-healthcheck.properties";
-
-    private static final Path healthCheckPropsPath =
-                    Paths.get(SystemPersistenceConstants.getManager().getConfigurationPath().toString(),
-                                    HEALTH_CHECK_PROPERTIES_FILE);
-
-    private static final Path healthCheckPropsBackupPath =
-                    Paths.get(SystemPersistenceConstants.getManager().getConfigurationPath().toString(),
-                                    HEALTH_CHECK_PROPERTIES_FILE + ".bak");
-
+    private static final Logger logger = LoggerFactory.getLogger(HealthCheckFeatureTest.class);
     private static final String EXPECTED = "expected exception";
 
-
-    /**
-     * logger.
-     */
-    private static Logger logger = LoggerFactory.getLogger(HealthCheckFeatureTest.class);
-
-    private static Properties httpProperties = new Properties();
-
     /**
      * Set up.
      */
     @BeforeClass
-    public static void setup() {
-
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES, "HEALTHCHECK");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_HOST_SUFFIX, "localhost");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_PORT_SUFFIX, "7777");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_AUTH_USERNAME_SUFFIX, "username");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_AUTH_PASSWORD_SUFFIX, "password");
-        httpProperties.setProperty(
-                        PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                                        + PolicyEndPointProperties.PROPERTY_HTTP_REST_CLASSES_SUFFIX,
-                        org.onap.policy.drools.healthcheck.RestMockHealthCheck.class.getName());
-        httpProperties.setProperty(
-                        PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                                        + PolicyEndPointProperties.PROPERTY_HTTP_FILTER_CLASSES_SUFFIX,
-                        org.onap.policy.drools.healthcheck.TestAafHealthCheckFilter.class.getName());
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_MANAGED_SUFFIX, "true");
-
-
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES, "HEALTHCHECK");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_HOST_SUFFIX, "localhost");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_PORT_SUFFIX, "7777");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_URL_SUFFIX, "healthcheck/test");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_HTTPS_SUFFIX, "false");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_AUTH_USERNAME_SUFFIX, "username");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_AUTH_PASSWORD_SUFFIX, "password");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_MANAGED_SUFFIX, "true");
-
-        configDirSetup();
-
+    public static void setup() throws IOException {
+        SystemPersistenceConstants.getManager().setConfigurationDir("target/test-classes");
+
+        LoggerUtils.setLevel("org.onap.policy.common.endpoints", "WARN");
+        LoggerUtils.setLevel("org.eclipse", "ERROR");
+        LoggerUtils.setLevel("org.onap.policy.drools.healthcheck", "DEBUG");
+        LoggerUtils.setLevel("ROOT", "INFO");
+
+        ReleaseId coords = KieUtils.installArtifact(Paths.get("src/test/resources/echo.kmodule").toFile(),
+            Paths.get("src/test/resources/echo.pom").toFile(),
+            "src/main/resources/kbecho/org/onap/policy/drools/healthcheck/",
+            List.of(Paths.get("src/test/resources/echo.drl").toFile()));
+
+        Properties controllerProps = new Properties();
+        controllerProps.put(DroolsPropertyConstants.PROPERTY_CONTROLLER_NAME, "echo");
+        controllerProps.put(DroolsPropertyConstants.RULES_GROUPID, coords.getGroupId());
+        controllerProps.put(DroolsPropertyConstants.RULES_ARTIFACTID, coords.getArtifactId());
+        controllerProps.put(DroolsPropertyConstants.RULES_VERSION, coords.getVersion());
+
+        PolicyController controller = PolicyControllerConstants.getFactory().build("echo", controllerProps);
+        controller.start();
     }
 
     /**
      * Tear down.
      */
     @AfterClass
-    public static void tearDown() {
-        logger.info("-- tearDown() --");
-
-        configDirCleanup();
+    public static void teardown() {
+        PolicyControllerConstants.getFactory().destroy();
+        HttpClientFactoryInstance.getClientFactory().destroy();
+        HttpServletServerFactoryInstance.getServerFactory().destroy();
     }
 
     @Test
-    public void test() throws IOException, InterruptedException {
+    public void test() throws InterruptedException {
+        var manager = spy(HealthCheckManager.class);
+        var feature = new HealthCheckFeatureImpl(manager);
+        when(manager.isEngineAlive()).thenReturn(true);
 
-        HealthCheckFeature feature = new HealthCheckFeature();
         feature.afterStart(PolicyEngineConstants.getManager());
+        feature.afterOpen(PolicyEngineConstants.getManager());
 
-        if (!NetworkUtil.isTcpPortOpen("localhost", 7777, 5, 10000L)) {
-            throw new IllegalStateException("cannot connect to port " + 7777);
-        }
+        checkOpen(7777);
+        checkOpen(7776);
 
-        Reports reports = HealthCheckConstants.getManager().healthCheck();
+        var reports = healthcheck(manager);
+        serverChecks(reports);
+        checkReports(reports, List.of("STUCK"),
+                HttpStatus.OK_200, HttpStatus.getMessage(200));
+        checkReports(reports, List.of("echo"), 1, "[echo:{java.lang.String=1}]");
 
-        assertTrue(reports.getDetails().size() > 0);
+        /* mock controller and clients stuck */
 
-        for (Report rpt : reports.getDetails()) {
-            if ("HEALTHCHECK".equals(rpt.getName())) {
-                assertTrue(rpt.isHealthy());
-                assertEquals(200, rpt.getCode());
-                assertEquals("All Alive", rpt.getMessage());
-                break;
-            }
-        }
+        RestMockHealthCheck.stuck = true;   // make the server named STUCK unresponsive
+        doAnswer(AdditionalAnswers
+                .answersWithDelay((manager.getTimeoutSeconds() + 2) * 1000L,
+                        invocationOnMock -> new HashMap<String, Integer>()))
+                .when(manager).getFactTypes(any(), any());
+
+        reports = healthcheck(manager);
+        RestMockHealthCheck.stuck = false;  // unstuck the server named STUCK
+
+        serverChecks(reports);
+        checkReports(reports, List.of("STUCK"),
+                HealthCheckManager.TIMEOUT_CODE, HealthCheckManager.TIMEOUT_MESSAGE);
+
+        assertTrue(RestMockHealthCheck.WAIT * 1000 > HealthCheckManagerTest.select(reports, "STUCK",
+                        HealthCheckManager.TIMEOUT_CODE, HealthCheckManager.TIMEOUT_MESSAGE)
+                .get(0).getElapsedTime());
 
         feature.afterShutdown(PolicyEngineConstants.getManager());
+    }
+
+    private void checkReports(Reports reports, List<String> reportNames, int code, String message) {
+        reportNames
+                .forEach(name -> assertEquals(1,
+                        HealthCheckManagerTest.select(reports, name, code, message).size()));
+    }
+
+    private Reports healthcheck(HealthCheck manager) {
+        var reports = manager.healthCheck();
+        logger.info("{}", reports);
+        return reports;
+    }
+
+    private void checkOpen(int port) throws InterruptedException {
+        if (!NetworkUtil.isTcpPortOpen("localhost", port, 5, 10000L)) {
+            throw new IllegalStateException("cannot connect to port " + port);
+        }
+    }
 
+    private void serverChecks(Reports reports) {
+        checkReports(reports, List.of("HEALTHCHECK", "LIVENESS"),
+                HttpStatus.OK_200, HttpStatus.getMessage(200));
+        checkReports(reports, List.of("UNAUTH"),
+                HttpStatus.UNAUTHORIZED_401, HttpStatus.getMessage(401));
+        checkReports(reports, List.of(HealthCheckManager.ENGINE_NAME),
+                HealthCheckManager.SUCCESS_CODE, HealthCheckManager.ENABLED_MESSAGE);
     }
 
     @Test
@@ -177,6 +187,22 @@ public class HealthCheckFeatureTest {
     }
 
     @Test
+    public void testAfterOpen() {
+        HealthCheck checker = mock(HealthCheck.class);
+        HealthCheckFeature feature = new HealthCheckFeatureImpl(checker);
+
+        // without exception
+        assertFalse(feature.afterOpen(null));
+        verify(checker).open();
+        verify(checker, never()).stop();
+
+        // with exception
+        doThrow(new IllegalStateException(EXPECTED)).when(checker).open();
+        assertFalse(feature.afterOpen(null));
+
+    }
+
+    @Test
     public void testAfterShutdown() {
         HealthCheck checker = mock(HealthCheck.class);
         HealthCheckFeature feature = new HealthCheckFeatureImpl(checker);
@@ -191,49 +217,6 @@ public class HealthCheckFeatureTest {
         assertFalse(feature.afterShutdown(null));
     }
 
-
-    /**
-     * setup up config directory.
-     */
-    private static void configDirSetup() {
-
-        File origPropsFile = new File(healthCheckPropsPath.toString());
-        File backupPropsFile = new File(healthCheckPropsBackupPath.toString());
-        Path configDir = Paths.get(SystemPersistenceConstants.DEFAULT_CONFIGURATION_DIR);
-
-        try {
-
-            if (Files.notExists(configDir)) {
-                Files.createDirectories(configDir);
-            }
-
-            Files.deleteIfExists(healthCheckPropsBackupPath);
-            origPropsFile.renameTo(backupPropsFile);
-
-            FileWriter writer = new FileWriter(origPropsFile);
-            httpProperties.store(writer, "Machine created healthcheck-feature Properties");
-
-        } catch (final Exception e) {
-            logger.info("Problem cleaning {}", healthCheckPropsPath, e);
-        }
-    }
-
-    /**
-     * cleanup up config directory.
-     */
-    private static void configDirCleanup() {
-
-        File origPropsFile = new File(healthCheckPropsBackupPath.toString());
-        File backupPropsFile = new File(healthCheckPropsPath.toString());
-
-        try {
-            Files.deleteIfExists(healthCheckPropsPath);
-            origPropsFile.renameTo(backupPropsFile);
-        } catch (final Exception e) {
-            logger.info("Problem cleaning {}", healthCheckPropsPath, e);
-        }
-    }
-
     /**
      * Feature that returns a particular monitor.
      */
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
new file mode 100644
index 00000000..73d70da9
--- /dev/null
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
@@ -0,0 +1,596 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.healthcheck;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.net.HttpURLConnection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import javax.ws.rs.core.Response;
+import org.eclipse.jetty.http.HttpStatus;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.AdditionalAnswers;
+import org.onap.policy.common.endpoints.http.client.HttpClient;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactory;
+import org.onap.policy.common.endpoints.http.server.HttpServletServer;
+import org.onap.policy.common.endpoints.http.server.HttpServletServerFactory;
+import org.onap.policy.drools.controller.DroolsController;
+import org.onap.policy.drools.healthcheck.HealthCheck.Report;
+import org.onap.policy.drools.healthcheck.HealthCheck.Reports;
+import org.onap.policy.drools.system.PolicyController;
+import org.onap.policy.drools.system.PolicyControllerFactory;
+import org.onap.policy.drools.system.PolicyEngine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HealthCheckManagerTest {
+
+    private static final Logger logger = LoggerFactory.getLogger(HealthCheckManagerTest.class);
+
+    protected static List<Report> select(Reports reports, String name, long code, String message) {
+        return reports.getDetails().stream()
+                .filter(report -> name.equals(report.getName()))
+                .filter(report -> report.getCode() == code)
+                .filter(report -> message.equals(report.getMessage()))
+                .collect(Collectors.toList());
+    }
+
+    private static final String RPT_MSG = "report-message";
+    private static final String RPT_NAME = "report-name";
+    private static final String EXPECTED = "expected exception";
+
+    private static final String CLIENT_NAME1 = "name-a";
+    private static final String CLIENT_URL1 = "url-a";
+    private static final String CLIENT_NAME2 = "name-b";
+    private static final String CLIENT_URL2 = "url-b";
+    private static final String CLIENT_NAME3 = "name-c";
+    private static final String CLIENT_URL3 = "url-c";
+
+    private Properties properties;
+    private HttpServletServerFactory servletFactory;
+    private HttpServletServer server1;
+    private HttpServletServer server2;
+    private HttpClientFactory clientFactory;
+    private HttpClient client1;
+    private HttpClient client2;
+    private HttpClient client3;
+
+    private PolicyControllerFactory controllerFactory;
+    private PolicyController controller1;
+    private PolicyController controller2;
+    private DroolsController drools1;
+    private DroolsController drools2;
+
+    private PolicyEngine engineMgr;
+    private HealthCheckManager monitor;
+
+    /**
+     * Initializes the object to be tested.
+     */
+    @Before
+    public void setUp() throws Exception {
+        properties = new Properties();
+        mocks();
+
+        List<HttpServletServer> servers = Arrays.asList(server1, server2);
+        List<HttpClient> clients = Arrays.asList(client1, client2, client3);
+
+        whenClients();
+
+        when(servletFactory.build(properties)).thenReturn(servers);
+        when(clientFactory.build(properties)).thenReturn(clients);
+
+        whenControllers();
+
+        when(engineMgr.isAlive()).thenReturn(true);
+
+        monitor = new HealthCheckMonitorImpl();
+    }
+
+    private void whenControllers() {
+        when(drools1.getGroupId()).thenReturn("1");
+        when(drools2.getGroupId()).thenReturn("2");
+
+        when(drools1.getArtifactId()).thenReturn("1");
+        when(drools2.getArtifactId()).thenReturn("2");
+
+        when(drools1.getVersion()).thenReturn("1");
+        when(drools2.getVersion()).thenReturn("2");
+
+        when(drools1.isAlive()).thenReturn(true);
+        when(drools2.isAlive()).thenReturn(true);
+
+        when(drools1.isBrained()).thenReturn(true);
+        when(drools2.isBrained()).thenReturn(true);
+
+        when(drools1.getSessionNames()).thenReturn(List.of("session1"));
+        when(drools2.getSessionNames()).thenReturn(List.of("session2"));
+
+        doAnswer(AdditionalAnswers
+                .answersWithDelay(15000L, invocationOnMock -> Map.of("TIMEOUT", 1)))
+                .when(drools1).factClassNames(anyString());
+
+        when(drools2.factClassNames(anyString()))
+                .thenReturn(Map.of("java.lang.Integer", 2));
+
+        when(drools1.factCount("session1")).thenReturn(1L);
+        when(drools2.factCount("session2")).thenReturn(2L);
+
+        when(controller1.getDrools()).thenReturn(drools1);
+        when(controller2.getDrools()).thenReturn(drools2);
+
+        when(controller1.getName()).thenReturn("drools1");
+        when(controller2.getName()).thenReturn("drools2");
+
+        when(controller1.isAlive()).thenReturn(true);
+        when(controller2.isAlive()).thenReturn(true);
+    }
+
+    private void whenClients() {
+        when(client1.getName()).thenReturn(CLIENT_NAME1);
+        when(client1.getBaseUrl()).thenReturn(CLIENT_URL1);
+        when(client2.getName()).thenReturn(CLIENT_NAME2);
+        when(client2.getBaseUrl()).thenReturn(CLIENT_URL2);
+        when(client3.getName()).thenReturn(CLIENT_NAME3);
+        when(client3.getBaseUrl()).thenReturn(CLIENT_URL3);
+    }
+
+    private void mocks() {
+        servletFactory = mock(HttpServletServerFactory.class);
+        server1 = mock(HttpServletServer.class);
+        server2 = mock(HttpServletServer.class);
+        clientFactory = mock(HttpClientFactory.class);
+        client1 = mock(HttpClient.class);
+        client2 = mock(HttpClient.class);
+        client3 = mock(HttpClient.class);
+        controllerFactory = mock(PolicyControllerFactory.class);
+        controller1 = mock(PolicyController.class);
+        controller2 = mock(PolicyController.class);
+        drools1 = mock(DroolsController.class);
+        drools2 = mock(DroolsController.class);
+        engineMgr = mock(PolicyEngine.class);
+    }
+
+    @Test
+    public void testHealthcheck() {
+        /* engine not alive */
+
+        when(engineMgr.isAlive()).thenReturn(false);
+        assertEngineDisabled(monitor.healthCheck());
+
+        /* engine alive + controllers + clients */
+
+        when(engineMgr.isAlive()).thenReturn(true);
+        assertEngineEnabled(monitor.healthCheck());
+
+        monitor.controllers = List.of(controller1, controller2);
+
+        mockClients();
+
+        var reports = monitor.healthCheck();
+        logger.info("{}", reports);
+
+        assertSummary(reports, 6, false);
+        assertClients(reports);
+        assertControllers(reports);
+    }
+
+    @Test
+    public void testControllerHealthcheck() {
+        /* engine not alive */
+
+        when(engineMgr.isAlive()).thenReturn(false);
+        assertEngineDisabled(monitor.controllerHealthcheck());
+
+        /* engine alive */
+
+        when(engineMgr.isAlive()).thenReturn(true);
+        assertEngineEnabled(monitor.healthCheck());
+
+        /* engine + controllers */
+
+        monitor.controllers = List.of(controller1, controller2);
+        var reports = monitor.healthCheck();
+        logger.info("{}", reports);
+
+        assertSummary(reports, 3, false);
+
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+
+        assertControllers(reports);
+
+        /* with argument */
+
+        reports = monitor.controllerHealthcheck(controller2);
+        logger.info("{}", reports);
+
+        assertSummary(reports, 2, true);
+    }
+
+    @Test
+    public void testClientHealthcheck() {
+        /* engine not alive */
+
+        when(engineMgr.isAlive()).thenReturn(false);
+        assertEngineDisabled(monitor.clientHealthcheck());
+
+        /* engine alive */
+
+        when(engineMgr.isAlive()).thenReturn(true);
+        assertEngineEnabled(monitor.clientHealthcheck());
+
+        /* engine alive + clients */
+
+        mockClients();
+
+        var reports = monitor.clientHealthcheck();
+        logger.info("{}", reports);
+
+        assertSummary(reports, 4, false);
+        assertClients(reports);
+
+        /* with argument */
+
+        reports = monitor.clientHealthcheck(client1);
+        logger.info("{}", reports);
+
+        assertSummary(reports, 2, true);
+    }
+
+    @Test
+    public void reportOnController() {
+
+        /* controller not alive */
+
+        when(controller1.isAlive()).thenReturn(false);
+
+        var reports = monitor.controllerHealthcheck(controller1);
+        assertSummary(reports, 2, false);
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+        assertReport(reports, false,
+                    controller1.getName(), controller1.getName(),
+                    HealthCheckManager.DISABLED_CODE, HealthCheckManager.DISABLED_MESSAGE);
+
+        /* drools not brained */
+
+        when(controller1.isAlive()).thenReturn(true);
+        when(drools1.isBrained()).thenReturn(false);
+
+        reports = monitor.controllerHealthcheck(controller1);
+        logger.info("{}", reports);
+
+        assertSummary(reports, 2, true);
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+        assertReport(reports, true,
+                controller1.getName(), "1:1:1",
+                HealthCheckManager.BRAINLESS_CODE, HealthCheckManager.BRAINLESS_MESSAGE);
+
+        /* drools not alive */
+
+        when(drools1.isBrained()).thenReturn(true);
+        when(drools1.isAlive()).thenReturn(false);
+
+        reports = monitor.controllerHealthcheck(controller1);
+        logger.info("{}", reports);
+
+        assertSummary(reports, 2, false);
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+        assertReport(reports, false,
+                controller1.getName(), "1:1:1",
+                HealthCheckManager.DISABLED_CODE, HealthCheckManager.DISABLED_MESSAGE);
+
+        /* ok */
+
+        when(drools1.isAlive()).thenReturn(true);
+
+        assertController2(monitor.controllerHealthcheck(controller2));
+    }
+
+    @Test
+    public void testReportOnUnknown() {
+        var reports = monitor.summary(monitor.engineHealthcheck(), monitor.futures(List.of(1)));
+        logger.info("{}", reports);
+
+        assertReport(reports, false,
+            HealthCheckManager.UNKNOWN_ENTITY, "java.lang.Integer",
+            HealthCheckManager.UNKNOWN_ENTITY_CODE, HealthCheckManager.UNKNOWN_ENTITY_MESSAGE);
+    }
+
+    @Test
+    public void testStart() {
+        // good start
+
+        when(server1.start()).thenReturn(true);
+        when(server1.getName()).thenReturn(HealthCheckManager.HEALTHCHECK_SERVER);
+        when(server2.getName()).thenReturn(HealthCheckManager.LIVENESS_SERVER);
+        assertTrue(monitor.start());
+
+        verify(server1).start();
+        verify(server2, never()).start();
+
+        assertEquals(server1, monitor.getHealthcheckServer());
+        assertEquals(server2, monitor.getLivenessServer());
+
+        // healthcheck server start error
+
+        when(server1.start()).thenThrow(new RuntimeException(EXPECTED));
+        assertFalse(monitor.start());
+
+        /*
+         * Generate exception during building.
+         */
+
+        // new monitor
+        monitor = new HealthCheckMonitorImpl() {
+            @Override
+            protected HttpServletServerFactory getServerFactory() {
+                throw new RuntimeException(EXPECTED);
+            }
+        };
+        assertFalse(monitor.start());
+
+    }
+
+    @Test
+    public void testOpen() {
+
+        /* nothing done */
+
+        monitor.healthCheckProperties = new Properties();
+        monitor.open();
+        assertEquals(List.of(), monitor.controllers);
+
+        /* star-controllers */
+
+        monitor.livenessServer = server1;
+        monitor.healthCheckProperties = new Properties();
+        monitor.healthCheckProperties.setProperty("liveness.controllers", "*");
+        when(server1.start()).thenReturn(true);
+
+        monitor.open();
+        assertEquals(controllerFactory.inventory(), monitor.controllers);
+        verify(server1).start();
+
+        /* comma-list-controllers */
+
+        monitor.controllers = new ArrayList<>();
+        monitor.healthCheckProperties.setProperty("liveness.controllers", "controller1,controller2,controller3");
+        when(controllerFactory.get("controller1")).thenReturn(controller1);
+        when(controllerFactory.get("controller2")).thenReturn(controller2);
+        when(controllerFactory.get("controller3")).thenThrow(new RuntimeException("no controller3"));
+        monitor.open();
+        assertEquals(List.of(controller1, controller2), monitor.controllers);
+    }
+
+    @Test
+    public void testShutdown() {
+        monitor.healthcheckServer = server1;
+        monitor.livenessServer = server2;
+        monitor.clients = List.of(client1, client2, client3);
+        when(server1.stop()).thenReturn(true);
+        when(server2.stop()).thenReturn(true);
+
+        monitor.shutdown();
+
+        verify(server1).stop();
+        verify(server2).stop();
+        verify(client1).stop();
+        verify(client2).stop();
+        verify(client3).stop();
+    }
+
+    @Test
+    public void testIsAlive() {
+        assertFalse(monitor.isAlive());
+    }
+
+    @Test
+    public void testToString() {
+        assertTrue(monitor.toString().contains("HealthCheckManager"));
+    }
+
+    private void mockClient1() {
+        // first client is healthy
+        Response resp = mock(Response.class);
+        when(resp.getStatus()).thenReturn(HttpURLConnection.HTTP_OK);
+        when(resp.readEntity(String.class)).thenReturn(RPT_MSG);
+        when(resp.getStatusInfo()).thenReturn(Response.Status.OK);
+        when(client1.get()).thenReturn(resp);
+    }
+
+    private void mockClient2() {
+        // second client throws an exception
+        when(client2.get()).thenThrow(new RuntimeException(EXPECTED));
+    }
+
+    private void mockClient3() {
+        // third client is not healthy
+        Response resp = mock(Response.class);
+        when(resp.getStatus()).thenReturn(HttpURLConnection.HTTP_NOT_FOUND);
+        when(resp.readEntity(String.class)).thenReturn(RPT_NAME);
+        when(resp.getStatusInfo()).thenReturn(Response.Status.NOT_FOUND);
+        when(client3.get()).thenReturn(resp);
+    }
+
+    private void mockClients() {
+        monitor.clients = List.of(client1, client2, client3);
+
+        mockClient1();
+        mockClient2();
+        mockClient3();
+    }
+
+    private void assertEngineEnabled(Reports summary) {
+        assertEquals(1, summary.getDetails().size());
+        assertReport(summary, true, HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE, HealthCheckManager.ENABLED_MESSAGE);
+    }
+
+    private void assertEngineDisabled(Reports summary) {
+        var report = summary.getDetails().get(0);
+        assertFalse(summary.isHealthy());
+        assertEquals(1, summary.getDetails().size());
+        assertFalse(report.isHealthy());
+        assertEquals(HealthCheckManager.ENGINE_NAME, report.getName());
+        assertEquals(HealthCheckManager.DISABLED_CODE, report.getCode());
+        assertEquals(HealthCheckManager.DISABLED_MESSAGE, report.getMessage());
+        assertNotEquals(0L, report.getStartTime());
+        assertNotEquals(0L, report.getEndTime());
+        assertEquals(report.getEndTime() - report.getStartTime(), report.getElapsedTime());
+    }
+
+    private void assertSummary(Reports reports, int size, boolean healthy) {
+        assertNotNull(reports);
+        assertEquals(size, reports.getDetails().size());
+        assertEquals(healthy, reports.isHealthy());
+        assertNotEquals(0L, reports.getStartTime());
+        assertNotEquals(0L, reports.getEndTime());
+        assertEquals(reports.getEndTime() - reports.getStartTime(), reports.getElapsedTime());
+    }
+
+    private void assertReport(Reports summary,
+            boolean healthy, String name, String url, long successCode, String message) {
+        var report = select(summary, name, successCode, message).get(0);
+
+        assertEquals(healthy, report.isHealthy());
+        assertEquals(name, report.getName());
+        assertEquals(url, report.getUrl());
+        assertEquals(successCode, report.getCode());
+        assertEquals(message, report.getMessage());
+        assertNotEquals(0L, report.getStartTime());
+        assertNotEquals(0L, report.getEndTime());
+        assertEquals(report.getEndTime() - report.getStartTime(), report.getElapsedTime());
+    }
+
+    private void assertClient1(Reports reports) {
+        assertReport(reports, true,
+                client1.getName(), client1.getBaseUrl(),
+                HttpStatus.OK_200,
+                HttpStatus.getMessage(200));
+    }
+
+    private void assertClient2(Reports reports) {
+        assertReport(reports, false,
+                client2.getName(), client2.getBaseUrl(),
+                HealthCheckManager.UNREACHABLE_CODE,
+                HealthCheckManager.UNREACHABLE_MESSAGE);
+    }
+
+    private void assertClient3(Reports reports) {
+        assertReport(reports, false,
+                client3.getName(), client3.getBaseUrl(),
+                HttpStatus.NOT_FOUND_404,
+                HttpStatus.getMessage(404));
+    }
+
+    private void assertClients(Reports reports) {
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+
+        assertClient1(reports);
+        assertClient2(reports);
+        assertClient3(reports);
+    }
+
+    private void assertController1(Reports reports) {
+        assertReport(reports, false,
+                controller1.getName(), "1:1:1",
+                HealthCheckManager.TIMEOUT_CODE, HealthCheckManager.TIMEOUT_MESSAGE);
+    }
+
+    private void assertController2(Reports reports) {
+        assertReport(reports, true,
+                controller2.getName(), "2:2:2",
+                2, "[session2:{java.lang.Integer=2}]");
+    }
+
+    private void assertControllers(Reports reports) {
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+
+        assertController1(reports);
+        assertController2(reports);
+    }
+
+    /**
+     * Monitor with overrides.
+     */
+    private class HealthCheckMonitorImpl extends HealthCheckManager {
+
+        @Override
+        protected PolicyEngine getEngineManager() {
+            return engineMgr;
+        }
+
+        @Override
+        protected HttpServletServerFactory getServerFactory() {
+            return servletFactory;
+        }
+
+        @Override
+        protected HttpClientFactory getClientFactory() {
+            return clientFactory;
+        }
+
+        @Override
+        protected Properties getPersistentProperties() {
+            return properties;
+        }
+
+        @Override
+        protected PolicyControllerFactory getControllerFactory() {
+            return controllerFactory;
+        }
+
+    }
+}
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckTest.java
index 7040f6d3..01f5063d 100644
--- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckTest.java
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckTest.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP
  * ================================================================================
- * Copyright (C) 2018, 2021 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2018, 2021-2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,94 +22,27 @@ package org.onap.policy.drools.healthcheck;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
-import java.net.HttpURLConnection;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.Properties;
-import javax.ws.rs.core.Response;
-import org.junit.Before;
 import org.junit.Test;
-import org.onap.policy.common.endpoints.http.client.HttpClient;
-import org.onap.policy.common.endpoints.http.client.HttpClientFactory;
-import org.onap.policy.common.endpoints.http.server.HttpServletServer;
-import org.onap.policy.common.endpoints.http.server.HttpServletServerFactory;
 import org.onap.policy.drools.healthcheck.HealthCheck.Report;
 import org.onap.policy.drools.healthcheck.HealthCheck.Reports;
-import org.onap.policy.drools.system.PolicyEngine;
 
 public class HealthCheckTest {
-
-    private static final int RPT_CODE = 100;
+    private static final long RPT_CODE = 100;
     private static final String RPT_MSG = "report-message";
     private static final String RPT_NAME = "report-name";
     private static final String RPT_URL = "report-url";
-    private static final String EXPECTED = "expected exception";
-
-    private static final String CLIENT_NAME1 = "name-a";
-    private static final String CLIENT_URL1 = "url-a";
-    private static final String CLIENT_NAME2 = "name-b";
-    private static final String CLIENT_URL2 = "url-b";
-    private static final String CLIENT_NAME3 = "name-c";
-    private static final String CLIENT_URL3 = "url-c";
-
-    private Properties properties;
-    private HttpServletServerFactory servletFactory;
-    private HttpServletServer server1;
-    private HttpServletServer server2;
-    private HttpClientFactory clientFactory;
-    private HttpClient client1;
-    private HttpClient client2;
-    private HttpClient client3;
-    private List<HttpServletServer> servers;
-    private List<HttpClient> clients;
-    private PolicyEngine engineMgr;
-    private HealthCheckManager monitor;
-
-    /**
-     * Initializes the object to be tested.
-     *
-     * @throws Exception if an error occurs
-     */
-    @Before
-    public void setUp() throws Exception {
-        properties = new Properties();
-        servletFactory = mock(HttpServletServerFactory.class);
-        server1 = mock(HttpServletServer.class);
-        server2 = mock(HttpServletServer.class);
-        clientFactory = mock(HttpClientFactory.class);
-        client1 = mock(HttpClient.class);
-        client2 = mock(HttpClient.class);
-        client3 = mock(HttpClient.class);
-        servers = Arrays.asList(server1, server2);
-        clients = Arrays.asList(client1, client2, client3);
-        engineMgr = mock(PolicyEngine.class);
-
-        when(client1.getName()).thenReturn(CLIENT_NAME1);
-        when(client1.getBaseUrl()).thenReturn(CLIENT_URL1);
-        when(client2.getName()).thenReturn(CLIENT_NAME2);
-        when(client2.getBaseUrl()).thenReturn(CLIENT_URL2);
-        when(client3.getName()).thenReturn(CLIENT_NAME3);
-        when(client3.getBaseUrl()).thenReturn(CLIENT_URL3);
-        when(servletFactory.build(properties)).thenReturn(servers);
-        when(clientFactory.build(properties)).thenReturn(clients);
-        when(engineMgr.isAlive()).thenReturn(true);
-
-        monitor = new HealthCheckMonitorImpl();
-    }
 
     @Test
     public void testReport() {
         Report rpt = new Report();
 
-        // toString should work with un-populated data
         assertNotNull(rpt.toString());
 
         rpt.setCode(RPT_CODE);
@@ -117,19 +50,26 @@ public class HealthCheckTest {
         rpt.setMessage(RPT_MSG);
         rpt.setName(RPT_NAME);
         rpt.setUrl(RPT_URL);
+        rpt.setEndTime();
 
         assertEquals(RPT_CODE, rpt.getCode());
-        assertEquals(true, rpt.isHealthy());
+        assertTrue(rpt.isHealthy());
         assertEquals(RPT_MSG, rpt.getMessage());
         assertEquals(RPT_NAME, rpt.getName());
         assertEquals(RPT_URL, rpt.getUrl());
 
+        assertNotEquals(0L, rpt.getStartTime());
+        assertNotEquals(0L, rpt.getEndTime());
+        assertEquals(rpt.getEndTime() - rpt.getStartTime(), rpt.getElapsedTime());
+
         // flip the flag
         rpt.setHealthy(false);
-        assertEquals(false, rpt.isHealthy());
+        assertFalse(rpt.isHealthy());
 
         // toString should work with populated data
         assertNotNull(rpt.toString());
+
+        assertEquals(rpt, new Report(rpt));
     }
 
     @Test
@@ -152,202 +92,11 @@ public class HealthCheckTest {
 
         // toString should work with populated data
         assertNotNull(reports.toString());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_HealthCheck() {
-        monitor.start();
-
-        // first client is healthy
-        Response resp = mock(Response.class);
-        when(resp.getStatus()).thenReturn(HttpURLConnection.HTTP_OK);
-        when(resp.readEntity(String.class)).thenReturn(RPT_MSG);
-        when(client1.get()).thenReturn(resp);
-
-        // second client throws an exception
-        when(client2.get()).thenThrow(new RuntimeException(EXPECTED));
-
-        // third client is not healthy
-        resp = mock(Response.class);
-        when(resp.getStatus()).thenReturn(HttpURLConnection.HTTP_INTERNAL_ERROR);
-        when(resp.readEntity(String.class)).thenReturn(RPT_NAME);
-        when(client3.get()).thenReturn(resp);
-
-        Reports reports = monitor.healthCheck();
-        assertNotNull(reports);
-        assertEquals(4, reports.getDetails().size());
-        assertFalse(reports.isHealthy());
-
-        int index = 0;
-
-        Report report = reports.getDetails().get(index++);
-        assertEquals(true, report.isHealthy());
-        assertEquals("PDP-D", report.getName());
-        assertEquals("self", report.getUrl());
-        assertEquals("alive", report.getMessage());
-        assertEquals(HttpURLConnection.HTTP_OK, report.getCode());
-
-        report = reports.getDetails().get(index++);
-        assertEquals(true, report.isHealthy());
-        assertEquals(client1.getName(), report.getName());
-        assertEquals(client1.getBaseUrl(), report.getUrl());
-        assertEquals(RPT_MSG, report.getMessage());
-        assertEquals(HttpURLConnection.HTTP_OK, report.getCode());
-
-        report = reports.getDetails().get(index++);
-        assertEquals(false, report.isHealthy());
-        assertEquals(client2.getName(), report.getName());
-        assertEquals(client2.getBaseUrl(), report.getUrl());
-
-        report = reports.getDetails().get(index++);
-        assertEquals(false, report.isHealthy());
-        assertEquals(client3.getName(), report.getName());
-        assertEquals(client3.getBaseUrl(), report.getUrl());
-        assertEquals(RPT_NAME, report.getMessage());
-        assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, report.getCode());
-
-        // indicate that engine is no longer healthy and re-run health check
-        when(engineMgr.isAlive()).thenReturn(false);
-
-        reports = monitor.healthCheck();
-        report = reports.getDetails().get(0);
-
-        assertEquals(false, report.isHealthy());
-        assertEquals("not alive", report.getMessage());
-        assertEquals(500, report.getCode());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_Start() {
-        // arrange for one server to throw an exception
-        when(server1.start()).thenThrow(new RuntimeException(EXPECTED));
-
-        assertTrue(monitor.start());
-
-        verify(server1).start();
-        verify(server2).start();
-
-        /*
-         * Generate exception during building.
-         */
-
-        // new monitor
-        monitor = new HealthCheckMonitorImpl() {
-            @Override
-            protected HttpServletServerFactory getServerFactory() {
-                throw new RuntimeException(EXPECTED);
-            }
-        };
-
-        assertFalse(monitor.start());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_Stop() {
-        monitor.start();
-
-        // arrange for one server and one client to throw an exception
-        when(server1.stop()).thenThrow(new RuntimeException(EXPECTED));
-        when(client2.stop()).thenThrow(new RuntimeException(EXPECTED));
-
-        assertTrue(monitor.stop());
-
-        verify(server1).stop();
-        verify(server2).stop();
-        verify(client1).stop();
-        verify(client2).stop();
-        verify(client3).stop();
-    }
-
-    @Test
-    public void testHealthCheckMonitor_Shutdown() {
-        monitor.start();
-        monitor.shutdown();
-
-        // at least one "stop" should have been called
-        verify(server1).stop();
-    }
-
-    @Test
-    public void testHealthCheckMonitor_IsAlive() {
-        assertFalse(monitor.isAlive());
-
-        monitor.start();
-        assertTrue(monitor.isAlive());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetServers_GetClients() {
-        monitor.start();
-        assertEquals(servers, monitor.getServers());
-        assertEquals(clients, monitor.getClients());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetHttpBody() {
-        Response response = mock(Response.class);
-        when(response.readEntity(String.class)).thenReturn(RPT_MSG);
-        assertEquals(RPT_MSG, monitor.getHttpBody(response, client1));
-
-        // readEntity() throws an exception
-        when(response.readEntity(String.class)).thenThrow(new RuntimeException(EXPECTED));
-        assertEquals(null, monitor.getHttpBody(response, client1));
-    }
-
-    @Test
-    public void testHealthCheckMonitor_StartServer() {
-        monitor.startServer(server1);
-        verify(server1).start();
-
-        // force start() to throw an exception - monitor should still work
-        when(server1.start()).thenThrow(new RuntimeException(EXPECTED));
-        monitor.startServer(server1);
-    }
-
-    @Test
-    public void testHealthCheckMonitor_ToString() {
-        assertTrue(monitor.toString().startsWith("HealthCheckManager("));
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetEngineManager() {
-        assertNotNull(new HealthCheckManager().getEngineManager());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetServerFactory() {
-        assertNotNull(new HealthCheckManager().getServerFactory());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetClientFactory() {
-        assertNotNull(new HealthCheckManager().getClientFactory());
-    }
-
-    /**
-     * Monitor with overrides.
-     */
-    private class HealthCheckMonitorImpl extends HealthCheckManager {
-
-        @Override
-        protected PolicyEngine getEngineManager() {
-            return engineMgr;
-        }
-
-        @Override
-        protected HttpServletServerFactory getServerFactory() {
-            return servletFactory;
-        }
-
-        @Override
-        protected HttpClientFactory getClientFactory() {
-            return clientFactory;
-        }
 
-        @Override
-        protected Properties getPersistentProperties(String propertyName) {
-            return properties;
-        }
+        assertNotEquals(0L, reports.getStartTime());
 
+        reports.setEndTime();
+        assertNotEquals(0L, reports.getEndTime());
+        assertEquals(reports.getEndTime() - reports.getStartTime(), reports.getElapsedTime());
     }
 }
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestHealthCheckTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestHealthCheckTest.java
new file mode 100644
index 00000000..2af39a80
--- /dev/null
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestHealthCheckTest.java
@@ -0,0 +1,181 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.healthcheck;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import javax.ws.rs.core.Response;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.onap.policy.common.endpoints.event.comm.bus.internal.BusTopicParams;
+import org.onap.policy.common.endpoints.http.client.HttpClient;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactory;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactoryInstance;
+import org.onap.policy.common.endpoints.http.server.HttpServletServer;
+import org.onap.policy.common.endpoints.http.server.HttpServletServerFactoryInstance;
+import org.onap.policy.common.endpoints.http.server.YamlJacksonHandler;
+import org.onap.policy.common.gson.JacksonHandler;
+import org.onap.policy.common.utils.logging.LoggerUtils;
+import org.onap.policy.common.utils.network.NetworkUtil;
+import org.onap.policy.drools.system.PolicyController;
+import org.onap.policy.drools.system.PolicyControllerFactory;
+
+/**
+ * REST Healthcheck Tests.
+ */
+public class RestHealthCheckTest {
+
+    private static HttpClientFactory clientFactory;
+    private static PolicyControllerFactory controllerFactory;
+    private static HealthCheckManager healthcheckManager;
+    private static PolicyController controller1;
+    private static HttpClient client1;
+
+    private static HealthCheck.Reports summary;
+    private static HttpClient client;
+
+    /**
+     * Set up.
+     */
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LoggerUtils.setLevel("org.onap.policy.common.endpoints", "WARN");
+        LoggerUtils.setLevel("org.eclipse", "ERROR");
+        LoggerUtils.setLevel("org.onap.policy.drools.healthcheck", "DEBUG");
+        LoggerUtils.setLevel("ROOT", "INFO");
+
+        clientFactory = mock(HttpClientFactory.class);
+        controllerFactory = mock(PolicyControllerFactory.class);
+        healthcheckManager = mock(HealthCheckManager.class);
+        controller1 = mock(PolicyController.class);
+        client1 = mock(HttpClient.class);
+
+        summary = new HealthCheck.Reports();
+
+        client = HttpClientFactoryInstance.getClientFactory().build(
+                    BusTopicParams.builder()
+                        .clientName("healthcheck")
+                        .hostname("localhost")
+                        .port(8768)
+                        .basePath("healthcheck")
+                        .managed(true)
+                        .build());
+
+        HttpServletServer server =
+            HttpServletServerFactoryInstance
+                .getServerFactory()
+                .build("lifecycle", "localhost", 8768, "/",
+                    true, true);
+
+        server.setSerializationProvider(
+                String.join(",", JacksonHandler.class.getName(),
+                        YamlJacksonHandler.class.getName()));
+        server.addServletClass("/*", RestMockHealthcheck.class.getName());
+        server.waitedStart(5000L);
+
+        assertTrue(NetworkUtil.isTcpPortOpen("localhost", 8768, 5, 10000L));
+    }
+
+    /**
+     * Tear down.
+     */
+
+    public static void tearDown() {
+        HttpClientFactoryInstance.getClientFactory().destroy();
+        HttpServletServerFactoryInstance.getServerFactory().destroy();
+    }
+
+    @Test
+    public void healthcheck() {
+        when(healthcheckManager.healthCheck()).thenReturn(summary);
+        assertHttp("/");
+    }
+
+    @Test
+    public void engine() {
+        when(healthcheckManager.engineHealthcheck()).thenReturn(summary);
+        assertHttp("engine");
+    }
+
+    @Test
+    public void controllers() {
+        when(healthcheckManager.controllerHealthcheck()).thenReturn(summary);
+        assertHttp("controllers");
+
+        when(controllerFactory.get("controller1")).thenReturn(controller1);
+        when(healthcheckManager.controllerHealthcheck(controller1)).thenReturn(summary);
+        assertHttp("controllers/controller1");
+
+        when(controllerFactory.get("controller1")).thenThrow(new IllegalArgumentException("expected"));
+        Response resp = client.get("controllers/controller1");
+        assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus());
+
+        when(controllerFactory.get("controller2")).thenThrow(new IllegalStateException("expected"));
+        resp = client.get("controllers/controller2");
+        assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), resp.getStatus());
+    }
+
+    @Test
+    public void clients() {
+        when(healthcheckManager.clientHealthcheck()).thenReturn(summary);
+        assertHttp("clients");
+
+        when(clientFactory.get("client1")).thenReturn(client1);
+        when(healthcheckManager.clientHealthcheck(client1)).thenReturn(summary);
+        assertHttp("clients/client1");
+
+        when(clientFactory.get("client2")).thenThrow(new IllegalArgumentException("expected"));
+        Response resp = client.get("clients/client2");
+        assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus());
+    }
+
+    private void assertHttp(String url) {
+        summary.setHealthy(true);
+        var resp = client.get(url);
+        assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus());
+
+        summary.setHealthy(false);
+        resp = client.get(url);
+        assertEquals(Response.Status.SERVICE_UNAVAILABLE.getStatusCode(), resp.getStatus());
+    }
+
+    public static class RestMockHealthcheck extends RestHealthCheck {
+        @Override
+        protected PolicyControllerFactory getControllerFactory() {
+            return controllerFactory;
+        }
+
+        @Override
+        protected HttpClientFactory getClientFactory() {
+            return clientFactory;
+        }
+
+        @Override
+        protected HealthCheck getHealthcheckManager() {
+            return healthcheckManager;
+        }
+
+    }
+}
\ No newline at end of file
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestMockHealthCheck.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestMockHealthCheck.java
index 8cbe1fbf..910974ef 100644
--- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestMockHealthCheck.java
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestMockHealthCheck.java
@@ -1,8 +1,8 @@
 /*-
  * ============LICENSE_START=======================================================
- * feature-healthcheck
+ * ONAP
  * ================================================================================
- * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2018,2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,9 @@
 
 package org.onap.policy.drools.healthcheck;
 
+import static org.awaitility.Awaitility.await;
+
+import java.util.concurrent.TimeUnit;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
@@ -31,11 +34,22 @@ import org.onap.policy.common.endpoints.http.server.YamlMessageBodyHandler;
 @Path("/")
 public class RestMockHealthCheck {
 
+    protected static final String OK_MESSAGE = "All Alive";
+    protected static volatile boolean stuck = false;
+    protected static volatile long WAIT = 15;
+
     @GET
     @Path("healthcheck/test")
     @Produces({MediaType.APPLICATION_JSON, YamlMessageBodyHandler.APPLICATION_YAML})
     public Response papHealthCheck() {
-        return Response.status(Status.OK).entity("All Alive").build();
+        return Response.status(Status.OK).entity(OK_MESSAGE).build();
     }
 
+    @GET
+    @Path("healthcheck/stuck")
+    @Produces({MediaType.APPLICATION_JSON, YamlMessageBodyHandler.APPLICATION_YAML})
+    public Response stuck() {
+        await().atMost(WAIT, TimeUnit.SECONDS).until(() -> !stuck);
+        return Response.status(Status.OK).entity("I may be stuck: " + stuck).build();
+    }
 }
diff --git a/feature-healthcheck/src/test/resources/echo.drl b/feature-healthcheck/src/test/resources/echo.drl
new file mode 100644
index 00000000..54847da9
--- /dev/null
+++ b/feature-healthcheck/src/test/resources/echo.drl
@@ -0,0 +1,35 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.healthcheck;
+
+rule "INIT"
+lock-on-active
+when
+then
+	insert(new String("I am up"));
+end
+
+rule "ECHO"
+when
+    $o : Object();
+then
+    System.out.println("ECHO: " + $o.toString());
+end
diff --git a/feature-healthcheck/src/test/resources/echo.kmodule b/feature-healthcheck/src/test/resources/echo.kmodule
new file mode 100644
index 00000000..7a1acc70
--- /dev/null
+++ b/feature-healthcheck/src/test/resources/echo.kmodule
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ============LICENSE_START=======================================================
+  ONAP
+  ================================================================================
+  Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+  ================================================================================
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  ============LICENSE_END=========================================================
+  -->
+
+<kmodule xmlns="http://jboss.org/kie/6.0.0/kmodule">
+    <kbase name="onap.policies.drools.healthcheck">
+        <ksession name="echo"/>
+    </kbase>
+</kmodule>
diff --git a/feature-healthcheck/src/test/resources/echo.pom b/feature-healthcheck/src/test/resources/echo.pom
new file mode 100644
index 00000000..2f84f4bd
--- /dev/null
+++ b/feature-healthcheck/src/test/resources/echo.pom
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ============LICENSE_START=======================================================
+  ONAP
+  ================================================================================
+  Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+  ================================================================================
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  ============LICENSE_END=========================================================
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.onap.policy.drools.healthcheck</groupId>
+    <artifactId>echo</artifactId>
+    <!-- the version below is irrelevant -->
+    <version>1.0.0</version>
+</project>
diff --git a/feature-healthcheck/src/test/resources/feature-healthcheck.properties b/feature-healthcheck/src/test/resources/feature-healthcheck.properties
new file mode 100644
index 00000000..63a71b6e
--- /dev/null
+++ b/feature-healthcheck/src/test/resources/feature-healthcheck.properties
@@ -0,0 +1,55 @@
+#
+# ============LICENSE_START=======================================================
+# ONAP
+# ================================================================================
+# Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+
+http.server.services=HEALTHCHECK,LIVENESS
+
+http.server.services.HEALTHCHECK.host=localhost
+http.server.services.HEALTHCHECK.userName=username
+http.server.services.HEALTHCHECK.password=password
+http.server.services.HEALTHCHECK.restClasses=org.onap.policy.drools.healthcheck.RestMockHealthCheck
+http.server.services.HEALTHCHECK.filterClasses=org.onap.policy.drools.healthcheck.TestAafHealthCheckFilter
+http.server.services.HEALTHCHECK.port=7777
+
+http.server.services.LIVENESS.host=localhost
+http.server.services.LIVENESS.port=7776
+http.server.services.LIVENESS.restClasses=org.onap.policy.drools.healthcheck.RestMockHealthCheck
+
+http.client.services=HEALTHCHECK,LIVENESS,UNAUTH,STUCK
+
+http.client.services.HEALTHCHECK.host=localhost
+http.client.services.HEALTHCHECK.port=7777
+http.client.services.HEALTHCHECK.contextUriPath=healthcheck/test
+http.client.services.HEALTHCHECK.userName=username
+http.client.services.HEALTHCHECK.password=password
+
+http.client.services.LIVENESS.host=localhost
+http.client.services.LIVENESS.port=7776
+http.client.services.LIVENESS.contextUriPath=healthcheck/test
+
+http.client.services.UNAUTH.host=localhost
+http.client.services.UNAUTH.port=7777
+http.client.services.UNAUTH.contextUriPath=healthcheck/test
+
+http.client.services.STUCK.host=localhost
+http.client.services.STUCK.port=7776
+http.client.services.STUCK.contextUriPath=healthcheck/stuck
+
+liveness.controllers=*
+liveness.controllers.timeout=5