From 515147480c8807219dc4cdff1cd7e178757196ba Mon Sep 17 00:00:00 2001
From: jhh <jorge.hernandez-herrero@att.com>
Date: Tue, 1 Mar 2022 10:54:33 -0600
Subject: Add controllers and remote servers healthchecks

Issue-ID: POLICY-3977
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
Change-Id: I4fd4db29f99989a2ef11b08f66f28535bfd15a36
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
---
 feature-healthcheck/pom.xml                        |  10 +-
 .../feature/config/feature-healthcheck.properties  |  39 +-
 .../policy/drools/healthcheck/HealthCheck.java     | 103 +++-
 .../drools/healthcheck/HealthCheckFeature.java     |  17 +-
 .../drools/healthcheck/HealthCheckManager.java     | 475 ++++++++++++----
 .../policy/drools/healthcheck/RestHealthCheck.java | 131 ++++-
 .../drools/healthcheck/HealthCheckFeatureTest.java | 247 ++++-----
 .../drools/healthcheck/HealthCheckManagerTest.java | 596 +++++++++++++++++++++
 .../policy/drools/healthcheck/HealthCheckTest.java | 283 +---------
 .../drools/healthcheck/RestHealthCheckTest.java    | 181 +++++++
 .../drools/healthcheck/RestMockHealthCheck.java    |  20 +-
 feature-healthcheck/src/test/resources/echo.drl    |  35 ++
 .../src/test/resources/echo.kmodule                |  26 +
 feature-healthcheck/src/test/resources/echo.pom    |  31 ++
 .../test/resources/feature-healthcheck.properties  |  55 ++
 .../server/restful/RestLifecycleManagerTest.java   |   1 -
 16 files changed, 1708 insertions(+), 542 deletions(-)
 create mode 100644 feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
 create mode 100644 feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestHealthCheckTest.java
 create mode 100644 feature-healthcheck/src/test/resources/echo.drl
 create mode 100644 feature-healthcheck/src/test/resources/echo.kmodule
 create mode 100644 feature-healthcheck/src/test/resources/echo.pom
 create mode 100644 feature-healthcheck/src/test/resources/feature-healthcheck.properties

diff --git a/feature-healthcheck/pom.xml b/feature-healthcheck/pom.xml
index ec547793..8ef99173 100644
--- a/feature-healthcheck/pom.xml
+++ b/feature-healthcheck/pom.xml
@@ -1,8 +1,8 @@
 <!--
   ============LICENSE_START=======================================================
-  ONAP Policy Engine - Drools PDP
+  ONAP
   ================================================================================
-  Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
+  Copyright (C) 2017-2022 AT&T Intellectual Property. All rights reserved.
   ================================================================================
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -66,7 +66,6 @@
                         </goals>
                         <phase>prepare-package</phase>
                         <configuration>
-                            <transitive>false</transitive>
                             <outputDirectory>${project.build.directory}/assembly/lib</outputDirectory>
                             <overWriteReleases>false</overWriteReleases>
                             <overWriteSnapshots>true</overWriteSnapshots>
@@ -112,6 +111,11 @@
             <artifactId>powermock-api-mockito2</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.awaitility</groupId>
+            <artifactId>awaitility</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/feature-healthcheck/src/main/feature/config/feature-healthcheck.properties b/feature-healthcheck/src/main/feature/config/feature-healthcheck.properties
index 7739c6e7..de4b8ace 100644
--- a/feature-healthcheck/src/main/feature/config/feature-healthcheck.properties
+++ b/feature-healthcheck/src/main/feature/config/feature-healthcheck.properties
@@ -1,8 +1,8 @@
-###
+#
 # ============LICENSE_START=======================================================
-# feature-healthcheck
+# ONAP
 # ================================================================================
-# Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
+# Copyright (C) 2017-2019,2022 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,9 +16,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============LICENSE_END=========================================================
-###
+#
+
+http.server.services=HEALTHCHECK,LIVENESS
 
-http.server.services=HEALTHCHECK
 http.server.services.HEALTHCHECK.host=0.0.0.0
 http.server.services.HEALTHCHECK.port=6969
 http.server.services.HEALTHCHECK.restClasses=org.onap.policy.drools.healthcheck.RestHealthCheck
@@ -29,3 +30,31 @@ http.server.services.HEALTHCHECK.password=${envd:HEALTHCHECK_PASSWORD}
 http.server.services.HEALTHCHECK.https=${envd:HTTP_SERVER_HTTPS:false}
 http.server.services.HEALTHCHECK.aaf=${envd:AAF:false}
 http.server.services.HEALTHCHECK.serialization.provider=org.onap.policy.common.gson.JacksonHandler,org.onap.policy.common.endpoints.http.server.YamlJacksonHandler
+
+http.server.services.LIVENESS.host=localhost
+http.server.services.LIVENESS.port=6968
+http.server.services.LIVENESS.restClasses=org.onap.policy.drools.healthcheck.RestHealthCheck
+http.server.services.LIVENESS.managed=false
+http.server.services.LIVENESS.swagger=true
+http.server.services.LIVENESS.serialization.provider=org.onap.policy.common.gson.JacksonHandler,org.onap.policy.common.endpoints.http.server.YamlJacksonHandler
+
+http.client.services=PAP,PDPX
+
+http.client.services.PAP.host=${envd:PAP_HOST}
+http.client.services.PAP.port=6969
+http.client.services.PAP.userName=${envd:PAP_USERNAME}
+http.client.services.PAP.password=${envd:PAP_PASSWORD}
+http.client.services.PAP.https=${envd:HTTP_SERVER_HTTPS:false}
+http.client.services.PAP.managed=true
+http.client.services.PAP.contextUriPath=policy/pap/v1/healthcheck
+
+http.client.services.PDPX.host=${envd:PDP_HOST}
+http.client.services.PDPX.port=6969
+http.client.services.PDPX.userName=${envd:PDP_USERNAME}
+http.client.services.PDPX.password=${envd:PDP_PASSWORD}
+http.client.services.PDPX.https=${envd:HTTP_SERVER_HTTPS}
+http.client.services.PDPX.managed=true
+http.client.services.PDPX.contextUriPath=policy/pdpx/v1/healthcheck"
+
+liveness.controllers=${envd:LIVENESS_CONTROLLERS}
+liveness.controllers.timeout=${envd:LIVENESS_TIMEOUT_SECONDS:10}
\ No newline at end of file
diff --git a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheck.java b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheck.java
index de00df88..06331bcc 100644
--- a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheck.java
+++ b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheck.java
@@ -1,8 +1,8 @@
 /*
  * ============LICENSE_START=======================================================
- * feature-healthcheck
+ * ONAP
  * ================================================================================
- * Copyright (C) 2017-2019, 2021 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2019, 2021-2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,10 +22,11 @@ package org.onap.policy.drools.healthcheck;
 
 import java.util.ArrayList;
 import java.util.List;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.ToString;
+import lombok.Data;
+import lombok.NoArgsConstructor;
 import org.onap.policy.common.capabilities.Startable;
+import org.onap.policy.common.endpoints.http.client.HttpClient;
+import org.onap.policy.drools.system.PolicyController;
 
 /**
  * Healthcheck.
@@ -35,10 +36,9 @@ public interface HealthCheck extends Startable {
     /**
      * Healthcheck Report.
      */
-    @Getter
-    @Setter
-    @ToString
-    public static class Report {
+    @Data
+    @NoArgsConstructor
+    class Report {
         /**
          * Named Entity in the report.
          */
@@ -57,29 +57,98 @@ public interface HealthCheck extends Startable {
         /**
          * return code.
          */
-        private int code;
+        private long code;
+
+        /**
+         * start time.
+         */
+        private long startTime = System.currentTimeMillis();
+
+        /**
+         * end time.
+         */
+        private long endTime;
+
+        /**
+         * elapsed time.
+         */
+        private long elapsedTime;
 
         /**
          * Message from remote entity.
          */
         private String message;
+
+
+        public Report(Report report) {
+            this.startTime = report.startTime;
+            this.code = report.code;
+            this.elapsedTime = report.elapsedTime;
+            this.endTime = report.endTime;
+            this.healthy = report.healthy;
+            this.message = report.message;
+            this.name = report.name;
+            this.url = report.url;
+        }
+
+        public Report setEndTime() {
+            setEndTime(System.currentTimeMillis());
+            setElapsedTime(endTime - startTime);
+            return this;
+        }
     }
 
     /**
      * Report aggregation.
      */
-    @Getter
-    @Setter
-    @ToString
-    public static class Reports {
+    @Data
+    class Reports {
         private boolean healthy;
+        private final long startTime = System.currentTimeMillis();
+        private long endTime;
+        private long elapsedTime;
         private List<Report> details = new ArrayList<>();
+
+        public Reports setEndTime() {
+            this.endTime = System.currentTimeMillis();
+            this.elapsedTime = this.endTime - this.startTime;
+            return this;
+        }
     }
 
     /**
-     * Perform a healthcheck.
-     *
-     * @return a report
+     * Process engine open status.
+     */
+    void open();
+
+    /**
+     *  System healthcheck.
      */
     Reports healthCheck();
+
+    /**
+     * Engine only healthcheck.
+     */
+    Reports engineHealthcheck();
+
+    /**
+     * Controllers only healthcheck.
+     */
+    Reports controllerHealthcheck();
+
+    /**
+     * Healthcheck on a controller.
+     */
+    Reports controllerHealthcheck(PolicyController controller);
+
+    /**
+     * HTTP Clients only healthcheck.
+     */
+    Reports clientHealthcheck();
+
+    /**
+     * Healthcheck on an HTTP Client.
+     */
+    Reports clientHealthcheck(HttpClient client);
+
 }
diff --git a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckFeature.java b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckFeature.java
index 961f4a1d..5da134da 100644
--- a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckFeature.java
+++ b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckFeature.java
@@ -1,8 +1,8 @@
 /*-
  * ============LICENSE_START=======================================================
- * feature-healthcheck
+ * ONAP
  * ================================================================================
- * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2019, 2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -33,7 +33,7 @@ public class HealthCheckFeature implements PolicyEngineFeatureApi {
     /**
      * Logger.
      */
-    private static Logger logger = LoggerFactory.getLogger(HealthCheckFeature.class);
+    private static final Logger logger = LoggerFactory.getLogger(HealthCheckFeature.class);
 
     /**
      * Properties Configuration Name.
@@ -56,6 +56,17 @@ public class HealthCheckFeature implements PolicyEngineFeatureApi {
         return false;
     }
 
+    @Override
+    public boolean afterOpen(PolicyEngine engine) {
+        try {
+            getManager().open();
+        } catch (IllegalStateException e) {
+            logger.error("Healthcheck Monitor cannot be opened", e);
+        }
+
+        return false;
+    }
+
     @Override
     public boolean afterShutdown(PolicyEngine engine) {
         try {
diff --git a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
index 8bbd19ad..b310168d 100644
--- a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
+++ b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP
  * ================================================================================
- * Copyright (C) 2019-2021 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2019-2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,20 +20,36 @@
 
 package org.onap.policy.drools.healthcheck;
 
+import com.google.common.base.Strings;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import javax.ws.rs.core.Response;
 import lombok.AccessLevel;
 import lombok.Getter;
-import lombok.ToString;
+import lombok.NonNull;
+import lombok.Setter;
+import org.apache.commons.lang3.ArrayUtils;
+import org.eclipse.jetty.http.HttpStatus;
 import org.onap.policy.common.endpoints.http.client.HttpClient;
 import org.onap.policy.common.endpoints.http.client.HttpClientFactory;
 import org.onap.policy.common.endpoints.http.client.HttpClientFactoryInstance;
 import org.onap.policy.common.endpoints.http.server.HttpServletServer;
 import org.onap.policy.common.endpoints.http.server.HttpServletServerFactory;
 import org.onap.policy.common.endpoints.http.server.HttpServletServerFactoryInstance;
+import org.onap.policy.drools.controller.DroolsController;
 import org.onap.policy.drools.persistence.SystemPersistenceConstants;
+import org.onap.policy.drools.system.PolicyController;
+import org.onap.policy.drools.system.PolicyControllerConstants;
+import org.onap.policy.drools.system.PolicyControllerFactory;
 import org.onap.policy.drools.system.PolicyEngine;
 import org.onap.policy.drools.system.PolicyEngineConstants;
 import org.slf4j.Logger;
@@ -43,170 +59,415 @@ import org.slf4j.LoggerFactory;
 /**
  * Healthcheck Monitor.
  */
+
 @Getter
-@ToString
 public class HealthCheckManager implements HealthCheck {
 
+    protected static final Logger logger = LoggerFactory.getLogger(HealthCheckManager.class);
+
+    protected static final Pattern COMMA_SPACE_PATTERN = Pattern.compile("\\s*,\\s*");
+
+    protected static final String ENGINE_NAME = "PDP-D";
+    protected static final String HEALTHCHECK_SERVER = "HEALTHCHECK";  // expected healthcheck server name in config
+    protected static final String LIVENESS_SERVER = "LIVENESS";        // expected liveness server name in config
+
+    protected static final int SUCCESS_CODE = 200;
+
+    protected static final int BRAINLESS_CODE = 201;
+    protected static final String BRAINLESS_MESSAGE = "no rules configured";
+
+    protected static final String ENABLED_MESSAGE = "enabled";
+
+    protected static final int DISABLED_CODE = 400;
+    protected static final String DISABLED_MESSAGE = "disabled";
+
+    protected static final int INPROG_CODE = 100;
+    protected static final String INPROG_MESSAGE = "test in progress";
+
+    protected static final int TIMEOUT_CODE = 3000;
+    protected static final String TIMEOUT_MESSAGE = "healthcheck timeout";
+
+    protected static final int UNREACHABLE_CODE = 9000;
+    protected static final String UNREACHABLE_MESSAGE = "cannot reach component";
+
+    public static final String  UNKNOWN_ENTITY = "unknown";
+    public static final int UNKNOWN_ENTITY_CODE = 9010;
+    public static final String UNKNOWN_ENTITY_MESSAGE = "unknown entity";
+
+    protected static final long DEFAULT_TIMEOUT_SECONDS = 10;
+
     /**
-     * Logger.
+     * Healthcheck Server.
      */
-    private static Logger logger = LoggerFactory.getLogger(HealthCheckManager.class);
+    protected HttpServletServer healthcheckServer;
 
     /**
-     * Attached http servers.
+     * Liveness Server.
      */
-    protected List<HttpServletServer> servers = new ArrayList<>();
+    protected HttpServletServer livenessServer;
 
     /**
      * Attached http clients.
      */
     protected List<HttpClient> clients = new ArrayList<>();
 
+    /**
+     * Attached controllers.
+     */
+    protected List<PolicyController> controllers = new ArrayList<>();
+
     /**
      * Healthcheck configuration.
      */
     @Getter(AccessLevel.NONE)
-    @ToString.Exclude
     protected Properties healthCheckProperties = null;
 
-    /**
-     * {@inheritDoc}.
-     */
+    @Setter
+    @Getter
+    protected Long timeoutSeconds = DEFAULT_TIMEOUT_SECONDS;
+
     @Override
     public Reports healthCheck() {
-        var reports = new Reports();
-        boolean thisEngineIsAlive = getEngineManager().isAlive();
-        reports.setHealthy(thisEngineIsAlive);
-
-        var engineReport = new Report();
-        engineReport.setHealthy(thisEngineIsAlive);
-        engineReport.setName("PDP-D");
-        engineReport.setUrl("self");
-        engineReport.setCode(thisEngineIsAlive ? 200 : 500);
-        engineReport.setMessage(thisEngineIsAlive ? "alive" : "not alive");
-        reports.getDetails().add(engineReport);
-
-        for (HttpClient client : clients) {
-            var report = new Report();
-            report.setName(client.getName());
-            report.setUrl(client.getBaseUrl());
+        // get first the engine summary report for setting start time
+        var engineSummary = engineHealthcheck();
+        if (!isEngineAlive()) {
+            logger.info("controller healthchecks ignored as engine is not active");
+            return engineSummary;
+        }
+
+        CompletableFuture<Report>[] reportFutures =
+            ArrayUtils.addAll(futures(getControllers()), futures(getClients()));
+
+        return summary(engineSummary, reportFutures);
+    }
+
+    @Override
+    public Reports engineHealthcheck() {
+        /*
+         * An engine report is special as there always should be 1
+         * report at each system or component healthcheck, since it
+         * is the umbrella component.  Since it does not do IO,
+         * it is generated synchronously which is different from
+         * HTTP clients or Policy Controllers which are asynchronous
+         * with timeout safeties.
+         */
+        var summary = new Reports();
+
+        var engineReport = reportOnEngine();
+        summary.getDetails().add(engineReport);
+        summary.setHealthy(engineReport.isHealthy());
+
+        return summary.setEndTime();
+    }
+
+    @Override
+    public Reports controllerHealthcheck() {
+        if (!isEngineAlive()) {
+            logger.info("controller healthchecks ignored as engine is not active");
+            return engineHealthcheck();
+        }
+
+        CompletableFuture<Report>[] reportFutures = futures(getControllers());
+        return summary(engineHealthcheck(), reportFutures);
+    }
+
+    @Override
+    public Reports controllerHealthcheck(@NonNull PolicyController controller) {
+        /*
+         * allow individual healthchecks without consulting engine state,
+         * it could be useful for troubleshooting.
+         */
+        CompletableFuture<Report>[] reportFutures = futures(List.of(controller));
+        return summary(engineHealthcheck(), reportFutures);
+    }
+
+    @Override
+    public Reports clientHealthcheck() {
+        if (!isEngineAlive()) {
+            logger.info("client healthchecks ignored as engine is not active");
+            return engineHealthcheck();
+        }
+
+        CompletableFuture<Report>[] reportFutures = futures(getClients());
+        return summary(engineHealthcheck(), reportFutures);
+    }
+
+    @Override
+    public Reports clientHealthcheck(@NonNull HttpClient client) {
+        /*
+         * allow individual healthchecks without consulting engine state,
+         * it could be useful for troubleshooting.
+         */
+        CompletableFuture<Report>[] reportFutures = futures(List.of(client));
+        return summary(engineHealthcheck(), reportFutures);
+    }
+
+    protected Reports summary(@NonNull Reports summary, @NonNull CompletableFuture<Report>[] futures) {
+        CompletableFuture.allOf(futures).join();
+
+        Arrays.stream(futures)
+                .map(CompletableFuture::join)
+                .forEach(summary.getDetails()::add);
+
+        summary.setHealthy(summary.getDetails()
+                .stream()
+                .map(this::timeout)
+                .map(Report::isHealthy)
+                .reduce(true, Boolean::logicalAnd));
+
+        return summary.setEndTime();
+    }
+
+    protected Report timeout(Report report) {
+        if (report.getCode() == INPROG_CODE && INPROG_MESSAGE.equals(report.getMessage())) {
+            report.setHealthy(false);
+            report.setCode(TIMEOUT_CODE);
+            report.setMessage(TIMEOUT_MESSAGE);
+        }
+
+        return report;
+    }
+
+    protected <T> CompletableFuture<Report>[] futures(List<T> entities) {
+        return entities.stream()
+                .map(this::supplier)
+                .toArray(CompletableFuture[]::new);
+    }
+
+    protected Report reportOnEngine() {
+        var report = new Report();
+
+        report.setName(ENGINE_NAME);
+        report.setUrl("engine");
+        report.setHealthy(isEngineAlive());
+
+        if (isEngineAlive()) {
+            report.setCode(SUCCESS_CODE);
+            report.setMessage(ENABLED_MESSAGE);
+        } else {
+            report.setCode(DISABLED_CODE);
+            report.setMessage(DISABLED_MESSAGE);
+        }
+
+        return report.setEndTime();
+    }
+
+    protected Report reportOn(@NonNull PolicyController controller, @NonNull Report report) {
+        report.setName(controller.getName());
+
+        if (!controller.isAlive()) {
+            report.setUrl(controller.getName());
+            report.setCode(DISABLED_CODE);
+            report.setMessage(DISABLED_MESSAGE);
+            report.setHealthy(false);
+            return report.setEndTime();
+        }
+
+        DroolsController drools = controller.getDrools();
+        report.setUrl(getControllerCoordinates(drools));
+
+        if (!drools.isBrained()) {
+            report.setCode(BRAINLESS_CODE);
+            report.setMessage(BRAINLESS_MESSAGE);
             report.setHealthy(true);
-            try {
-                var response = client.get();
-                report.setCode(response.getStatus());
-                if (report.getCode() != 200) {
-                    report.setHealthy(false);
-                    reports.setHealthy(false);
-                }
-
-                report.setMessage(getHttpBody(response, client));
-            } catch (Exception e) {
-                logger.warn("{}: cannot contact http-client {}", this, client, e);
-
-                report.setHealthy(false);
-                reports.setHealthy(false);
-            }
-            reports.getDetails().add(report);
+            return report.setEndTime();
         }
-        return reports;
+
+        /*
+         * potentially blocking drools application operation
+         */
+
+        return reportOn(controller.getDrools(), report).setEndTime();
+    }
+
+    private Report reportOn(@NonNull DroolsController drools, @NonNull Report report) {
+        if (!drools.isAlive()) {
+            report.setCode(DISABLED_CODE);
+            report.setMessage(DISABLED_MESSAGE);
+            return report;
+        }
+
+        /*
+         * The code below will block in unresponsive applications.
+         */
+        long factCount = 0;
+        StringBuilder message = new StringBuilder();
+        for (String sessionName: drools.getSessionNames()) {
+            message.append("[").append(sessionName).append(":").append(getFactTypes(drools, sessionName)).append("]");
+            factCount += getFactCount(drools, sessionName);
+        }
+
+        /* success */
+
+        report.setHealthy(true);
+        report.setCode(factCount);
+        report.setMessage("" + message);
+
+        return report;
+    }
+
+    protected Report reportOn(@NonNull HttpClient client, @NonNull Report report) {
+        report.setName(client.getName());
+        report.setUrl(client.getBaseUrl());
+
+        try {
+            Response response = client.get();
+            report.setHealthy(response.getStatus() == HttpStatus.OK_200);
+            report.setCode(response.getStatus());
+            report.setMessage(response.getStatusInfo().getReasonPhrase());
+        } catch (Exception e) {
+            report.setHealthy(false);
+            report.setCode(UNREACHABLE_CODE);
+            report.setMessage(UNREACHABLE_MESSAGE);
+            logger.info("{}: cannot contact http-client {}", this, client.getName(), e);
+        }
+
+        return report.setEndTime();
+    }
+
+    private <T> Supplier<Report> createSupplier(@NonNull T entity, @NonNull Report report) {
+        if (entity instanceof PolicyController) {
+            return () -> reportOn((PolicyController) entity, report);
+        } else if (entity instanceof HttpClient) {
+            return () -> reportOn((HttpClient) entity, report);
+        } else {
+            return () -> reportOnUnknown(entity, report);
+        }
+    }
+
+    private Report reportOnUnknown(Object o, Report report) {
+        report.setName(UNKNOWN_ENTITY);
+        report.setCode(UNKNOWN_ENTITY_CODE);
+        report.setMessage(UNKNOWN_ENTITY_MESSAGE);
+        report.setHealthy(false);
+        report.setUrl(o.getClass().getName());
+        return report.setEndTime();
+    }
+
+    protected <T> CompletableFuture<Report> supplier(T entity) {
+        var report = new Report();
+        report.setHealthy(false);
+        report.setCode(INPROG_CODE);
+        report.setMessage(INPROG_MESSAGE);
+
+        return CompletableFuture
+                       .supplyAsync(createSupplier(entity, report))
+                       .completeOnTimeout(report, getTimeoutSeconds(), TimeUnit.SECONDS)
+                       .thenApply(HealthCheck.Report::setEndTime);
+    }
+
+    private String getControllerCoordinates(DroolsController drools) {
+        return drools.getGroupId() + ":" + drools.getArtifactId() + ":" + drools.getVersion();
+    }
+
+    protected long getFactCount(DroolsController drools, String sessionName) {
+        return drools.factCount(sessionName);
+    }
+
+    protected Map<String, Integer> getFactTypes(DroolsController drools, String sessionName) {
+        return drools.factClassNames(sessionName);
     }
 
-    /**
-     * {@inheritDoc}.
-     */
     @Override
     public boolean start() {
-
         try {
-            this.healthCheckProperties = getPersistentProperties(HealthCheckFeature.CONFIGURATION_PROPERTIES_NAME);
-            this.servers = getServerFactory().build(healthCheckProperties);
+            this.healthCheckProperties = getPersistentProperties();
+            Map<String, HttpServletServer> servers =
+                    getServerFactory().build(healthCheckProperties).stream()
+                        .collect(Collectors.toMap(HttpServletServer::getName, Function.identity()));
+
+            this.healthcheckServer = servers.get(HEALTHCHECK_SERVER);
+            this.livenessServer = servers.get(LIVENESS_SERVER);
+
+            setTimeoutSeconds(
+                    Long.valueOf(this.healthCheckProperties
+                            .getProperty("liveness.controllers.timeout",
+                                    "" + DEFAULT_TIMEOUT_SECONDS)));
+
             this.clients = getClientFactory().build(healthCheckProperties);
 
-            for (HttpServletServer server : servers) {
-                if (server.isAaf()) {
-                    server.addFilterClass(null, AafHealthCheckFilter.class.getName());
-                }
-                startServer(server);
-            }
+            return startHealthcheckServer();
         } catch (Exception e) {
-            logger.warn("{}: cannot start", this, e);
+            logger.warn("{}: cannot start", HEALTHCHECK_SERVER, e);
             return false;
         }
-
-        return true;
     }
 
-    /**
-     * {@inheritDoc}.
-     */
     @Override
-    public boolean stop() {
+    public void open() {
+        if (this.livenessServer != null) {
+            startServer(this.livenessServer);
+        }
 
-        for (HttpServletServer server : servers) {
-            try {
-                server.stop();
-            } catch (Exception e) {
-                logger.warn("{}: cannot stop http-server {}", this, server, e);
-            }
+        String controllerNames = this.healthCheckProperties.getProperty("liveness.controllers");
+        if (Strings.isNullOrEmpty(controllerNames)) {
+            logger.info("no controllers to live check");
+            return;
+        }
+
+        if ("*".equals(controllerNames)) {
+            // monitor all controllers
+            this.controllers = getControllerFactory().inventory();
+            return;
         }
 
-        for (HttpClient client : clients) {
+        for (String controllerName : COMMA_SPACE_PATTERN.split(controllerNames)) {
             try {
-                client.stop();
-            } catch (Exception e) {
-                logger.warn("{}: cannot stop http-client {}", this, client, e);
+                this.controllers.add(getControllerFactory().get(controllerName));
+            } catch (RuntimeException rex) {
+                logger.warn("cannot get controller {}", controllerName);
             }
         }
+    }
+
+    @Override
+    public boolean stop() {
+        if (this.healthcheckServer != null) {
+            this.healthcheckServer.stop();
+        }
+
+        if (this.livenessServer != null) {
+            this.livenessServer.stop();
+        }
+
+        for (HttpClient client : getClients()) {
+            logger.warn("{}: cannot stop http-client", client.getName());
+            client.stop();
+        }
 
         return true;
     }
 
-    /**
-     * {@inheritDoc}.
-     */
     @Override
     public void shutdown() {
         this.stop();
     }
 
-    /**
-     * {@inheritDoc}.
-     */
     @Override
     public synchronized boolean isAlive() {
         return this.healthCheckProperties != null;
     }
 
-    /**
-     * Gets the body from the response.
-     *
-     * @param response response from which to get the body
-     * @param client HTTP client from which the response was received
-     * @return the response body
-     */
-    public String getHttpBody(Response response, HttpClient client) {
+    protected boolean startHealthcheckServer() {
+        if (this.healthcheckServer == null) {
+            logger.warn("no {} server found", HEALTHCHECK_SERVER);
+            return false;
+        }
 
-        String body = null;
-        try {
-            body = HttpClient.getBody(response, String.class);
-        } catch (Exception e) {
-            logger.info("{}: cannot get body from http-client {}", this, client, e);
+        if (this.healthcheckServer.isAaf()) {
+            this.healthcheckServer.addFilterClass(null, AafHealthCheckFilter.class.getName());
         }
 
-        return body;
+        return startServer(this.healthcheckServer);
     }
 
-    /**
-     * Starts an HTTP server.
-     *
-     * @param server server to be started
-     */
-    public void startServer(HttpServletServer server) {
+    protected boolean startServer(HttpServletServer server) {
         try {
-            server.start();
+            return server.start();
         } catch (Exception e) {
-            logger.warn("{}: cannot start http-server {}", this, server, e);
+            logger.warn("cannot start http-server {}", server.getName(), e);
         }
+        return false;
     }
 
     // the following methods may be overridden by junit tests
@@ -215,6 +476,10 @@ public class HealthCheckManager implements HealthCheck {
         return PolicyEngineConstants.getManager();
     }
 
+    protected boolean isEngineAlive() {
+        return getEngineManager().isAlive();
+    }
+
     protected HttpServletServerFactory getServerFactory() {
         return HttpServletServerFactoryInstance.getServerFactory();
     }
@@ -223,7 +488,11 @@ public class HealthCheckManager implements HealthCheck {
         return HttpClientFactoryInstance.getClientFactory();
     }
 
-    protected Properties getPersistentProperties(String propertyName) {
-        return SystemPersistenceConstants.getManager().getProperties(propertyName);
+    protected PolicyControllerFactory getControllerFactory() {
+        return PolicyControllerConstants.getFactory();
+    }
+
+    protected Properties getPersistentProperties() {
+        return SystemPersistenceConstants.getManager().getProperties(HealthCheckFeature.CONFIGURATION_PROPERTIES_NAME);
     }
 }
diff --git a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/RestHealthCheck.java b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/RestHealthCheck.java
index e323d405..5b36c5a3 100644
--- a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/RestHealthCheck.java
+++ b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/RestHealthCheck.java
@@ -1,8 +1,8 @@
 /*-
  * ============LICENSE_START=======================================================
- * feature-healthcheck
+ * ONAP
  * ================================================================================
- * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2019, 2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,17 +22,26 @@ package org.onap.policy.drools.healthcheck;
 
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
 import io.swagger.annotations.Info;
 import io.swagger.annotations.SwaggerDefinition;
 import io.swagger.annotations.Tag;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactory;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactoryInstance;
 import org.onap.policy.common.endpoints.http.server.YamlMessageBodyHandler;
 import org.onap.policy.drools.healthcheck.HealthCheck.Reports;
+import org.onap.policy.drools.system.PolicyControllerConstants;
+import org.onap.policy.drools.system.PolicyControllerFactory;
 
+/**
+ * REST Healthcheck JAX-RS.
+ */
 @Path("/")
 @Api
 @Produces({MediaType.APPLICATION_JSON, YamlMessageBodyHandler.APPLICATION_YAML})
@@ -51,6 +60,10 @@ import org.onap.policy.drools.healthcheck.HealthCheck.Reports;
     )
 public class RestHealthCheck {
 
+    /**
+     * System healthcheck per configuration.
+     */
+
     @GET
     @Path("healthcheck")
     @ApiOperation(
@@ -60,17 +73,119 @@ public class RestHealthCheck {
             response = Reports.class
             )
     public Response healthcheck() {
-        return Response.status(Response.Status.OK).entity(HealthCheckConstants.getManager().healthCheck()).build();
+        var summary = getHealthcheckManager().healthCheck();
+        return getResponse(summary);
     }
 
+    /**
+     * Engine Healthcheck.
+     */
+
     @GET
-    @Path("healthcheck/configuration")
+    @Path("healthcheck/engine")
     @ApiOperation(
-            value = "Configuration",
-            notes = "Provides the Healthcheck server configuration and monitored REST clients",
+            value = "Engine Healthcheck",
+            notes = "Provides a Healthcheck on the engine",
             response = HealthCheck.class
-            )
-    public HealthCheck configuration() {
+    )
+    public Response engine() {
+        var summary = getHealthcheckManager().engineHealthcheck();
+        return getResponse(summary);
+    }
+
+    /**
+     * Healthcheck on the controllers.
+     */
+
+    @GET
+    @Path("healthcheck/controllers")
+    @ApiOperation(
+            value = "Controllers Healthcheck",
+            notes = "Provides a Healthcheck on the configured controllers",
+            response = Reports.class
+    )
+    public Response controllers() {
+        var summary = getHealthcheckManager().controllerHealthcheck();
+        return getResponse(summary);
+    }
+
+    /**
+     * Healthcheck a controller.
+     */
+
+    @GET
+    @Path("healthcheck/controllers/{controllerName}")
+    @ApiOperation(
+            value = "Controller Healthcheck",
+            notes = "Provides a Healthcheck on a configured controller",
+            response = Reports.class
+    )
+    public Response controllers(@ApiParam(value = "Policy Controller Name",
+            required = true) @PathParam("controllerName") String controllerName) {
+        try {
+            var controller = getControllerFactory().get(controllerName);
+            var summary = getHealthcheckManager().controllerHealthcheck(controller);
+            return getResponse(summary);
+        } catch (final IllegalArgumentException e) {
+            return Response.status(Response.Status.NOT_FOUND).build();
+        } catch (final IllegalStateException e) {
+            return Response.status(Response.Status.NOT_ACCEPTABLE).build();
+        }
+    }
+
+    /**
+     * Healthcheck on the Http Clients per configuration.
+     */
+
+    @GET
+    @Path("healthcheck/clients")
+    @ApiOperation(
+            value = "Http Clients Healthcheck",
+            notes = "Provides a Healthcheck on the configured HTTP clients",
+            response = Reports.class
+    )
+    public Response clients() {
+        var summary = getHealthcheckManager().clientHealthcheck();
+        return getResponse(summary);
+    }
+
+    /**
+     * Healthcheck a on a Http Client.
+     */
+
+    @GET
+    @Path("healthcheck/clients/{clientName}")
+    @ApiOperation(
+            value = "Http Client Healthcheck",
+            notes = "Provides a Healthcheck on a configured HTTP client",
+            response = Reports.class
+    )
+    public Response clients(@ApiParam(value = "Http Client Name",
+            required = true) @PathParam("clientName") String clientName) {
+        try {
+            var client = getClientFactory().get(clientName);
+            var summary = getHealthcheckManager().clientHealthcheck(client);
+            return getResponse(summary);
+        } catch (final IllegalArgumentException e) {
+            return Response.status(Response.Status.NOT_FOUND).build();
+        }
+    }
+
+    protected Response getResponse(Reports summary) {
+        return Response.status(summary.isHealthy() ? Response.Status.OK : Response.Status.SERVICE_UNAVAILABLE)
+                .entity(summary).build();
+    }
+
+    protected HttpClientFactory getClientFactory() {
+        return HttpClientFactoryInstance.getClientFactory();
+    }
+
+    protected PolicyControllerFactory getControllerFactory() {
+        return PolicyControllerConstants.getFactory();
+    }
+
+    protected HealthCheck getHealthcheckManager() {
         return HealthCheckConstants.getManager();
     }
+
 }
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckFeatureTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckFeatureTest.java
index 4d93af23..0567595f 100644
--- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckFeatureTest.java
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckFeatureTest.java
@@ -1,8 +1,8 @@
 /*-
  * ============LICENSE_START=======================================================
- * feature-healthcheck
+ * ONAP
  * ================================================================================
- * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2019,2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,137 +23,147 @@ package org.onap.policy.drools.healthcheck;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
-import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Properties;
+import org.eclipse.jetty.http.HttpStatus;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.onap.policy.common.endpoints.properties.PolicyEndPointProperties;
+import org.kie.api.builder.ReleaseId;
+import org.mockito.AdditionalAnswers;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactoryInstance;
+import org.onap.policy.common.endpoints.http.server.HttpServletServerFactoryInstance;
+import org.onap.policy.common.utils.logging.LoggerUtils;
 import org.onap.policy.common.utils.network.NetworkUtil;
-import org.onap.policy.drools.healthcheck.HealthCheck.Report;
 import org.onap.policy.drools.healthcheck.HealthCheck.Reports;
 import org.onap.policy.drools.persistence.SystemPersistenceConstants;
+import org.onap.policy.drools.properties.DroolsPropertyConstants;
+import org.onap.policy.drools.system.PolicyController;
+import org.onap.policy.drools.system.PolicyControllerConstants;
 import org.onap.policy.drools.system.PolicyEngineConstants;
+import org.onap.policy.drools.util.KieUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class HealthCheckFeatureTest {
 
-    /**
-     * Healthcheck Configuration File.
-     */
-    private static final String HEALTH_CHECK_PROPERTIES_FILE = "feature-healthcheck.properties";
-
-    private static final Path healthCheckPropsPath =
-                    Paths.get(SystemPersistenceConstants.getManager().getConfigurationPath().toString(),
-                                    HEALTH_CHECK_PROPERTIES_FILE);
-
-    private static final Path healthCheckPropsBackupPath =
-                    Paths.get(SystemPersistenceConstants.getManager().getConfigurationPath().toString(),
-                                    HEALTH_CHECK_PROPERTIES_FILE + ".bak");
-
+    private static final Logger logger = LoggerFactory.getLogger(HealthCheckFeatureTest.class);
     private static final String EXPECTED = "expected exception";
 
-
-    /**
-     * logger.
-     */
-    private static Logger logger = LoggerFactory.getLogger(HealthCheckFeatureTest.class);
-
-    private static Properties httpProperties = new Properties();
-
     /**
      * Set up.
      */
     @BeforeClass
-    public static void setup() {
-
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES, "HEALTHCHECK");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_HOST_SUFFIX, "localhost");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_PORT_SUFFIX, "7777");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_AUTH_USERNAME_SUFFIX, "username");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_AUTH_PASSWORD_SUFFIX, "password");
-        httpProperties.setProperty(
-                        PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                                        + PolicyEndPointProperties.PROPERTY_HTTP_REST_CLASSES_SUFFIX,
-                        org.onap.policy.drools.healthcheck.RestMockHealthCheck.class.getName());
-        httpProperties.setProperty(
-                        PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                                        + PolicyEndPointProperties.PROPERTY_HTTP_FILTER_CLASSES_SUFFIX,
-                        org.onap.policy.drools.healthcheck.TestAafHealthCheckFilter.class.getName());
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_MANAGED_SUFFIX, "true");
-
-
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES, "HEALTHCHECK");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_HOST_SUFFIX, "localhost");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_PORT_SUFFIX, "7777");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_URL_SUFFIX, "healthcheck/test");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_HTTPS_SUFFIX, "false");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_AUTH_USERNAME_SUFFIX, "username");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_HTTP_AUTH_PASSWORD_SUFFIX, "password");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + "." + "HEALTHCHECK"
-                        + PolicyEndPointProperties.PROPERTY_MANAGED_SUFFIX, "true");
-
-        configDirSetup();
-
+    public static void setup() throws IOException {
+        SystemPersistenceConstants.getManager().setConfigurationDir("target/test-classes");
+
+        LoggerUtils.setLevel("org.onap.policy.common.endpoints", "WARN");
+        LoggerUtils.setLevel("org.eclipse", "ERROR");
+        LoggerUtils.setLevel("org.onap.policy.drools.healthcheck", "DEBUG");
+        LoggerUtils.setLevel("ROOT", "INFO");
+
+        ReleaseId coords = KieUtils.installArtifact(Paths.get("src/test/resources/echo.kmodule").toFile(),
+            Paths.get("src/test/resources/echo.pom").toFile(),
+            "src/main/resources/kbecho/org/onap/policy/drools/healthcheck/",
+            List.of(Paths.get("src/test/resources/echo.drl").toFile()));
+
+        Properties controllerProps = new Properties();
+        controllerProps.put(DroolsPropertyConstants.PROPERTY_CONTROLLER_NAME, "echo");
+        controllerProps.put(DroolsPropertyConstants.RULES_GROUPID, coords.getGroupId());
+        controllerProps.put(DroolsPropertyConstants.RULES_ARTIFACTID, coords.getArtifactId());
+        controllerProps.put(DroolsPropertyConstants.RULES_VERSION, coords.getVersion());
+
+        PolicyController controller = PolicyControllerConstants.getFactory().build("echo", controllerProps);
+        controller.start();
     }
 
     /**
      * Tear down.
      */
     @AfterClass
-    public static void tearDown() {
-        logger.info("-- tearDown() --");
-
-        configDirCleanup();
+    public static void teardown() {
+        PolicyControllerConstants.getFactory().destroy();
+        HttpClientFactoryInstance.getClientFactory().destroy();
+        HttpServletServerFactoryInstance.getServerFactory().destroy();
     }
 
     @Test
-    public void test() throws IOException, InterruptedException {
+    public void test() throws InterruptedException {
+        var manager = spy(HealthCheckManager.class);
+        var feature = new HealthCheckFeatureImpl(manager);
+        when(manager.isEngineAlive()).thenReturn(true);
 
-        HealthCheckFeature feature = new HealthCheckFeature();
         feature.afterStart(PolicyEngineConstants.getManager());
+        feature.afterOpen(PolicyEngineConstants.getManager());
 
-        if (!NetworkUtil.isTcpPortOpen("localhost", 7777, 5, 10000L)) {
-            throw new IllegalStateException("cannot connect to port " + 7777);
-        }
+        checkOpen(7777);
+        checkOpen(7776);
 
-        Reports reports = HealthCheckConstants.getManager().healthCheck();
+        var reports = healthcheck(manager);
+        serverChecks(reports);
+        checkReports(reports, List.of("STUCK"),
+                HttpStatus.OK_200, HttpStatus.getMessage(200));
+        checkReports(reports, List.of("echo"), 1, "[echo:{java.lang.String=1}]");
 
-        assertTrue(reports.getDetails().size() > 0);
+        /* mock controller and clients stuck */
 
-        for (Report rpt : reports.getDetails()) {
-            if ("HEALTHCHECK".equals(rpt.getName())) {
-                assertTrue(rpt.isHealthy());
-                assertEquals(200, rpt.getCode());
-                assertEquals("All Alive", rpt.getMessage());
-                break;
-            }
-        }
+        RestMockHealthCheck.stuck = true;   // make the server named STUCK unresponsive
+        doAnswer(AdditionalAnswers
+                .answersWithDelay((manager.getTimeoutSeconds() + 2) * 1000L,
+                        invocationOnMock -> new HashMap<String, Integer>()))
+                .when(manager).getFactTypes(any(), any());
+
+        reports = healthcheck(manager);
+        RestMockHealthCheck.stuck = false;  // unstuck the server named STUCK
+
+        serverChecks(reports);
+        checkReports(reports, List.of("STUCK"),
+                HealthCheckManager.TIMEOUT_CODE, HealthCheckManager.TIMEOUT_MESSAGE);
+
+        assertTrue(RestMockHealthCheck.WAIT * 1000 > HealthCheckManagerTest.select(reports, "STUCK",
+                        HealthCheckManager.TIMEOUT_CODE, HealthCheckManager.TIMEOUT_MESSAGE)
+                .get(0).getElapsedTime());
 
         feature.afterShutdown(PolicyEngineConstants.getManager());
+    }
+
+    private void checkReports(Reports reports, List<String> reportNames, int code, String message) {
+        reportNames
+                .forEach(name -> assertEquals(1,
+                        HealthCheckManagerTest.select(reports, name, code, message).size()));
+    }
+
+    private Reports healthcheck(HealthCheck manager) {
+        var reports = manager.healthCheck();
+        logger.info("{}", reports);
+        return reports;
+    }
+
+    private void checkOpen(int port) throws InterruptedException {
+        if (!NetworkUtil.isTcpPortOpen("localhost", port, 5, 10000L)) {
+            throw new IllegalStateException("cannot connect to port " + port);
+        }
+    }
 
+    private void serverChecks(Reports reports) {
+        checkReports(reports, List.of("HEALTHCHECK", "LIVENESS"),
+                HttpStatus.OK_200, HttpStatus.getMessage(200));
+        checkReports(reports, List.of("UNAUTH"),
+                HttpStatus.UNAUTHORIZED_401, HttpStatus.getMessage(401));
+        checkReports(reports, List.of(HealthCheckManager.ENGINE_NAME),
+                HealthCheckManager.SUCCESS_CODE, HealthCheckManager.ENABLED_MESSAGE);
     }
 
     @Test
@@ -176,6 +186,22 @@ public class HealthCheckFeatureTest {
         assertFalse(feature.afterStart(null));
     }
 
+    @Test
+    public void testAfterOpen() {
+        HealthCheck checker = mock(HealthCheck.class);
+        HealthCheckFeature feature = new HealthCheckFeatureImpl(checker);
+
+        // without exception
+        assertFalse(feature.afterOpen(null));
+        verify(checker).open();
+        verify(checker, never()).stop();
+
+        // with exception
+        doThrow(new IllegalStateException(EXPECTED)).when(checker).open();
+        assertFalse(feature.afterOpen(null));
+
+    }
+
     @Test
     public void testAfterShutdown() {
         HealthCheck checker = mock(HealthCheck.class);
@@ -191,49 +217,6 @@ public class HealthCheckFeatureTest {
         assertFalse(feature.afterShutdown(null));
     }
 
-
-    /**
-     * setup up config directory.
-     */
-    private static void configDirSetup() {
-
-        File origPropsFile = new File(healthCheckPropsPath.toString());
-        File backupPropsFile = new File(healthCheckPropsBackupPath.toString());
-        Path configDir = Paths.get(SystemPersistenceConstants.DEFAULT_CONFIGURATION_DIR);
-
-        try {
-
-            if (Files.notExists(configDir)) {
-                Files.createDirectories(configDir);
-            }
-
-            Files.deleteIfExists(healthCheckPropsBackupPath);
-            origPropsFile.renameTo(backupPropsFile);
-
-            FileWriter writer = new FileWriter(origPropsFile);
-            httpProperties.store(writer, "Machine created healthcheck-feature Properties");
-
-        } catch (final Exception e) {
-            logger.info("Problem cleaning {}", healthCheckPropsPath, e);
-        }
-    }
-
-    /**
-     * cleanup up config directory.
-     */
-    private static void configDirCleanup() {
-
-        File origPropsFile = new File(healthCheckPropsBackupPath.toString());
-        File backupPropsFile = new File(healthCheckPropsPath.toString());
-
-        try {
-            Files.deleteIfExists(healthCheckPropsPath);
-            origPropsFile.renameTo(backupPropsFile);
-        } catch (final Exception e) {
-            logger.info("Problem cleaning {}", healthCheckPropsPath, e);
-        }
-    }
-
     /**
      * Feature that returns a particular monitor.
      */
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
new file mode 100644
index 00000000..73d70da9
--- /dev/null
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
@@ -0,0 +1,596 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.healthcheck;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.net.HttpURLConnection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import javax.ws.rs.core.Response;
+import org.eclipse.jetty.http.HttpStatus;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.AdditionalAnswers;
+import org.onap.policy.common.endpoints.http.client.HttpClient;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactory;
+import org.onap.policy.common.endpoints.http.server.HttpServletServer;
+import org.onap.policy.common.endpoints.http.server.HttpServletServerFactory;
+import org.onap.policy.drools.controller.DroolsController;
+import org.onap.policy.drools.healthcheck.HealthCheck.Report;
+import org.onap.policy.drools.healthcheck.HealthCheck.Reports;
+import org.onap.policy.drools.system.PolicyController;
+import org.onap.policy.drools.system.PolicyControllerFactory;
+import org.onap.policy.drools.system.PolicyEngine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HealthCheckManagerTest {
+
+    private static final Logger logger = LoggerFactory.getLogger(HealthCheckManagerTest.class);
+
+    protected static List<Report> select(Reports reports, String name, long code, String message) {
+        return reports.getDetails().stream()
+                .filter(report -> name.equals(report.getName()))
+                .filter(report -> report.getCode() == code)
+                .filter(report -> message.equals(report.getMessage()))
+                .collect(Collectors.toList());
+    }
+
+    private static final String RPT_MSG = "report-message";
+    private static final String RPT_NAME = "report-name";
+    private static final String EXPECTED = "expected exception";
+
+    private static final String CLIENT_NAME1 = "name-a";
+    private static final String CLIENT_URL1 = "url-a";
+    private static final String CLIENT_NAME2 = "name-b";
+    private static final String CLIENT_URL2 = "url-b";
+    private static final String CLIENT_NAME3 = "name-c";
+    private static final String CLIENT_URL3 = "url-c";
+
+    private Properties properties;
+    private HttpServletServerFactory servletFactory;
+    private HttpServletServer server1;
+    private HttpServletServer server2;
+    private HttpClientFactory clientFactory;
+    private HttpClient client1;
+    private HttpClient client2;
+    private HttpClient client3;
+
+    private PolicyControllerFactory controllerFactory;
+    private PolicyController controller1;
+    private PolicyController controller2;
+    private DroolsController drools1;
+    private DroolsController drools2;
+
+    private PolicyEngine engineMgr;
+    private HealthCheckManager monitor;
+
+    /**
+     * Initializes the object to be tested.
+     */
+    @Before
+    public void setUp() throws Exception {
+        properties = new Properties();
+        mocks();
+
+        List<HttpServletServer> servers = Arrays.asList(server1, server2);
+        List<HttpClient> clients = Arrays.asList(client1, client2, client3);
+
+        whenClients();
+
+        when(servletFactory.build(properties)).thenReturn(servers);
+        when(clientFactory.build(properties)).thenReturn(clients);
+
+        whenControllers();
+
+        when(engineMgr.isAlive()).thenReturn(true);
+
+        monitor = new HealthCheckMonitorImpl();
+    }
+
+    private void whenControllers() {
+        when(drools1.getGroupId()).thenReturn("1");
+        when(drools2.getGroupId()).thenReturn("2");
+
+        when(drools1.getArtifactId()).thenReturn("1");
+        when(drools2.getArtifactId()).thenReturn("2");
+
+        when(drools1.getVersion()).thenReturn("1");
+        when(drools2.getVersion()).thenReturn("2");
+
+        when(drools1.isAlive()).thenReturn(true);
+        when(drools2.isAlive()).thenReturn(true);
+
+        when(drools1.isBrained()).thenReturn(true);
+        when(drools2.isBrained()).thenReturn(true);
+
+        when(drools1.getSessionNames()).thenReturn(List.of("session1"));
+        when(drools2.getSessionNames()).thenReturn(List.of("session2"));
+
+        doAnswer(AdditionalAnswers
+                .answersWithDelay(15000L, invocationOnMock -> Map.of("TIMEOUT", 1)))
+                .when(drools1).factClassNames(anyString());
+
+        when(drools2.factClassNames(anyString()))
+                .thenReturn(Map.of("java.lang.Integer", 2));
+
+        when(drools1.factCount("session1")).thenReturn(1L);
+        when(drools2.factCount("session2")).thenReturn(2L);
+
+        when(controller1.getDrools()).thenReturn(drools1);
+        when(controller2.getDrools()).thenReturn(drools2);
+
+        when(controller1.getName()).thenReturn("drools1");
+        when(controller2.getName()).thenReturn("drools2");
+
+        when(controller1.isAlive()).thenReturn(true);
+        when(controller2.isAlive()).thenReturn(true);
+    }
+
+    private void whenClients() {
+        when(client1.getName()).thenReturn(CLIENT_NAME1);
+        when(client1.getBaseUrl()).thenReturn(CLIENT_URL1);
+        when(client2.getName()).thenReturn(CLIENT_NAME2);
+        when(client2.getBaseUrl()).thenReturn(CLIENT_URL2);
+        when(client3.getName()).thenReturn(CLIENT_NAME3);
+        when(client3.getBaseUrl()).thenReturn(CLIENT_URL3);
+    }
+
+    private void mocks() {
+        servletFactory = mock(HttpServletServerFactory.class);
+        server1 = mock(HttpServletServer.class);
+        server2 = mock(HttpServletServer.class);
+        clientFactory = mock(HttpClientFactory.class);
+        client1 = mock(HttpClient.class);
+        client2 = mock(HttpClient.class);
+        client3 = mock(HttpClient.class);
+        controllerFactory = mock(PolicyControllerFactory.class);
+        controller1 = mock(PolicyController.class);
+        controller2 = mock(PolicyController.class);
+        drools1 = mock(DroolsController.class);
+        drools2 = mock(DroolsController.class);
+        engineMgr = mock(PolicyEngine.class);
+    }
+
+    @Test
+    public void testHealthcheck() {
+        /* engine not alive */
+
+        when(engineMgr.isAlive()).thenReturn(false);
+        assertEngineDisabled(monitor.healthCheck());
+
+        /* engine alive + controllers + clients */
+
+        when(engineMgr.isAlive()).thenReturn(true);
+        assertEngineEnabled(monitor.healthCheck());
+
+        monitor.controllers = List.of(controller1, controller2);
+
+        mockClients();
+
+        var reports = monitor.healthCheck();
+        logger.info("{}", reports);
+
+        assertSummary(reports, 6, false);
+        assertClients(reports);
+        assertControllers(reports);
+    }
+
+    @Test
+    public void testControllerHealthcheck() {
+        /* engine not alive */
+
+        when(engineMgr.isAlive()).thenReturn(false);
+        assertEngineDisabled(monitor.controllerHealthcheck());
+
+        /* engine alive */
+
+        when(engineMgr.isAlive()).thenReturn(true);
+        assertEngineEnabled(monitor.healthCheck());
+
+        /* engine + controllers */
+
+        monitor.controllers = List.of(controller1, controller2);
+        var reports = monitor.healthCheck();
+        logger.info("{}", reports);
+
+        assertSummary(reports, 3, false);
+
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+
+        assertControllers(reports);
+
+        /* with argument */
+
+        reports = monitor.controllerHealthcheck(controller2);
+        logger.info("{}", reports);
+
+        assertSummary(reports, 2, true);
+    }
+
+    @Test
+    public void testClientHealthcheck() {
+        /* engine not alive */
+
+        when(engineMgr.isAlive()).thenReturn(false);
+        assertEngineDisabled(monitor.clientHealthcheck());
+
+        /* engine alive */
+
+        when(engineMgr.isAlive()).thenReturn(true);
+        assertEngineEnabled(monitor.clientHealthcheck());
+
+        /* engine alive + clients */
+
+        mockClients();
+
+        var reports = monitor.clientHealthcheck();
+        logger.info("{}", reports);
+
+        assertSummary(reports, 4, false);
+        assertClients(reports);
+
+        /* with argument */
+
+        reports = monitor.clientHealthcheck(client1);
+        logger.info("{}", reports);
+
+        assertSummary(reports, 2, true);
+    }
+
+    @Test
+    public void reportOnController() {
+
+        /* controller not alive */
+
+        when(controller1.isAlive()).thenReturn(false);
+
+        var reports = monitor.controllerHealthcheck(controller1);
+        assertSummary(reports, 2, false);
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+        assertReport(reports, false,
+                    controller1.getName(), controller1.getName(),
+                    HealthCheckManager.DISABLED_CODE, HealthCheckManager.DISABLED_MESSAGE);
+
+        /* drools not brained */
+
+        when(controller1.isAlive()).thenReturn(true);
+        when(drools1.isBrained()).thenReturn(false);
+
+        reports = monitor.controllerHealthcheck(controller1);
+        logger.info("{}", reports);
+
+        assertSummary(reports, 2, true);
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+        assertReport(reports, true,
+                controller1.getName(), "1:1:1",
+                HealthCheckManager.BRAINLESS_CODE, HealthCheckManager.BRAINLESS_MESSAGE);
+
+        /* drools not alive */
+
+        when(drools1.isBrained()).thenReturn(true);
+        when(drools1.isAlive()).thenReturn(false);
+
+        reports = monitor.controllerHealthcheck(controller1);
+        logger.info("{}", reports);
+
+        assertSummary(reports, 2, false);
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+        assertReport(reports, false,
+                controller1.getName(), "1:1:1",
+                HealthCheckManager.DISABLED_CODE, HealthCheckManager.DISABLED_MESSAGE);
+
+        /* ok */
+
+        when(drools1.isAlive()).thenReturn(true);
+
+        assertController2(monitor.controllerHealthcheck(controller2));
+    }
+
+    @Test
+    public void testReportOnUnknown() {
+        var reports = monitor.summary(monitor.engineHealthcheck(), monitor.futures(List.of(1)));
+        logger.info("{}", reports);
+
+        assertReport(reports, false,
+            HealthCheckManager.UNKNOWN_ENTITY, "java.lang.Integer",
+            HealthCheckManager.UNKNOWN_ENTITY_CODE, HealthCheckManager.UNKNOWN_ENTITY_MESSAGE);
+    }
+
+    @Test
+    public void testStart() {
+        // good start
+
+        when(server1.start()).thenReturn(true);
+        when(server1.getName()).thenReturn(HealthCheckManager.HEALTHCHECK_SERVER);
+        when(server2.getName()).thenReturn(HealthCheckManager.LIVENESS_SERVER);
+        assertTrue(monitor.start());
+
+        verify(server1).start();
+        verify(server2, never()).start();
+
+        assertEquals(server1, monitor.getHealthcheckServer());
+        assertEquals(server2, monitor.getLivenessServer());
+
+        // healthcheck server start error
+
+        when(server1.start()).thenThrow(new RuntimeException(EXPECTED));
+        assertFalse(monitor.start());
+
+        /*
+         * Generate exception during building.
+         */
+
+        // new monitor
+        monitor = new HealthCheckMonitorImpl() {
+            @Override
+            protected HttpServletServerFactory getServerFactory() {
+                throw new RuntimeException(EXPECTED);
+            }
+        };
+        assertFalse(monitor.start());
+
+    }
+
+    @Test
+    public void testOpen() {
+
+        /* nothing done */
+
+        monitor.healthCheckProperties = new Properties();
+        monitor.open();
+        assertEquals(List.of(), monitor.controllers);
+
+        /* star-controllers */
+
+        monitor.livenessServer = server1;
+        monitor.healthCheckProperties = new Properties();
+        monitor.healthCheckProperties.setProperty("liveness.controllers", "*");
+        when(server1.start()).thenReturn(true);
+
+        monitor.open();
+        assertEquals(controllerFactory.inventory(), monitor.controllers);
+        verify(server1).start();
+
+        /* comma-list-controllers */
+
+        monitor.controllers = new ArrayList<>();
+        monitor.healthCheckProperties.setProperty("liveness.controllers", "controller1,controller2,controller3");
+        when(controllerFactory.get("controller1")).thenReturn(controller1);
+        when(controllerFactory.get("controller2")).thenReturn(controller2);
+        when(controllerFactory.get("controller3")).thenThrow(new RuntimeException("no controller3"));
+        monitor.open();
+        assertEquals(List.of(controller1, controller2), monitor.controllers);
+    }
+
+    @Test
+    public void testShutdown() {
+        monitor.healthcheckServer = server1;
+        monitor.livenessServer = server2;
+        monitor.clients = List.of(client1, client2, client3);
+        when(server1.stop()).thenReturn(true);
+        when(server2.stop()).thenReturn(true);
+
+        monitor.shutdown();
+
+        verify(server1).stop();
+        verify(server2).stop();
+        verify(client1).stop();
+        verify(client2).stop();
+        verify(client3).stop();
+    }
+
+    @Test
+    public void testIsAlive() {
+        assertFalse(monitor.isAlive());
+    }
+
+    @Test
+    public void testToString() {
+        assertTrue(monitor.toString().contains("HealthCheckManager"));
+    }
+
+    private void mockClient1() {
+        // first client is healthy
+        Response resp = mock(Response.class);
+        when(resp.getStatus()).thenReturn(HttpURLConnection.HTTP_OK);
+        when(resp.readEntity(String.class)).thenReturn(RPT_MSG);
+        when(resp.getStatusInfo()).thenReturn(Response.Status.OK);
+        when(client1.get()).thenReturn(resp);
+    }
+
+    private void mockClient2() {
+        // second client throws an exception
+        when(client2.get()).thenThrow(new RuntimeException(EXPECTED));
+    }
+
+    private void mockClient3() {
+        // third client is not healthy
+        Response resp = mock(Response.class);
+        when(resp.getStatus()).thenReturn(HttpURLConnection.HTTP_NOT_FOUND);
+        when(resp.readEntity(String.class)).thenReturn(RPT_NAME);
+        when(resp.getStatusInfo()).thenReturn(Response.Status.NOT_FOUND);
+        when(client3.get()).thenReturn(resp);
+    }
+
+    private void mockClients() {
+        monitor.clients = List.of(client1, client2, client3);
+
+        mockClient1();
+        mockClient2();
+        mockClient3();
+    }
+
+    private void assertEngineEnabled(Reports summary) {
+        assertEquals(1, summary.getDetails().size());
+        assertReport(summary, true, HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE, HealthCheckManager.ENABLED_MESSAGE);
+    }
+
+    private void assertEngineDisabled(Reports summary) {
+        var report = summary.getDetails().get(0);
+        assertFalse(summary.isHealthy());
+        assertEquals(1, summary.getDetails().size());
+        assertFalse(report.isHealthy());
+        assertEquals(HealthCheckManager.ENGINE_NAME, report.getName());
+        assertEquals(HealthCheckManager.DISABLED_CODE, report.getCode());
+        assertEquals(HealthCheckManager.DISABLED_MESSAGE, report.getMessage());
+        assertNotEquals(0L, report.getStartTime());
+        assertNotEquals(0L, report.getEndTime());
+        assertEquals(report.getEndTime() - report.getStartTime(), report.getElapsedTime());
+    }
+
+    private void assertSummary(Reports reports, int size, boolean healthy) {
+        assertNotNull(reports);
+        assertEquals(size, reports.getDetails().size());
+        assertEquals(healthy, reports.isHealthy());
+        assertNotEquals(0L, reports.getStartTime());
+        assertNotEquals(0L, reports.getEndTime());
+        assertEquals(reports.getEndTime() - reports.getStartTime(), reports.getElapsedTime());
+    }
+
+    private void assertReport(Reports summary,
+            boolean healthy, String name, String url, long successCode, String message) {
+        var report = select(summary, name, successCode, message).get(0);
+
+        assertEquals(healthy, report.isHealthy());
+        assertEquals(name, report.getName());
+        assertEquals(url, report.getUrl());
+        assertEquals(successCode, report.getCode());
+        assertEquals(message, report.getMessage());
+        assertNotEquals(0L, report.getStartTime());
+        assertNotEquals(0L, report.getEndTime());
+        assertEquals(report.getEndTime() - report.getStartTime(), report.getElapsedTime());
+    }
+
+    private void assertClient1(Reports reports) {
+        assertReport(reports, true,
+                client1.getName(), client1.getBaseUrl(),
+                HttpStatus.OK_200,
+                HttpStatus.getMessage(200));
+    }
+
+    private void assertClient2(Reports reports) {
+        assertReport(reports, false,
+                client2.getName(), client2.getBaseUrl(),
+                HealthCheckManager.UNREACHABLE_CODE,
+                HealthCheckManager.UNREACHABLE_MESSAGE);
+    }
+
+    private void assertClient3(Reports reports) {
+        assertReport(reports, false,
+                client3.getName(), client3.getBaseUrl(),
+                HttpStatus.NOT_FOUND_404,
+                HttpStatus.getMessage(404));
+    }
+
+    private void assertClients(Reports reports) {
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+
+        assertClient1(reports);
+        assertClient2(reports);
+        assertClient3(reports);
+    }
+
+    private void assertController1(Reports reports) {
+        assertReport(reports, false,
+                controller1.getName(), "1:1:1",
+                HealthCheckManager.TIMEOUT_CODE, HealthCheckManager.TIMEOUT_MESSAGE);
+    }
+
+    private void assertController2(Reports reports) {
+        assertReport(reports, true,
+                controller2.getName(), "2:2:2",
+                2, "[session2:{java.lang.Integer=2}]");
+    }
+
+    private void assertControllers(Reports reports) {
+        assertReport(reports, true,
+                HealthCheckManager.ENGINE_NAME, "engine",
+                HealthCheckManager.SUCCESS_CODE,
+                HealthCheckManager.ENABLED_MESSAGE);
+
+        assertController1(reports);
+        assertController2(reports);
+    }
+
+    /**
+     * Monitor with overrides.
+     */
+    private class HealthCheckMonitorImpl extends HealthCheckManager {
+
+        @Override
+        protected PolicyEngine getEngineManager() {
+            return engineMgr;
+        }
+
+        @Override
+        protected HttpServletServerFactory getServerFactory() {
+            return servletFactory;
+        }
+
+        @Override
+        protected HttpClientFactory getClientFactory() {
+            return clientFactory;
+        }
+
+        @Override
+        protected Properties getPersistentProperties() {
+            return properties;
+        }
+
+        @Override
+        protected PolicyControllerFactory getControllerFactory() {
+            return controllerFactory;
+        }
+
+    }
+}
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckTest.java
index 7040f6d3..01f5063d 100644
--- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckTest.java
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckTest.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP
  * ================================================================================
- * Copyright (C) 2018, 2021 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2018, 2021-2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,94 +22,27 @@ package org.onap.policy.drools.healthcheck;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
-import java.net.HttpURLConnection;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.Properties;
-import javax.ws.rs.core.Response;
-import org.junit.Before;
 import org.junit.Test;
-import org.onap.policy.common.endpoints.http.client.HttpClient;
-import org.onap.policy.common.endpoints.http.client.HttpClientFactory;
-import org.onap.policy.common.endpoints.http.server.HttpServletServer;
-import org.onap.policy.common.endpoints.http.server.HttpServletServerFactory;
 import org.onap.policy.drools.healthcheck.HealthCheck.Report;
 import org.onap.policy.drools.healthcheck.HealthCheck.Reports;
-import org.onap.policy.drools.system.PolicyEngine;
 
 public class HealthCheckTest {
-
-    private static final int RPT_CODE = 100;
+    private static final long RPT_CODE = 100;
     private static final String RPT_MSG = "report-message";
     private static final String RPT_NAME = "report-name";
     private static final String RPT_URL = "report-url";
-    private static final String EXPECTED = "expected exception";
-
-    private static final String CLIENT_NAME1 = "name-a";
-    private static final String CLIENT_URL1 = "url-a";
-    private static final String CLIENT_NAME2 = "name-b";
-    private static final String CLIENT_URL2 = "url-b";
-    private static final String CLIENT_NAME3 = "name-c";
-    private static final String CLIENT_URL3 = "url-c";
-
-    private Properties properties;
-    private HttpServletServerFactory servletFactory;
-    private HttpServletServer server1;
-    private HttpServletServer server2;
-    private HttpClientFactory clientFactory;
-    private HttpClient client1;
-    private HttpClient client2;
-    private HttpClient client3;
-    private List<HttpServletServer> servers;
-    private List<HttpClient> clients;
-    private PolicyEngine engineMgr;
-    private HealthCheckManager monitor;
-
-    /**
-     * Initializes the object to be tested.
-     *
-     * @throws Exception if an error occurs
-     */
-    @Before
-    public void setUp() throws Exception {
-        properties = new Properties();
-        servletFactory = mock(HttpServletServerFactory.class);
-        server1 = mock(HttpServletServer.class);
-        server2 = mock(HttpServletServer.class);
-        clientFactory = mock(HttpClientFactory.class);
-        client1 = mock(HttpClient.class);
-        client2 = mock(HttpClient.class);
-        client3 = mock(HttpClient.class);
-        servers = Arrays.asList(server1, server2);
-        clients = Arrays.asList(client1, client2, client3);
-        engineMgr = mock(PolicyEngine.class);
-
-        when(client1.getName()).thenReturn(CLIENT_NAME1);
-        when(client1.getBaseUrl()).thenReturn(CLIENT_URL1);
-        when(client2.getName()).thenReturn(CLIENT_NAME2);
-        when(client2.getBaseUrl()).thenReturn(CLIENT_URL2);
-        when(client3.getName()).thenReturn(CLIENT_NAME3);
-        when(client3.getBaseUrl()).thenReturn(CLIENT_URL3);
-        when(servletFactory.build(properties)).thenReturn(servers);
-        when(clientFactory.build(properties)).thenReturn(clients);
-        when(engineMgr.isAlive()).thenReturn(true);
-
-        monitor = new HealthCheckMonitorImpl();
-    }
 
     @Test
     public void testReport() {
         Report rpt = new Report();
 
-        // toString should work with un-populated data
         assertNotNull(rpt.toString());
 
         rpt.setCode(RPT_CODE);
@@ -117,19 +50,26 @@ public class HealthCheckTest {
         rpt.setMessage(RPT_MSG);
         rpt.setName(RPT_NAME);
         rpt.setUrl(RPT_URL);
+        rpt.setEndTime();
 
         assertEquals(RPT_CODE, rpt.getCode());
-        assertEquals(true, rpt.isHealthy());
+        assertTrue(rpt.isHealthy());
         assertEquals(RPT_MSG, rpt.getMessage());
         assertEquals(RPT_NAME, rpt.getName());
         assertEquals(RPT_URL, rpt.getUrl());
 
+        assertNotEquals(0L, rpt.getStartTime());
+        assertNotEquals(0L, rpt.getEndTime());
+        assertEquals(rpt.getEndTime() - rpt.getStartTime(), rpt.getElapsedTime());
+
         // flip the flag
         rpt.setHealthy(false);
-        assertEquals(false, rpt.isHealthy());
+        assertFalse(rpt.isHealthy());
 
         // toString should work with populated data
         assertNotNull(rpt.toString());
+
+        assertEquals(rpt, new Report(rpt));
     }
 
     @Test
@@ -152,202 +92,11 @@ public class HealthCheckTest {
 
         // toString should work with populated data
         assertNotNull(reports.toString());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_HealthCheck() {
-        monitor.start();
-
-        // first client is healthy
-        Response resp = mock(Response.class);
-        when(resp.getStatus()).thenReturn(HttpURLConnection.HTTP_OK);
-        when(resp.readEntity(String.class)).thenReturn(RPT_MSG);
-        when(client1.get()).thenReturn(resp);
-
-        // second client throws an exception
-        when(client2.get()).thenThrow(new RuntimeException(EXPECTED));
-
-        // third client is not healthy
-        resp = mock(Response.class);
-        when(resp.getStatus()).thenReturn(HttpURLConnection.HTTP_INTERNAL_ERROR);
-        when(resp.readEntity(String.class)).thenReturn(RPT_NAME);
-        when(client3.get()).thenReturn(resp);
-
-        Reports reports = monitor.healthCheck();
-        assertNotNull(reports);
-        assertEquals(4, reports.getDetails().size());
-        assertFalse(reports.isHealthy());
-
-        int index = 0;
-
-        Report report = reports.getDetails().get(index++);
-        assertEquals(true, report.isHealthy());
-        assertEquals("PDP-D", report.getName());
-        assertEquals("self", report.getUrl());
-        assertEquals("alive", report.getMessage());
-        assertEquals(HttpURLConnection.HTTP_OK, report.getCode());
-
-        report = reports.getDetails().get(index++);
-        assertEquals(true, report.isHealthy());
-        assertEquals(client1.getName(), report.getName());
-        assertEquals(client1.getBaseUrl(), report.getUrl());
-        assertEquals(RPT_MSG, report.getMessage());
-        assertEquals(HttpURLConnection.HTTP_OK, report.getCode());
-
-        report = reports.getDetails().get(index++);
-        assertEquals(false, report.isHealthy());
-        assertEquals(client2.getName(), report.getName());
-        assertEquals(client2.getBaseUrl(), report.getUrl());
-
-        report = reports.getDetails().get(index++);
-        assertEquals(false, report.isHealthy());
-        assertEquals(client3.getName(), report.getName());
-        assertEquals(client3.getBaseUrl(), report.getUrl());
-        assertEquals(RPT_NAME, report.getMessage());
-        assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, report.getCode());
-
-        // indicate that engine is no longer healthy and re-run health check
-        when(engineMgr.isAlive()).thenReturn(false);
-
-        reports = monitor.healthCheck();
-        report = reports.getDetails().get(0);
-
-        assertEquals(false, report.isHealthy());
-        assertEquals("not alive", report.getMessage());
-        assertEquals(500, report.getCode());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_Start() {
-        // arrange for one server to throw an exception
-        when(server1.start()).thenThrow(new RuntimeException(EXPECTED));
-
-        assertTrue(monitor.start());
-
-        verify(server1).start();
-        verify(server2).start();
-
-        /*
-         * Generate exception during building.
-         */
-
-        // new monitor
-        monitor = new HealthCheckMonitorImpl() {
-            @Override
-            protected HttpServletServerFactory getServerFactory() {
-                throw new RuntimeException(EXPECTED);
-            }
-        };
-
-        assertFalse(monitor.start());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_Stop() {
-        monitor.start();
-
-        // arrange for one server and one client to throw an exception
-        when(server1.stop()).thenThrow(new RuntimeException(EXPECTED));
-        when(client2.stop()).thenThrow(new RuntimeException(EXPECTED));
-
-        assertTrue(monitor.stop());
-
-        verify(server1).stop();
-        verify(server2).stop();
-        verify(client1).stop();
-        verify(client2).stop();
-        verify(client3).stop();
-    }
-
-    @Test
-    public void testHealthCheckMonitor_Shutdown() {
-        monitor.start();
-        monitor.shutdown();
-
-        // at least one "stop" should have been called
-        verify(server1).stop();
-    }
-
-    @Test
-    public void testHealthCheckMonitor_IsAlive() {
-        assertFalse(monitor.isAlive());
-
-        monitor.start();
-        assertTrue(monitor.isAlive());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetServers_GetClients() {
-        monitor.start();
-        assertEquals(servers, monitor.getServers());
-        assertEquals(clients, monitor.getClients());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetHttpBody() {
-        Response response = mock(Response.class);
-        when(response.readEntity(String.class)).thenReturn(RPT_MSG);
-        assertEquals(RPT_MSG, monitor.getHttpBody(response, client1));
-
-        // readEntity() throws an exception
-        when(response.readEntity(String.class)).thenThrow(new RuntimeException(EXPECTED));
-        assertEquals(null, monitor.getHttpBody(response, client1));
-    }
-
-    @Test
-    public void testHealthCheckMonitor_StartServer() {
-        monitor.startServer(server1);
-        verify(server1).start();
-
-        // force start() to throw an exception - monitor should still work
-        when(server1.start()).thenThrow(new RuntimeException(EXPECTED));
-        monitor.startServer(server1);
-    }
-
-    @Test
-    public void testHealthCheckMonitor_ToString() {
-        assertTrue(monitor.toString().startsWith("HealthCheckManager("));
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetEngineManager() {
-        assertNotNull(new HealthCheckManager().getEngineManager());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetServerFactory() {
-        assertNotNull(new HealthCheckManager().getServerFactory());
-    }
-
-    @Test
-    public void testHealthCheckMonitor_GetClientFactory() {
-        assertNotNull(new HealthCheckManager().getClientFactory());
-    }
-
-    /**
-     * Monitor with overrides.
-     */
-    private class HealthCheckMonitorImpl extends HealthCheckManager {
-
-        @Override
-        protected PolicyEngine getEngineManager() {
-            return engineMgr;
-        }
-
-        @Override
-        protected HttpServletServerFactory getServerFactory() {
-            return servletFactory;
-        }
-
-        @Override
-        protected HttpClientFactory getClientFactory() {
-            return clientFactory;
-        }
 
-        @Override
-        protected Properties getPersistentProperties(String propertyName) {
-            return properties;
-        }
+        assertNotEquals(0L, reports.getStartTime());
 
+        reports.setEndTime();
+        assertNotEquals(0L, reports.getEndTime());
+        assertEquals(reports.getEndTime() - reports.getStartTime(), reports.getElapsedTime());
     }
 }
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestHealthCheckTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestHealthCheckTest.java
new file mode 100644
index 00000000..2af39a80
--- /dev/null
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestHealthCheckTest.java
@@ -0,0 +1,181 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.healthcheck;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import javax.ws.rs.core.Response;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.onap.policy.common.endpoints.event.comm.bus.internal.BusTopicParams;
+import org.onap.policy.common.endpoints.http.client.HttpClient;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactory;
+import org.onap.policy.common.endpoints.http.client.HttpClientFactoryInstance;
+import org.onap.policy.common.endpoints.http.server.HttpServletServer;
+import org.onap.policy.common.endpoints.http.server.HttpServletServerFactoryInstance;
+import org.onap.policy.common.endpoints.http.server.YamlJacksonHandler;
+import org.onap.policy.common.gson.JacksonHandler;
+import org.onap.policy.common.utils.logging.LoggerUtils;
+import org.onap.policy.common.utils.network.NetworkUtil;
+import org.onap.policy.drools.system.PolicyController;
+import org.onap.policy.drools.system.PolicyControllerFactory;
+
+/**
+ * REST Healthcheck Tests.
+ */
+public class RestHealthCheckTest {
+
+    private static HttpClientFactory clientFactory;
+    private static PolicyControllerFactory controllerFactory;
+    private static HealthCheckManager healthcheckManager;
+    private static PolicyController controller1;
+    private static HttpClient client1;
+
+    private static HealthCheck.Reports summary;
+    private static HttpClient client;
+
+    /**
+     * Set up.
+     */
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LoggerUtils.setLevel("org.onap.policy.common.endpoints", "WARN");
+        LoggerUtils.setLevel("org.eclipse", "ERROR");
+        LoggerUtils.setLevel("org.onap.policy.drools.healthcheck", "DEBUG");
+        LoggerUtils.setLevel("ROOT", "INFO");
+
+        clientFactory = mock(HttpClientFactory.class);
+        controllerFactory = mock(PolicyControllerFactory.class);
+        healthcheckManager = mock(HealthCheckManager.class);
+        controller1 = mock(PolicyController.class);
+        client1 = mock(HttpClient.class);
+
+        summary = new HealthCheck.Reports();
+
+        client = HttpClientFactoryInstance.getClientFactory().build(
+                    BusTopicParams.builder()
+                        .clientName("healthcheck")
+                        .hostname("localhost")
+                        .port(8768)
+                        .basePath("healthcheck")
+                        .managed(true)
+                        .build());
+
+        HttpServletServer server =
+            HttpServletServerFactoryInstance
+                .getServerFactory()
+                .build("lifecycle", "localhost", 8768, "/",
+                    true, true);
+
+        server.setSerializationProvider(
+                String.join(",", JacksonHandler.class.getName(),
+                        YamlJacksonHandler.class.getName()));
+        server.addServletClass("/*", RestMockHealthcheck.class.getName());
+        server.waitedStart(5000L);
+
+        assertTrue(NetworkUtil.isTcpPortOpen("localhost", 8768, 5, 10000L));
+    }
+
+    /**
+     * Tear down.
+     */
+
+    public static void tearDown() {
+        HttpClientFactoryInstance.getClientFactory().destroy();
+        HttpServletServerFactoryInstance.getServerFactory().destroy();
+    }
+
+    @Test
+    public void healthcheck() {
+        when(healthcheckManager.healthCheck()).thenReturn(summary);
+        assertHttp("/");
+    }
+
+    @Test
+    public void engine() {
+        when(healthcheckManager.engineHealthcheck()).thenReturn(summary);
+        assertHttp("engine");
+    }
+
+    @Test
+    public void controllers() {
+        when(healthcheckManager.controllerHealthcheck()).thenReturn(summary);
+        assertHttp("controllers");
+
+        when(controllerFactory.get("controller1")).thenReturn(controller1);
+        when(healthcheckManager.controllerHealthcheck(controller1)).thenReturn(summary);
+        assertHttp("controllers/controller1");
+
+        when(controllerFactory.get("controller1")).thenThrow(new IllegalArgumentException("expected"));
+        Response resp = client.get("controllers/controller1");
+        assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus());
+
+        when(controllerFactory.get("controller2")).thenThrow(new IllegalStateException("expected"));
+        resp = client.get("controllers/controller2");
+        assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), resp.getStatus());
+    }
+
+    @Test
+    public void clients() {
+        when(healthcheckManager.clientHealthcheck()).thenReturn(summary);
+        assertHttp("clients");
+
+        when(clientFactory.get("client1")).thenReturn(client1);
+        when(healthcheckManager.clientHealthcheck(client1)).thenReturn(summary);
+        assertHttp("clients/client1");
+
+        when(clientFactory.get("client2")).thenThrow(new IllegalArgumentException("expected"));
+        Response resp = client.get("clients/client2");
+        assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus());
+    }
+
+    private void assertHttp(String url) {
+        summary.setHealthy(true);
+        var resp = client.get(url);
+        assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus());
+
+        summary.setHealthy(false);
+        resp = client.get(url);
+        assertEquals(Response.Status.SERVICE_UNAVAILABLE.getStatusCode(), resp.getStatus());
+    }
+
+    public static class RestMockHealthcheck extends RestHealthCheck {
+        @Override
+        protected PolicyControllerFactory getControllerFactory() {
+            return controllerFactory;
+        }
+
+        @Override
+        protected HttpClientFactory getClientFactory() {
+            return clientFactory;
+        }
+
+        @Override
+        protected HealthCheck getHealthcheckManager() {
+            return healthcheckManager;
+        }
+
+    }
+}
\ No newline at end of file
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestMockHealthCheck.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestMockHealthCheck.java
index 8cbe1fbf..910974ef 100644
--- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestMockHealthCheck.java
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/RestMockHealthCheck.java
@@ -1,8 +1,8 @@
 /*-
  * ============LICENSE_START=======================================================
- * feature-healthcheck
+ * ONAP
  * ================================================================================
- * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2018,2022 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,9 @@
 
 package org.onap.policy.drools.healthcheck;
 
+import static org.awaitility.Awaitility.await;
+
+import java.util.concurrent.TimeUnit;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
@@ -31,11 +34,22 @@ import org.onap.policy.common.endpoints.http.server.YamlMessageBodyHandler;
 @Path("/")
 public class RestMockHealthCheck {
 
+    protected static final String OK_MESSAGE = "All Alive";
+    protected static volatile boolean stuck = false;
+    protected static volatile long WAIT = 15;
+
     @GET
     @Path("healthcheck/test")
     @Produces({MediaType.APPLICATION_JSON, YamlMessageBodyHandler.APPLICATION_YAML})
     public Response papHealthCheck() {
-        return Response.status(Status.OK).entity("All Alive").build();
+        return Response.status(Status.OK).entity(OK_MESSAGE).build();
     }
 
+    @GET
+    @Path("healthcheck/stuck")
+    @Produces({MediaType.APPLICATION_JSON, YamlMessageBodyHandler.APPLICATION_YAML})
+    public Response stuck() {
+        await().atMost(WAIT, TimeUnit.SECONDS).until(() -> !stuck);
+        return Response.status(Status.OK).entity("I may be stuck: " + stuck).build();
+    }
 }
diff --git a/feature-healthcheck/src/test/resources/echo.drl b/feature-healthcheck/src/test/resources/echo.drl
new file mode 100644
index 00000000..54847da9
--- /dev/null
+++ b/feature-healthcheck/src/test/resources/echo.drl
@@ -0,0 +1,35 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.healthcheck;
+
+rule "INIT"
+lock-on-active
+when
+then
+	insert(new String("I am up"));
+end
+
+rule "ECHO"
+when
+    $o : Object();
+then
+    System.out.println("ECHO: " + $o.toString());
+end
diff --git a/feature-healthcheck/src/test/resources/echo.kmodule b/feature-healthcheck/src/test/resources/echo.kmodule
new file mode 100644
index 00000000..7a1acc70
--- /dev/null
+++ b/feature-healthcheck/src/test/resources/echo.kmodule
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ============LICENSE_START=======================================================
+  ONAP
+  ================================================================================
+  Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+  ================================================================================
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  ============LICENSE_END=========================================================
+  -->
+
+<kmodule xmlns="http://jboss.org/kie/6.0.0/kmodule">
+    <kbase name="onap.policies.drools.healthcheck">
+        <ksession name="echo"/>
+    </kbase>
+</kmodule>
diff --git a/feature-healthcheck/src/test/resources/echo.pom b/feature-healthcheck/src/test/resources/echo.pom
new file mode 100644
index 00000000..2f84f4bd
--- /dev/null
+++ b/feature-healthcheck/src/test/resources/echo.pom
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ============LICENSE_START=======================================================
+  ONAP
+  ================================================================================
+  Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+  ================================================================================
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  ============LICENSE_END=========================================================
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.onap.policy.drools.healthcheck</groupId>
+    <artifactId>echo</artifactId>
+    <!-- the version below is irrelevant -->
+    <version>1.0.0</version>
+</project>
diff --git a/feature-healthcheck/src/test/resources/feature-healthcheck.properties b/feature-healthcheck/src/test/resources/feature-healthcheck.properties
new file mode 100644
index 00000000..63a71b6e
--- /dev/null
+++ b/feature-healthcheck/src/test/resources/feature-healthcheck.properties
@@ -0,0 +1,55 @@
+#
+# ============LICENSE_START=======================================================
+# ONAP
+# ================================================================================
+# Copyright (C) 2022 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+
+http.server.services=HEALTHCHECK,LIVENESS
+
+http.server.services.HEALTHCHECK.host=localhost
+http.server.services.HEALTHCHECK.userName=username
+http.server.services.HEALTHCHECK.password=password
+http.server.services.HEALTHCHECK.restClasses=org.onap.policy.drools.healthcheck.RestMockHealthCheck
+http.server.services.HEALTHCHECK.filterClasses=org.onap.policy.drools.healthcheck.TestAafHealthCheckFilter
+http.server.services.HEALTHCHECK.port=7777
+
+http.server.services.LIVENESS.host=localhost
+http.server.services.LIVENESS.port=7776
+http.server.services.LIVENESS.restClasses=org.onap.policy.drools.healthcheck.RestMockHealthCheck
+
+http.client.services=HEALTHCHECK,LIVENESS,UNAUTH,STUCK
+
+http.client.services.HEALTHCHECK.host=localhost
+http.client.services.HEALTHCHECK.port=7777
+http.client.services.HEALTHCHECK.contextUriPath=healthcheck/test
+http.client.services.HEALTHCHECK.userName=username
+http.client.services.HEALTHCHECK.password=password
+
+http.client.services.LIVENESS.host=localhost
+http.client.services.LIVENESS.port=7776
+http.client.services.LIVENESS.contextUriPath=healthcheck/test
+
+http.client.services.UNAUTH.host=localhost
+http.client.services.UNAUTH.port=7777
+http.client.services.UNAUTH.contextUriPath=healthcheck/test
+
+http.client.services.STUCK.host=localhost
+http.client.services.STUCK.port=7776
+http.client.services.STUCK.contextUriPath=healthcheck/stuck
+
+liveness.controllers=*
+liveness.controllers.timeout=5
diff --git a/feature-lifecycle/src/test/java/org/onap/policy/drools/server/restful/RestLifecycleManagerTest.java b/feature-lifecycle/src/test/java/org/onap/policy/drools/server/restful/RestLifecycleManagerTest.java
index 7676929d..7fcf1e5a 100644
--- a/feature-lifecycle/src/test/java/org/onap/policy/drools/server/restful/RestLifecycleManagerTest.java
+++ b/feature-lifecycle/src/test/java/org/onap/policy/drools/server/restful/RestLifecycleManagerTest.java
@@ -25,7 +25,6 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
-import io.prometheus.client.CollectorRegistry;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-- 
cgit 1.2.3-korg