From 82ae8066c3e031ef326161e962221e166acc4d4e Mon Sep 17 00:00:00 2001
From: jhh <jorge.hernandez-herrero@att.com>
Date: Mon, 11 Jul 2022 19:41:36 -0500
Subject: Add thread dump capabilities

A script provided can be executed at any time.
The programmatic solution will happen when
controllers (application) timeouts are detected.

Issue-ID: POLICY-4197
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
Change-Id: I1cca5f38e1f9b4f3fd7047e64dc58625ead5e1d1
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
---
 .../drools/healthcheck/HealthCheckManager.java     | 22 ++++++++++++++++--
 .../drools/healthcheck/HealthCheckManagerTest.java |  3 +++
 policy-management/src/main/server-gen/bin/jvm-dump | 27 ++++++++++++++++++++++
 3 files changed, 50 insertions(+), 2 deletions(-)
 create mode 100644 policy-management/src/main/server-gen/bin/jvm-dump

diff --git a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
index b310168d..59d0bfbe 100644
--- a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
+++ b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
@@ -21,6 +21,7 @@
 package org.onap.policy.drools.healthcheck;
 
 import com.google.common.base.Strings;
+import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -168,7 +169,7 @@ public class HealthCheckManager implements HealthCheck {
         }
 
         CompletableFuture<Report>[] reportFutures = futures(getControllers());
-        return summary(engineHealthcheck(), reportFutures);
+        return log(summary(engineHealthcheck(), reportFutures));
     }
 
     @Override
@@ -178,7 +179,7 @@ public class HealthCheckManager implements HealthCheck {
          * it could be useful for troubleshooting.
          */
         CompletableFuture<Report>[] reportFutures = futures(List.of(controller));
-        return summary(engineHealthcheck(), reportFutures);
+        return log(summary(engineHealthcheck(), reportFutures));
     }
 
     @Override
@@ -470,6 +471,23 @@ public class HealthCheckManager implements HealthCheck {
         return false;
     }
 
+    protected boolean isTimeout(Reports reports) {
+        return
+            reports.getDetails()
+                .stream()
+                .anyMatch(report -> report.getCode() == TIMEOUT_CODE);
+    }
+
+    protected Reports log(Reports reports) {
+        if (isTimeout(reports)) {
+            logger.warn("Healthcheck Timeout encountered");
+            Arrays.stream(ManagementFactory.getThreadMXBean()
+                    .dumpAllThreads(true, true, 100))
+                .forEach(threadInfo -> logger.info("Healthcheck Timeout Encountered:\n{}", threadInfo));
+        }
+        return reports;
+    }
+
     // the following methods may be overridden by junit tests
 
     protected PolicyEngine getEngineManager() {
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
index 73d70da9..20cfa9e0 100644
--- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
@@ -238,6 +238,9 @@ public class HealthCheckManagerTest {
 
         /* with argument */
 
+        reports = monitor.controllerHealthcheck(controller1);
+        logger.info("{}", reports);
+
         reports = monitor.controllerHealthcheck(controller2);
         logger.info("{}", reports);
 
diff --git a/policy-management/src/main/server-gen/bin/jvm-dump b/policy-management/src/main/server-gen/bin/jvm-dump
new file mode 100644
index 00000000..0fa52e39
--- /dev/null
+++ b/policy-management/src/main/server-gen/bin/jvm-dump
@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+
+# ########################################################################
+# Copyright 2022 AT&T Intellectual Property. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ########################################################################
+
+
+if [ "${DEBUG}" = "y" ]; then
+    echo "-- $0 $* --"
+    set -x
+fi
+
+source "${POLICY_HOME}"/etc/profile.d/env.sh
+jstack -l -e $(pgrep java)
+
-- 
cgit 1.2.3-korg