aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorjhh <jorge.hernandez-herrero@att.com>2022-07-11 19:41:36 -0500
committerjhh <jorge.hernandez-herrero@att.com>2022-07-12 09:42:57 -0500
commit82ae8066c3e031ef326161e962221e166acc4d4e (patch)
tree4a38f636dfaaa4518143a57228e99e1a96ecf6e2
parentbbc16d30056655a1a569305d546b68289f81972b (diff)
Add thread dump capabilities
A script provided can be executed at any time. The programmatic solution will happen when controllers (application) timeouts are detected. Issue-ID: POLICY-4197 Signed-off-by: jhh <jorge.hernandez-herrero@att.com> Change-Id: I1cca5f38e1f9b4f3fd7047e64dc58625ead5e1d1 Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
-rw-r--r--feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java22
-rw-r--r--feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java3
-rw-r--r--policy-management/src/main/server-gen/bin/jvm-dump27
3 files changed, 50 insertions, 2 deletions
diff --git a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
index b310168d..59d0bfbe 100644
--- a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
+++ b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
@@ -21,6 +21,7 @@
package org.onap.policy.drools.healthcheck;
import com.google.common.base.Strings;
+import java.lang.management.ManagementFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -168,7 +169,7 @@ public class HealthCheckManager implements HealthCheck {
}
CompletableFuture<Report>[] reportFutures = futures(getControllers());
- return summary(engineHealthcheck(), reportFutures);
+ return log(summary(engineHealthcheck(), reportFutures));
}
@Override
@@ -178,7 +179,7 @@ public class HealthCheckManager implements HealthCheck {
* it could be useful for troubleshooting.
*/
CompletableFuture<Report>[] reportFutures = futures(List.of(controller));
- return summary(engineHealthcheck(), reportFutures);
+ return log(summary(engineHealthcheck(), reportFutures));
}
@Override
@@ -470,6 +471,23 @@ public class HealthCheckManager implements HealthCheck {
return false;
}
+ protected boolean isTimeout(Reports reports) {
+ return
+ reports.getDetails()
+ .stream()
+ .anyMatch(report -> report.getCode() == TIMEOUT_CODE);
+ }
+
+ protected Reports log(Reports reports) {
+ if (isTimeout(reports)) {
+ logger.warn("Healthcheck Timeout encountered");
+ Arrays.stream(ManagementFactory.getThreadMXBean()
+ .dumpAllThreads(true, true, 100))
+ .forEach(threadInfo -> logger.info("Healthcheck Timeout Encountered:\n{}", threadInfo));
+ }
+ return reports;
+ }
+
// the following methods may be overridden by junit tests
protected PolicyEngine getEngineManager() {
diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
index 73d70da9..20cfa9e0 100644
--- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
@@ -238,6 +238,9 @@ public class HealthCheckManagerTest {
/* with argument */
+ reports = monitor.controllerHealthcheck(controller1);
+ logger.info("{}", reports);
+
reports = monitor.controllerHealthcheck(controller2);
logger.info("{}", reports);
diff --git a/policy-management/src/main/server-gen/bin/jvm-dump b/policy-management/src/main/server-gen/bin/jvm-dump
new file mode 100644
index 00000000..0fa52e39
--- /dev/null
+++ b/policy-management/src/main/server-gen/bin/jvm-dump
@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+
+# ########################################################################
+# Copyright 2022 AT&T Intellectual Property. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ########################################################################
+
+
+if [ "${DEBUG}" = "y" ]; then
+ echo "-- $0 $* --"
+ set -x
+fi
+
+source "${POLICY_HOME}"/etc/profile.d/env.sh
+jstack -l -e $(pgrep java)
+