diff options
author | jhh <jorge.hernandez-herrero@att.com> | 2022-07-11 19:41:36 -0500 |
---|---|---|
committer | jhh <jorge.hernandez-herrero@att.com> | 2022-07-12 09:42:57 -0500 |
commit | 82ae8066c3e031ef326161e962221e166acc4d4e (patch) | |
tree | 4a38f636dfaaa4518143a57228e99e1a96ecf6e2 | |
parent | bbc16d30056655a1a569305d546b68289f81972b (diff) |
Add thread dump capabilities
A script provided can be executed at any time.
The programmatic solution will happen when
controllers (application) timeouts are detected.
Issue-ID: POLICY-4197
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
Change-Id: I1cca5f38e1f9b4f3fd7047e64dc58625ead5e1d1
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
3 files changed, 50 insertions, 2 deletions
diff --git a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java index b310168d..59d0bfbe 100644 --- a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java +++ b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java @@ -21,6 +21,7 @@ package org.onap.policy.drools.healthcheck; import com.google.common.base.Strings; +import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -168,7 +169,7 @@ public class HealthCheckManager implements HealthCheck { } CompletableFuture<Report>[] reportFutures = futures(getControllers()); - return summary(engineHealthcheck(), reportFutures); + return log(summary(engineHealthcheck(), reportFutures)); } @Override @@ -178,7 +179,7 @@ public class HealthCheckManager implements HealthCheck { * it could be useful for troubleshooting. */ CompletableFuture<Report>[] reportFutures = futures(List.of(controller)); - return summary(engineHealthcheck(), reportFutures); + return log(summary(engineHealthcheck(), reportFutures)); } @Override @@ -470,6 +471,23 @@ public class HealthCheckManager implements HealthCheck { return false; } + protected boolean isTimeout(Reports reports) { + return + reports.getDetails() + .stream() + .anyMatch(report -> report.getCode() == TIMEOUT_CODE); + } + + protected Reports log(Reports reports) { + if (isTimeout(reports)) { + logger.warn("Healthcheck Timeout encountered"); + Arrays.stream(ManagementFactory.getThreadMXBean() + .dumpAllThreads(true, true, 100)) + .forEach(threadInfo -> logger.info("Healthcheck Timeout Encountered:\n{}", threadInfo)); + } + return reports; + } + // the following methods may be overridden by junit tests protected PolicyEngine getEngineManager() { diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java index 73d70da9..20cfa9e0 100644 --- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java +++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java @@ -238,6 +238,9 @@ public class HealthCheckManagerTest { /* with argument */ + reports = monitor.controllerHealthcheck(controller1); + logger.info("{}", reports); + reports = monitor.controllerHealthcheck(controller2); logger.info("{}", reports); diff --git a/policy-management/src/main/server-gen/bin/jvm-dump b/policy-management/src/main/server-gen/bin/jvm-dump new file mode 100644 index 00000000..0fa52e39 --- /dev/null +++ b/policy-management/src/main/server-gen/bin/jvm-dump @@ -0,0 +1,27 @@ +#!/usr/bin/env sh + +# ######################################################################## +# Copyright 2022 AT&T Intellectual Property. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ######################################################################## + + +if [ "${DEBUG}" = "y" ]; then + echo "-- $0 $* --" + set -x +fi + +source "${POLICY_HOME}"/etc/profile.d/env.sh +jstack -l -e $(pgrep java) + |