Merge "support transaction reports in engine"

14 files changed, 472 insertions, 66 deletions

diff --git a/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStats.java b/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStats.java
new file mode 100644
index 00000000..d6d8d500
--- /dev/null
+++ b/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStats.java
@@ -0,0 +1,121 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.stats;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+import java.time.Instant;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.ToString;
+import org.onap.policy.drools.metrics.TransMetric;
+import org.slf4j.Logger;
+
+/**
+ * Basic policy execution statistics.
+ */
+
+@Data
+@NoArgsConstructor
+@EqualsAndHashCode
+@ToString
+public class PolicyStats {
+
+    private static final Logger logger = getLogger(PolicyStats.class);
+
+    /**
+     * Number of executed policy transactions.
+     */
+    private long policyExecutedCount;
+
+    /**
+     * Number of successfully executed policy transactions.
+     */
+    private long policyExecutedSuccessCount;
+
+    /**
+     * Number of failed executions of policy transactions.
+     */
+    private long policyExecutedFailCount;
+
+    /**
+     * Last time the policy transaction was executed.
+     */
+    private long lastExecutionTime;
+
+    /**
+     * Average execution time of a policy transaction.
+     */
+    private double averageExecutionTime;
+
+    /**
+     * Total policy execution times.
+     */
+    private double totalElapsedTime;
+
+    /**
+     * Uptime of the entity holding the stats.
+     */
+    private long birthTime = Instant.now().toEpochMilli();
+
+    /**
+     * Time last transaction was started.
+     */
+    private long lastStart;
+
+    /**
+     * add a stat transaction record.
+     */
+    public void stat(@NonNull TransMetric trans) {
+        policyExecutedCount++;
+        if (trans.isSuccess()) {
+            policyExecutedSuccessCount++;
+        } else {
+            policyExecutedFailCount++;
+        }
+
+        // make sure transaction has values that we care about
+
+        if (trans.getStartTime() == null) {
+            logger.warn("policy transaction contains no start time: {}", trans);
+            trans.setStartTime(null);
+        }
+
+        if (trans.getEndTime() == null) {
+            logger.warn("policy transaction contains no end time: {}", trans);
+            trans.setEndTime(null);
+        }
+
+        if (trans.getElapsedTime() == null) {
+            logger.warn("policy transaction contains no elapsed time: {}", trans);
+            trans.setElapsedTime(null);
+        }
+
+        // compute after the preconditions are satisfied
+
+        lastExecutionTime = trans.getEndTime().toEpochMilli();
+        totalElapsedTime += trans.getElapsedTime();
+        averageExecutionTime = totalElapsedTime / policyExecutedCount;
+        lastStart = trans.getStartTime().toEpochMilli();
+    }
+}
diff --git a/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStatsManager.java b/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStatsManager.java
new file mode 100644
index 00000000..bb1b2679
--- /dev/null
+++ b/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStatsManager.java
@@ -0,0 +1,52 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.stats;
+
+import java.util.HashMap;
+import java.util.Map;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.ToString;
+import org.onap.policy.drools.metrics.TransMetric;
+
+/**
+ * Policy Stats Manager that manages PDP-D statistics.
+ */
+
+@NoArgsConstructor
+@Data
+@EqualsAndHashCode
+@ToString
+public class PolicyStatsManager {
+
+    private final PolicyStats groupStat = new PolicyStats();
+    private final Map<String, PolicyStats> subgroupStats = new HashMap<>();
+
+    /**
+     * stat a new transaction.
+     */
+    public synchronized void stat(@NonNull String subGroupName, @NonNull TransMetric transaction) {
+        groupStat.stat(transaction);
+        subgroupStats.computeIfAbsent(subGroupName, key -> new PolicyStats()).stat(transaction);
+    }
+}
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
index 685cf235..ef641f31 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP
  * ================================================================================
- * Copyright (C) 2017-2020 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -33,9 +33,12 @@ import org.onap.policy.common.endpoints.http.server.HttpServletServer;
 import org.onap.policy.drools.core.lock.Lock;
 import org.onap.policy.drools.core.lock.LockCallback;
 import org.onap.policy.drools.features.PolicyEngineFeatureApi;
+import org.onap.policy.drools.metrics.Metric;
+import org.onap.policy.drools.metrics.TransMetric;
 import org.onap.policy.drools.policies.DomainMaker;
 import org.onap.policy.drools.protocol.configuration.ControllerConfiguration;
 import org.onap.policy.drools.protocol.configuration.PdpdConfiguration;
+import org.onap.policy.drools.stats.PolicyStatsManager;
 
 /**
  * Policy Engine, the top abstraction for the Drools PDP Policy Engine. It abstracts away a Drools
@@ -147,7 +150,6 @@ public interface PolicyEngine extends Startable, Lockable, TopicListener {
      * @return the updated Policy Controller
      * @throws IllegalArgumentException in the configuration is invalid
      * @throws IllegalStateException    if the controller is in a bad state
-     * @throws Exception                any other reason
      */
     PolicyController updatePolicyController(ControllerConfiguration configuration);
 
@@ -234,15 +236,21 @@ public interface PolicyEngine extends Startable, Lockable, TopicListener {
      */
     List<String> getFeatures();
 
-
     /**
      * get domain maker.
      *
-     * @return the domain maker.
+     * @return the domain maker
      */
     DomainMaker getDomainMaker();
 
     /**
+     * get statistics for this PDP.
+     *
+     * @return statistics
+     */
+    PolicyStatsManager getStats();
+
+    /**
      * Attempts the dispatching of an "event" object.
      *
      * @param topic topic
@@ -337,4 +345,22 @@ public interface PolicyEngine extends Startable, Lockable, TopicListener {
      * @return policy engine configuration
      */
     Properties defaultTelemetryConfig();
+
+    /**
+     * Track a policy execution metric.
+     *
+     * @param controllerName controller name
+     * @param policyName policy name
+     * @param metric metric
+     */
+    void metric(String controllerName, String policyName, Metric metric);
+
+    /**
+     * Track a policy execution transaction.
+     *
+     * @param controllerName controller name
+     * @param policyName policy name
+     * @param transaction transaction
+     */
+    void transaction(String controllerName, String policyName, TransMetric transaction);
 }
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngineManager.java b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngineManager.java
index 49461210..e20986cb 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngineManager.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngineManager.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP
  * ================================================================================
- * Copyright (C) 2019-2020 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2019-2021 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -67,6 +67,8 @@ import org.onap.policy.drools.features.PolicyControllerFeatureApi;
 import org.onap.policy.drools.features.PolicyControllerFeatureApiConstants;
 import org.onap.policy.drools.features.PolicyEngineFeatureApi;
 import org.onap.policy.drools.features.PolicyEngineFeatureApiConstants;
+import org.onap.policy.drools.metrics.Metric;
+import org.onap.policy.drools.metrics.TransMetric;
 import org.onap.policy.drools.persistence.SystemPersistence;
 import org.onap.policy.drools.persistence.SystemPersistenceConstants;
 import org.onap.policy.drools.policies.DomainMaker;
@@ -77,6 +79,7 @@ import org.onap.policy.drools.protocol.configuration.ControllerConfiguration;
 import org.onap.policy.drools.protocol.configuration.PdpdConfiguration;
 import org.onap.policy.drools.server.restful.RestManager;
 import org.onap.policy.drools.server.restful.aaf.AafTelemetryAuthFilter;
+import org.onap.policy.drools.stats.PolicyStatsManager;
 import org.onap.policy.drools.system.internal.SimpleLockManager;
 import org.onap.policy.drools.utils.PropertyUtil;
 import org.onap.policy.drools.utils.logging.LoggerUtil;
@@ -84,7 +87,6 @@ import org.onap.policy.drools.utils.logging.MdcTransaction;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
 /**
  * Policy Engine Manager Implementation.
  */
@@ -157,7 +159,10 @@ class PolicyEngineManager implements PolicyEngine {
     @Getter(AccessLevel.PROTECTED)
     private PolicyResourceLockManager lockManager = null;
 
-    private DomainMaker domainMaker = new DomainMaker();
+    private final DomainMaker domainMaker = new DomainMaker();
+
+    @Getter
+    private final PolicyStatsManager stats = new PolicyStatsManager();
 
     /**
      * gson parser to decode configuration requests.
@@ -245,6 +250,21 @@ class PolicyEngineManager implements PolicyEngine {
     }
 
     @Override
+    public void metric(String controllerName, String policyName, Metric metric) {
+        // sub-operations are not being tracked
+    }
+
+    @Override
+    public void transaction(@NonNull String controllerName,
+            @NonNull String policyName, @NonNull TransMetric transaction) {
+
+        // will stat on a per policy name that for an admin would
+        // be more significant than a controller name.
+
+        getStats().stat(controllerName + "[" + policyName + "]", transaction);
+    }
+
+    @Override
     @GsonJsonIgnore
     public ScheduledExecutorService getExecutorService() {
         return executorService;
@@ -252,8 +272,9 @@ class PolicyEngineManager implements PolicyEngine {
 
     private ScheduledExecutorService makeExecutorService(Properties properties) {
         int nthreads = DEFAULT_EXECUTOR_THREADS;
+
         try {
-            nthreads = Integer.valueOf(
+            nthreads = Integer.parseInt(
                             properties.getProperty(EXECUTOR_THREAD_PROP, String.valueOf(DEFAULT_EXECUTOR_THREADS)));
 
         } catch (NumberFormatException e) {
@@ -370,7 +391,7 @@ class PolicyEngineManager implements PolicyEngine {
         } else {
             final String msg = "Configuration Entity is not supported: " + entity;
             mdcTrans.resetSubTransaction().setStatusCode(false).setResponseDescription(msg).flush();
-            logger.warn(LoggerUtil.TRANSACTION_LOG_MARKER_NAME, msg);
+            logger.warn(LoggerUtil.TRANSACTION_LOG_MARKER, msg);
             throw new IllegalArgumentException(msg);
         }
     }
@@ -440,7 +461,7 @@ class PolicyEngineManager implements PolicyEngine {
             } catch (final Exception e) {
                 mdcTrans.setStatusCode(false).setResponseCode(e.getClass().getName())
                         .setResponseDescription(e.getMessage()).flush();
-                logger.error(LoggerUtil.TRANSACTION_LOG_MARKER_NAME,
+                logger.error(LoggerUtil.TRANSACTION_LOG_MARKER,
                         "{}: cannot update-policy-controllers because of {}", this, e.getMessage(), e);
             }
         }
@@ -679,7 +700,7 @@ class PolicyEngineManager implements PolicyEngine {
     }
 
     @FunctionalInterface
-    private static interface PredicateWithEx<T> {
+    private interface PredicateWithEx<T> {
         boolean test(T value) throws InterruptedException;
     }
 
@@ -961,10 +982,10 @@ class PolicyEngineManager implements PolicyEngine {
 
         this.locked = false;
 
-        boolean success = true;
+        boolean success;
 
         try {
-            success = (this.lockManager == null || this.lockManager.unlock()) && success;
+            success = this.lockManager == null || this.lockManager.unlock();
         } catch (final RuntimeException e) {
             logger.warn("{}: cannot unlock() lock manager because of {}", this, e.getMessage(), e);
             success = false;
diff --git a/policy-management/src/test/java/org/onap/policy/drools/stats/PolicyStatsManagerTest.java b/policy-management/src/test/java/org/onap/policy/drools/stats/PolicyStatsManagerTest.java
new file mode 100644
index 00000000..9ab23979
--- /dev/null
+++ b/policy-management/src/test/java/org/onap/policy/drools/stats/PolicyStatsManagerTest.java
@@ -0,0 +1,44 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.stats;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.onap.policy.drools.metrics.TransMetric;
+
+public class PolicyStatsManagerTest {
+
+    @Test
+    public void testStat() {
+        PolicyStatsManager stats = new PolicyStatsManager();
+        assertEquals(0, stats.getGroupStat().getPolicyExecutedCount());
+
+        TransMetric trans = new TransMetric();
+        stats.stat("foo", trans);
+        stats.stat("blah", trans);
+        stats.stat("blah", trans);
+        assertEquals(3, stats.getGroupStat().getPolicyExecutedCount());
+
+        assertEquals(1, stats.getSubgroupStats().get("foo").getPolicyExecutedFailCount());
+        assertEquals(2, stats.getSubgroupStats().get("blah").getPolicyExecutedFailCount());
+    }
+}
\ No newline at end of file
diff --git a/policy-management/src/test/java/org/onap/policy/drools/stats/PolicyStatsTest.java b/policy-management/src/test/java/org/onap/policy/drools/stats/PolicyStatsTest.java
new file mode 100644
index 00000000..29518e93
--- /dev/null
+++ b/policy-management/src/test/java/org/onap/policy/drools/stats/PolicyStatsTest.java
@@ -0,0 +1,86 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.stats;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.onap.policy.drools.metrics.TransMetric;
+
+public class PolicyStatsTest {
+
+    @Test
+    public void testStat() {
+        TransMetric trans1 = createTrans();
+        trans1.setSuccess(true);
+
+        PolicyStats stats = new PolicyStats();
+        stats.stat(trans1);
+
+        assertEquals(1, stats.getPolicyExecutedCount());
+        assertEquals(trans1.getStartTime().toEpochMilli(), stats.getLastStart());
+        assertEquals((double) trans1.getElapsedTime(), stats.getAverageExecutionTime(), 0.0d);
+        assertEquals(trans1.getEndTime().toEpochMilli(), stats.getLastExecutionTime());
+        assertEquals(0, stats.getPolicyExecutedFailCount());
+        assertEquals(1, stats.getPolicyExecutedSuccessCount());
+        assertThat(stats.getBirthTime()).isGreaterThanOrEqualTo(trans1.getStartTime().toEpochMilli());
+
+        TransMetric trans2 = createTrans();
+        trans2.setSuccess(false);
+        trans2.setEndTime(trans2.getStartTime().plusMillis(5));
+        trans2.setElapsedTime(null);
+        stats.stat(trans2);
+
+        assertEquals(2, stats.getPolicyExecutedCount());
+        assertEquals(trans2.getStartTime().toEpochMilli(), stats.getLastStart());
+        assertEquals((5 + 1) / 2d, stats.getAverageExecutionTime(), 0.0d);
+        assertEquals(trans2.getEndTime().toEpochMilli(), stats.getLastExecutionTime());
+        assertEquals(1, stats.getPolicyExecutedFailCount());
+        assertEquals(1, stats.getPolicyExecutedSuccessCount());
+        assertThat(stats.getBirthTime()).isLessThanOrEqualTo(trans2.getStartTime().toEpochMilli());
+
+        TransMetric trans3 = createTrans();
+        trans3.setSuccess(false);
+        trans3.setEndTime(trans3.getStartTime().plusMillis(9));
+        trans3.setElapsedTime(null);
+        stats.stat(trans3);
+
+        assertEquals(3, stats.getPolicyExecutedCount());
+        assertEquals(trans3.getStartTime().toEpochMilli(), stats.getLastStart());
+        assertEquals((5 + 1 + 9) / 3d, stats.getAverageExecutionTime(), 0.0d);
+        assertEquals(trans3.getEndTime().toEpochMilli(), stats.getLastExecutionTime());
+        assertEquals(2, stats.getPolicyExecutedFailCount());
+        assertEquals(1, stats.getPolicyExecutedSuccessCount());
+        assertThat(stats.getBirthTime()).isLessThanOrEqualTo(trans2.getStartTime().toEpochMilli());
+    }
+
+    private TransMetric createTrans() {
+        TransMetric trans = new TransMetric();
+        trans.setStartTime(null);
+        trans.setEndTime(trans.getStartTime().plusMillis(1));
+
+        trans.setElapsedTime(null);
+        assertEquals(1L, trans.getElapsedTime().longValue());
+        return trans;
+    }
+
+}
\ No newline at end of file
diff --git a/policy-management/src/test/java/org/onap/policy/drools/system/PolicyEngineManagerTest.java b/policy-management/src/test/java/org/onap/policy/drools/system/PolicyEngineManagerTest.java
index f4876a95..f9b101c3 100644
--- a/policy-management/src/test/java/org/onap/policy/drools/system/PolicyEngineManagerTest.java
+++ b/policy-management/src/test/java/org/onap/policy/drools/system/PolicyEngineManagerTest.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP
  * ================================================================================
- * Copyright (C) 2018-2020 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2018-2021 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -64,12 +64,15 @@ import org.onap.policy.drools.core.lock.LockCallback;
 import org.onap.policy.drools.core.lock.PolicyResourceLockManager;
 import org.onap.policy.drools.features.PolicyControllerFeatureApi;
 import org.onap.policy.drools.features.PolicyEngineFeatureApi;
+import org.onap.policy.drools.metrics.Metric;
+import org.onap.policy.drools.metrics.TransMetric;
 import org.onap.policy.drools.persistence.SystemPersistence;
 import org.onap.policy.drools.properties.DroolsPropertyConstants;
 import org.onap.policy.drools.protocol.coders.EventProtocolCoder;
 import org.onap.policy.drools.protocol.configuration.ControllerConfiguration;
 import org.onap.policy.drools.protocol.configuration.DroolsConfiguration;
 import org.onap.policy.drools.protocol.configuration.PdpdConfiguration;
+import org.onap.policy.drools.stats.PolicyStatsManager;
 import org.onap.policy.drools.system.internal.SimpleLockManager;
 import org.onap.policy.drools.system.internal.SimpleLockProperties;
 
@@ -115,7 +118,6 @@ public class PolicyEngineManagerTest {
     private HttpClientFactory clientFactory;
     private HttpClient client1;
     private HttpClient client2;
-    private List<HttpClient> clients;
     private TopicEndpoint endpoint;
     private PolicyController controller;
     private PolicyController controller2;
@@ -143,6 +145,7 @@ public class PolicyEngineManagerTest {
     private PolicyEngineManager mgr;
     private ScheduledExecutorService exsvc;
     private PolicyResourceLockManager lockmgr;
+    private PolicyStatsManager statsManager;
 
     /**
      * Initializes the object to be tested.
@@ -172,7 +175,6 @@ public class PolicyEngineManagerTest {
         serverFactory = mock(HttpServletServerFactory.class);
         client1 = mock(HttpClient.class);
         client2 = mock(HttpClient.class);
-        clients = Arrays.asList(client1, client2);
         clientFactory = mock(HttpClientFactory.class);
         endpoint = mock(TopicEndpoint.class);
         controller = mock(PolicyController.class);
@@ -200,6 +202,8 @@ public class PolicyEngineManagerTest {
         pdpConfig = new PdpdConfiguration();
         exsvc = mock(ScheduledExecutorService.class);
         lockmgr = mock(PolicyResourceLockManager.class);
+        statsManager = new PolicyStatsManager();
+        statsManager.getGroupStat().setBirthTime(0L);
 
         when(lockmgr.start()).thenReturn(true);
         when(lockmgr.stop()).thenReturn(true);
@@ -235,7 +239,7 @@ public class PolicyEngineManagerTest {
         when(client2.start()).thenReturn(true);
         when(client2.stop()).thenReturn(true);
 
-        when(clientFactory.inventory()).thenReturn(clients);
+        when(clientFactory.inventory()).thenReturn(List.of(client1, client2));
 
         when(source1.getTopic()).thenReturn("source1-topic");
         when(source1.start()).thenReturn(true);
@@ -286,7 +290,7 @@ public class PolicyEngineManagerTest {
         when(endpoint.lock()).thenReturn(true);
         when(endpoint.unlock()).thenReturn(true);
         when(endpoint.getTopicSink(CommInfrastructure.NOOP, MY_TOPIC)).thenReturn(sink1);
-        when(endpoint.getTopicSinks(MY_TOPIC)).thenReturn(Arrays.asList(sink1));
+        when(endpoint.getTopicSinks(MY_TOPIC)).thenReturn(Collections.singletonList(sink1));
 
         when(coder.encode(any(), any())).thenReturn(MESSAGE);
 
@@ -295,6 +299,7 @@ public class PolicyEngineManagerTest {
 
         when(engine.createPolicyController(CONTROLLER3, properties)).thenReturn(controller3);
         when(engine.createPolicyController(CONTROLLER4, properties)).thenReturn(controller4);
+        when(engine.getStats()).thenReturn(statsManager);
 
         config3.setName(CONTROLLER3);
         config3.setOperation(ControllerConfiguration.CONFIG_CONTROLLER_OPERATION_CREATE);
@@ -574,7 +579,7 @@ public class PolicyEngineManagerTest {
 
         // source list of size 1
         setUp();
-        when(endpoint.addTopicSources(any(Properties.class))).thenReturn(Arrays.asList(source1));
+        when(endpoint.addTopicSources(any(Properties.class))).thenReturn(Collections.singletonList(source1));
         mgr.configure(properties);
         assertTrue(mgr.configure(pdpConfig));
 
@@ -671,7 +676,7 @@ public class PolicyEngineManagerTest {
         // force exception in the first controller with invalid operation
         setUp();
         config3.setOperation("unknown-operation");
-        assertEquals(Arrays.asList(controller4), mgr.updatePolicyControllers(pdpConfig.getControllers()));
+        assertEquals(Collections.singletonList(controller4), mgr.updatePolicyControllers(pdpConfig.getControllers()));
 
         // controller3 should NOT have been done
         verify(controllerFactory, never()).patch(controller3, drools3);
@@ -1355,6 +1360,18 @@ public class PolicyEngineManagerTest {
     }
 
     @Test
+    public void testTransaction() {
+        mgr.metric("foo", "bar", new Metric());
+        assertEquals(0, mgr.getStats().getGroupStat().getPolicyExecutedCount());
+        assertEquals(0, mgr.getStats().getSubgroupStats().size());
+
+        mgr.transaction("foo", "bar", new TransMetric());
+        assertEquals(1, mgr.getStats().getGroupStat().getPolicyExecutedCount());
+        assertEquals(1, mgr.getStats().getSubgroupStats().size());
+        assertEquals(1, mgr.getStats().getSubgroupStats().get("foo[bar]").getPolicyExecutedFailCount());
+    }
+
+    @Test
     public void testOnTopicEvent() {
         mgr.onTopicEvent(CommInfrastructure.NOOP, MY_TOPIC, pdpConfigJson);
 
@@ -1913,7 +1930,7 @@ public class PolicyEngineManagerTest {
         // remaining methods should not have been invoked
         assertThatThrownBy(() -> verifyBefore.accept(prov2)).isInstanceOf(AssertionError.class);
 
-        assertThatThrownBy(() -> verifyMiddle.run()).isInstanceOf(AssertionError.class);
+        assertThatThrownBy(verifyMiddle::run).isInstanceOf(AssertionError.class);
 
         assertThatThrownBy(() -> verifyAfter.accept(prov1)).isInstanceOf(AssertionError.class);
         assertThatThrownBy(() -> verifyAfter.accept(prov2)).isInstanceOf(AssertionError.class);
@@ -1995,6 +2012,11 @@ public class PolicyEngineManagerTest {
             return exsvc;
         }
 
+        @Override
+        public PolicyStatsManager getStats() {
+            return statsManager;
+        }
+
         /**
          * Shutdown thread with overrides.
          */
@@ -2027,7 +2049,7 @@ public class PolicyEngineManagerTest {
     }
 
     @FunctionalInterface
-    private static interface RunnableWithEx {
+    private interface RunnableWithEx {
         void run() throws Exception;
     }
 }
diff --git a/policy-management/src/test/java/org/onap/policy/drools/system/PolicyEngineTest.java b/policy-management/src/test/java/org/onap/policy/drools/system/PolicyEngineTest.java
index 2864d5de..30d38ce1 100644
--- a/policy-management/src/test/java/org/onap/policy/drools/system/PolicyEngineTest.java
+++ b/policy-management/src/test/java/org/onap/policy/drools/system/PolicyEngineTest.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP
  * ================================================================================
- * Copyright (C) 2017-2020 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -100,7 +100,7 @@ public class PolicyEngineTest {
     /**
      * logger.
      */
-    private static Logger logger = LoggerFactory.getLogger(PolicyEngineTest.class);
+    private static final Logger logger = LoggerFactory.getLogger(PolicyEngineTest.class);
 
     private static GsonTestUtils gson;
 
@@ -148,7 +148,7 @@ public class PolicyEngineTest {
     }
 
     @AfterClass
-    public static void tearDown() throws IOException {
+    public static void tearDown() {
         logger.info("enter");
         cleanUpWorkingDir();
     }
@@ -170,6 +170,7 @@ public class PolicyEngineTest {
 
         logger.info("engine {} has configuration {}", PolicyEngineConstants.getManager(), engineProps);
 
+        PolicyEngineConstants.getManager().getStats().getGroupStat().setBirthTime(0L);
         gson.compareGson(PolicyEngineConstants.getManager(),
                         new File(PolicyEngineTest.class.getSimpleName() + "Config.json"));
     }
@@ -234,7 +235,7 @@ public class PolicyEngineTest {
     }
 
     @Test
-    public void test400ControllerAdd() throws Exception {
+    public void test400ControllerAdd() {
         logger.info("enter");
 
         final Properties controllerProperties = new Properties();
@@ -243,6 +244,7 @@ public class PolicyEngineTest {
 
         assertEquals(1, PolicyControllerConstants.getFactory().inventory().size());
 
+        PolicyEngineConstants.getManager().getStats().getGroupStat().setBirthTime(0L);
         gson.compareGson(PolicyEngineConstants.getManager(),
                         new File(PolicyEngineTest.class.getSimpleName() + "Add.json"));
     }
@@ -263,7 +265,7 @@ public class PolicyEngineTest {
     }
 
     @Test
-    public void test500Deactivate() throws Exception {
+    public void test500Deactivate() {
         logger.info("enter");
 
         PolicyEngineConstants.getManager().deactivate();
@@ -276,7 +278,7 @@ public class PolicyEngineTest {
     }
 
     @Test
-    public void test501Activate() throws Exception {
+    public void test501Activate() {
         logger.info("enter");
 
         PolicyEngineConstants.getManager().activate();
@@ -289,7 +291,7 @@ public class PolicyEngineTest {
     }
 
     @Test
-    public void test900ControllerRemove() throws Exception {
+    public void test900ControllerRemove() {
         logger.info("enter");
 
         PolicyEngineConstants.getManager().removePolicyController(TEST_CONTROLLER_NAME);
@@ -297,7 +299,7 @@ public class PolicyEngineTest {
     }
 
     @Test
-    public void test901Stop() throws InterruptedException {
+    public void test901Stop() {
         logger.info("enter");
 
         /* Shutdown managed resources */
diff --git a/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineManagerTest.json b/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineManagerTest.json
index db50e53c..d2d2075e 100644
--- a/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineManagerTest.json
+++ b/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineManagerTest.json
@@ -20,5 +20,17 @@
     "sources": [
         { "name": "source1-topic" },
         { "name": "source2-topic" }
-    ]
+    ],
+    "stats": {
+        "groupStat": {
+            "averageExecutionTime":0.0,
+            "birthTime": 0,
+            "lastExecutionTime":0,
+            "lastStart":0,
+            "policyExecutedCount":0,
+            "policyExecutedFailCount":0,
+            "policyExecutedSuccessCount":0,
+            "totalElapsedTime": 0.0
+        },"subgroupStats":{}
+    }
 }
diff --git a/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineTestAdd.json b/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineTestAdd.json
index 7a247d32..cbd696da 100644
--- a/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineTestAdd.json
+++ b/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineTestAdd.json
@@ -16,5 +16,18 @@
     "features": [],
     "controllers": [
         "foo"
-    ]
+    ],
+    "stats": {
+        "groupStat": {
+            "averageExecutionTime": 0.0,
+            "birthTime": 0,
+            "lastExecutionTime": 0,
+            "lastStart": 0,
+            "policyExecutedCount": 0,
+            "policyExecutedFailCount": 0,
+            "policyExecutedSuccessCount": 0,
+            "totalElapsedTime": 0.0
+        },
+        "subgroupStats": {}
+    }
 }
diff --git a/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineTestConfig.json b/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineTestConfig.json
index 7f06f5a0..68c52da4 100644
--- a/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineTestConfig.json
+++ b/policy-management/src/test/resources/org/onap/policy/drools/system/PolicyEngineTestConfig.json
@@ -14,5 +14,18 @@
         }
     ],
     "features": [],
-    "controllers": []
+    "controllers": [],
+    "stats": {
+        "groupStat": {
+            "averageExecutionTime": 0.0,
+            "birthTime": 0,
+            "lastExecutionTime": 0,
+            "lastStart": 0,
+            "policyExecutedCount": 0,
+            "policyExecutedFailCount": 0,
+            "policyExecutedSuccessCount": 0,
+            "totalElapsedTime": 0.0
+        },
+        "subgroupStats": {}
+    }
 }
diff --git a/policy-utils/src/main/java/org/onap/policy/drools/metrics/Metric.java b/policy-utils/src/main/java/org/onap/policy/drools/metrics/Metric.java
index 189371e5..01ff58f5 100644
--- a/policy-utils/src/main/java/org/onap/policy/drools/metrics/Metric.java
+++ b/policy-utils/src/main/java/org/onap/policy/drools/metrics/Metric.java
@@ -48,9 +48,20 @@ public class Metric {
      */
     protected static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS+00:00";
 
-    protected static final String HOSTNAME = NetworkUtil.getHostname();
-    protected static final String HOSTIP = NetworkUtil.getHostIp();
-    protected static final String HOST_TYPE = "PDP-D";
+    /**
+     * Host name.
+     */
+    public static final String HOSTNAME = NetworkUtil.getHostname();
+
+    /**
+     * Host IP address.
+     */
+    public static final String HOSTIP = NetworkUtil.getHostIp();
+
+    /**
+     * Host Type.
+     */
+    public static final String HOST_TYPE = "PDP-D";
 
     /* transaction inheritable fields */
 
@@ -88,6 +99,8 @@ public class Metric {
     private String customField3;
     private String customField4;
 
+    private boolean success = false;
+
     /**
      * converts time to timestamp with format yyyy-MM-dd'T'HH:mm:ss.SSS+00:00.
      */
diff --git a/policy-utils/src/main/java/org/onap/policy/drools/utils/logging/MdcTransactionConstants.java b/policy-utils/src/main/java/org/onap/policy/drools/utils/logging/MdcTransactionConstants.java
index 936449c8..1c98eacd 100644
--- a/policy-utils/src/main/java/org/onap/policy/drools/utils/logging/MdcTransactionConstants.java
+++ b/policy-utils/src/main/java/org/onap/policy/drools/utils/logging/MdcTransactionConstants.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * policy-utils
  * ================================================================================
- * Copyright (C) 2019-2020 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2019-2021 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,8 +19,6 @@
 
 package org.onap.policy.drools.utils.logging;
 
-import org.onap.policy.common.utils.network.NetworkUtil;
-
 public class MdcTransactionConstants {
     /*
      * The fields must match the naming given at
@@ -164,21 +162,6 @@ public class MdcTransactionConstants {
     public static final String CUSTOM_FIELD4 = "CustomField4";
 
     /**
-     * Default Service Name.
-     */
-    public static final String DEFAULT_SERVICE_NAME = "PDP-D";
-
-    /**
-     * Default Host Name.
-     */
-    public static final String DEFAULT_HOSTNAME = NetworkUtil.getHostname();
-
-    /**
-     * Default Host IP.
-     */
-    public static final String DEFAULT_HOSTIP = NetworkUtil.getHostIp();
-
-    /**
      * Status Code Complete.
      */
     public static final String STATUS_CODE_COMPLETE = "COMPLETE";
diff --git a/policy-utils/src/main/java/org/onap/policy/drools/utils/logging/MdcTransactionImpl.java b/policy-utils/src/main/java/org/onap/policy/drools/utils/logging/MdcTransactionImpl.java
index fa3c0a58..f4f39edd 100644
--- a/policy-utils/src/main/java/org/onap/policy/drools/utils/logging/MdcTransactionImpl.java
+++ b/policy-utils/src/main/java/org/onap/policy/drools/utils/logging/MdcTransactionImpl.java
@@ -26,9 +26,6 @@ import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.CUSTO
 import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.CUSTOM_FIELD2;
 import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.CUSTOM_FIELD3;
 import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.CUSTOM_FIELD4;
-import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.DEFAULT_HOSTIP;
-import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.DEFAULT_HOSTNAME;
-import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.DEFAULT_SERVICE_NAME;
 import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.ELAPSED_TIME;
 import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.END_TIMESTAMP;
 import static org.onap.policy.drools.utils.logging.MdcTransactionConstants.INSTANCE_UUID;
@@ -89,11 +86,11 @@ class MdcTransactionImpl implements MdcTransaction {
         setRequestId(requestId);
         setPartner(partner);
 
-        setServiceName(DEFAULT_SERVICE_NAME);
-        setServer(DEFAULT_HOSTNAME);
-        setServerIpAddress(DEFAULT_HOSTIP);
-        setServerFqdn(DEFAULT_HOSTNAME);
-        setVirtualServerName(DEFAULT_HOSTNAME);
+        setServiceName(Metric.HOST_TYPE);
+        setServer(Metric.HOSTNAME);
+        setServerIpAddress(Metric.HOSTIP);
+        setServerFqdn(Metric.HOSTNAME);
+        setVirtualServerName(Metric.HOSTNAME);
 
         setStartTime(Instant.now());
     }
@@ -203,12 +200,10 @@ class MdcTransactionImpl implements MdcTransaction {
         setMdc(BEGIN_TIMESTAMP, Metric.toTimestamp(execItem.getStartTime()));
         setMdc(END_TIMESTAMP, Metric.toTimestamp(execItem.getEndTime()));
 
-        if (execItem.getElapsedTime() != null) {
-            MDC.put(ELAPSED_TIME, String.valueOf(execItem.getElapsedTime()));
-        } else {
+        if (execItem.getElapsedTime() == null) {
             execItem.setElapsedTime(null);  // this computes elapsed time appropriately with start and end times
-            MDC.put(ELAPSED_TIME, String.valueOf(execItem.getElapsedTime()));
         }
+        MDC.put(ELAPSED_TIME, String.valueOf(execItem.getElapsedTime()));
 
         setMdc(SERVICE_INSTANCE_ID, execItem.getServiceInstanceId());
         setMdc(INSTANCE_UUID, execItem.getInstanceUuid());
@@ -377,11 +372,14 @@ class MdcTransactionImpl implements MdcTransaction {
     @Override
     public MdcTransaction setStatusCode(String statusCode) {
         execItem.setStatusCode(statusCode);
+        execItem.setSuccess(STATUS_CODE_COMPLETE.equals(execItem.getStatusCode()));
         return this;
     }
 
     @Override
     public MdcTransaction setStatusCode(boolean success) {
+        execItem.setSuccess(success);
+
         if (success) {
             execItem.setStatusCode(STATUS_CODE_COMPLETE);
         } else {