Merge "deploy stats in lifecycle module"

10 files changed, 221 insertions, 54 deletions

diff --git a/feature-lifecycle/pom.xml b/feature-lifecycle/pom.xml
index a7500f10..cdf6147d 100644
--- a/feature-lifecycle/pom.xml
+++ b/feature-lifecycle/pom.xml
@@ -2,7 +2,7 @@
   ============LICENSE_START=======================================================
   ONAP
   ================================================================================
-  Copyright (C) 2019-2020 AT&T Intellectual Property. All rights reserved.
+  Copyright (C) 2019-2021 AT&T Intellectual Property. All rights reserved.
   ================================================================================
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -143,9 +143,8 @@
         </dependency>
 
         <dependency>
-            <groupId>com.openpojo</groupId>
-            <artifactId>openpojo</artifactId>
-            <scope>test</scope>
+            <groupId>commons-beanutils</groupId>
+            <artifactId>commons-beanutils</artifactId>
         </dependency>
 
     </dependencies>
diff --git a/feature-lifecycle/src/main/java/org/onap/policy/drools/lifecycle/LifecycleFsm.java b/feature-lifecycle/src/main/java/org/onap/policy/drools/lifecycle/LifecycleFsm.java
index ff53a00a..e2c50d4f 100644
--- a/feature-lifecycle/src/main/java/org/onap/policy/drools/lifecycle/LifecycleFsm.java
+++ b/feature-lifecycle/src/main/java/org/onap/policy/drools/lifecycle/LifecycleFsm.java
@@ -21,8 +21,10 @@
 
 package org.onap.policy.drools.lifecycle;
 
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -38,6 +40,7 @@ import java.util.stream.Collectors;
 import lombok.Getter;
 import lombok.NonNull;
 import lombok.Setter;
+import org.apache.commons.beanutils.BeanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.onap.policy.common.capabilities.Startable;
 import org.onap.policy.common.endpoints.event.comm.Topic.CommInfrastructure;
@@ -50,12 +53,14 @@ import org.onap.policy.common.endpoints.listeners.ScoListener;
 import org.onap.policy.common.gson.annotation.GsonJsonIgnore;
 import org.onap.policy.common.utils.coder.StandardCoderObject;
 import org.onap.policy.common.utils.network.NetworkUtil;
+import org.onap.policy.drools.metrics.Metric;
 import org.onap.policy.drools.persistence.SystemPersistenceConstants;
 import org.onap.policy.drools.policies.DomainMaker;
 import org.onap.policy.drools.system.PolicyController;
 import org.onap.policy.drools.system.PolicyEngineConstants;
 import org.onap.policy.models.pdp.concepts.PdpResponseDetails;
 import org.onap.policy.models.pdp.concepts.PdpStateChange;
+import org.onap.policy.models.pdp.concepts.PdpStatistics;
 import org.onap.policy.models.pdp.concepts.PdpStatus;
 import org.onap.policy.models.pdp.concepts.PdpUpdate;
 import org.onap.policy.models.pdp.enums.PdpHealthStatus;
@@ -125,12 +130,10 @@ public class LifecycleFsm implements Startable {
     protected long statusTimerSeconds = DEFAULT_STATUS_TIMER_SECONDS;
 
     @Getter
-    @Setter
     private String group;
 
     @Getter
-    @Setter
-    protected String subgroup;
+    protected String subGroup;
 
     @Getter
     protected Set<String> mandatoryPolicyTypes = new HashSet<>();
@@ -141,26 +144,31 @@ public class LifecycleFsm implements Startable {
     @Getter
     protected final Map<ToscaConceptIdentifier, ToscaPolicy> policiesMap = new HashMap<>();
 
+    @Getter
+    protected final PdpStatistics stats = new PdpStatistics();
+
     /**
      * Constructor.
      */
     public LifecycleFsm() {
-        this.properties = SystemPersistenceConstants.getManager().getProperties(CONFIGURATION_PROPERTIES_NAME);
-        this.group = this.properties.getProperty(GROUP_NAME, DEFAULT_PDP_GROUP);
+        properties = SystemPersistenceConstants.getManager().getProperties(CONFIGURATION_PROPERTIES_NAME);
+        setGroup(properties.getProperty(GROUP_NAME, DEFAULT_PDP_GROUP));
 
-        this.policyTypesMap.put(POLICY_TYPE_DROOLS_NATIVE_CONTROLLER,
+        policyTypesMap.put(POLICY_TYPE_DROOLS_NATIVE_CONTROLLER,
                 new PolicyTypeNativeDroolsController(this, POLICY_TYPE_DROOLS_NATIVE_CONTROLLER));
-        this.policyTypesMap.put(
+        policyTypesMap.put(
                 POLICY_TYPE_DROOLS_NATIVE_RULES,
                  new PolicyTypeNativeArtifactController(this, POLICY_TYPE_DROOLS_NATIVE_RULES));
 
-        String commaSeparatedPolicyTypes = this.properties.getProperty(MANDATORY_POLICY_TYPES);
+        String commaSeparatedPolicyTypes = properties.getProperty(MANDATORY_POLICY_TYPES);
         if (!StringUtils.isBlank(commaSeparatedPolicyTypes)) {
-            Collections.addAll(this.mandatoryPolicyTypes, commaSeparatedPolicyTypes.split("\\s*,\\s*"));
+            Collections.addAll(mandatoryPolicyTypes, commaSeparatedPolicyTypes.split("\\s*,\\s*"));
         }
 
         logger.info("The mandatory Policy Types are {}. Compliance is {}",
-                this.mandatoryPolicyTypes, this.isMandatoryPolicyTypesCompliant());
+                mandatoryPolicyTypes, isMandatoryPolicyTypesCompliant());
+
+        stats.setPdpInstanceId(Metric.HOSTNAME);
     }
 
     @GsonJsonIgnore
@@ -180,6 +188,21 @@ public class LifecycleFsm implements Startable {
         return state.state();
     }
 
+    /**
+     * set group.
+     */
+    public synchronized void setGroup(String group) {
+        this.group = group;
+        this.stats.setPdpGroupName(group);
+    }
+
+    /**
+     * set subgroup.
+     */
+    public synchronized void setSubGroup(String subGroup) {
+        this.subGroup = subGroup;
+        this.stats.setPdpSubGroupName(subGroup);
+    }
 
     /* ** FSM events - entry points of events into the FSM ** */
 
@@ -274,7 +297,8 @@ public class LifecycleFsm implements Startable {
         logger.info("lifecycle event: update");
         return state.update(update);
     }
-    /* ** FSM State Actions ** */
+
+    /* FSM State Actions (executed sequentially) */
 
     protected boolean startAction() {
         if (isAlive()) {
@@ -300,7 +324,7 @@ public class LifecycleFsm implements Startable {
     }
 
     protected boolean statusAction() {
-        return statusAction(state(), null);
+        return statusAction(null);
     }
 
     protected boolean statusAction(PdpResponseDetails response) {
@@ -314,17 +338,13 @@ public class LifecycleFsm implements Startable {
 
         PdpStatus status = statusPayload(state);
         if (response != null) {
-            status.setRequestId(response.getResponseTo());     // for standard logging of transactions
+            status.setRequestId(response.getResponseTo());
             status.setResponse(response);
         }
 
         return client.send(status);
     }
 
-    protected void setSubGroupAction(String subgroup) {
-        this.subgroup = subgroup;
-    }
-
     protected synchronized void transitionToAction(@NonNull LifecycleState newState) {
         state = newState;
     }
@@ -403,14 +423,58 @@ public class LifecycleFsm implements Startable {
     }
 
     protected void deployedPolicyAction(@NonNull ToscaPolicy policy) {
-        policiesMap.put(policy.getIdentifier(), policy);
+        policiesMap.computeIfAbsent(policy.getIdentifier(), key -> {
+            // avoid counting reapplies in a second pass when a mix of native and non-native
+            // policies are present.
+            getStats().setPolicyDeployCount(getStats().getPolicyDeployCount() + 1);
+            getStats().setPolicyDeploySuccessCount(getStats().getPolicyDeploySuccessCount() + 1);
+            return policy;
+        });
     }
 
     protected void undeployedPolicyAction(@NonNull ToscaPolicy policy) {
-        policiesMap.remove(policy.getIdentifier());
+        policiesMap.computeIfPresent(policy.getIdentifier(), (key, value) -> {
+            // avoid counting reapplies in a second pass when a mix of native and non-native
+            // policies are present.
+            getStats().setPolicyDeployCount(getStats().getPolicyDeployCount() + 1);
+            getStats().setPolicyDeploySuccessCount(getStats().getPolicyDeploySuccessCount() + 1);
+            return null;
+        });
+    }
+
+    protected void failedDeployPolicyAction(@NonNull ToscaPolicy failedPolicy) {    // NOSONAR
+        getStats().setPolicyDeployCount(getStats().getPolicyDeployCount() + 1);
+        getStats().setPolicyDeployFailCount(getStats().getPolicyDeployFailCount() + 1);
+    }
+
+    protected void failedUndeployPolicyAction(ToscaPolicy failedPolicy) {
+        failedDeployPolicyAction(failedPolicy);
+        policiesMap.remove(failedPolicy.getIdentifier());
+    }
+
+    protected void updateDeployCountsAction(Long deployCount, Long deploySuccesses, Long deployFailures) {
+        PdpStatistics statistics = getStats();
+        if (deployCount != null) {
+            statistics.setPolicyDeployCount(deployCount);
+        }
+
+        if (deploySuccesses != null) {
+            statistics.setPolicyDeploySuccessCount(deploySuccesses);
+        }
+
+        if (deployFailures != null) {
+            statistics.setPolicyDeployFailCount(deployFailures);
+        }
+    }
+
+    protected void resetDeployCountsAction() {
+        getStats().setPolicyDeployCount(0);
+        getStats().setPolicyDeployFailCount(0);
+        getStats().setPolicyDeploySuccessCount(0);
     }
 
     protected List<ToscaPolicy> resetPoliciesAction() {
+        resetDeployCountsAction();
         List<ToscaPolicy> policies = new ArrayList<>(getActivePolicies());
         policiesMap.clear();
         return policies;
@@ -509,18 +573,32 @@ public class LifecycleFsm implements Startable {
         scheduler.shutdownNow();
     }
 
-    private PdpStatus statusPayload(@NonNull PdpState state) {
+    protected PdpStatus statusPayload(@NonNull PdpState state) {
         PdpStatus status = new PdpStatus();
         status.setName(name);
         status.setPdpGroup(group);
-        status.setPdpSubgroup(subgroup);
+        status.setPdpSubgroup(subGroup);
         status.setState(state);
         status.setHealthy(isAlive() ? PdpHealthStatus.HEALTHY : PdpHealthStatus.NOT_HEALTHY);
         status.setPdpType("drools");
         status.setPolicies(new ArrayList<>(policiesMap.keySet()));
+        status.setStatistics(statisticsPayload());
         return status;
     }
 
+    private PdpStatistics statisticsPayload() {
+        PdpStatistics updateStats = new PdpStatistics(stats);
+        updateStats.setTimeStamp(new Date());
+
+        try {
+            BeanUtils.copyProperties(updateStats, PolicyEngineConstants.getManager().getStats());
+        } catch (IllegalAccessException | InvocationTargetException ex) {
+            logger.debug("statistics mapping failure", ex);
+        }
+
+        return updateStats;
+    }
+
     private boolean source() {
         List<TopicSource> sources = TopicEndpointManager.getManager().addTopicSources(properties);
         if (sources.isEmpty()) {
@@ -560,7 +638,7 @@ public class LifecycleFsm implements Startable {
 
         return name == null && group != null
             && Objects.equals(group, getGroup())
-            && Objects.equals(subgroup, getSubgroup());
+            && Objects.equals(subgroup, getSubGroup());
     }
 
     /* **** IO listeners ***** */
diff --git a/feature-lifecycle/src/main/java/org/onap/policy/drools/lifecycle/LifecycleStateRunning.java b/feature-lifecycle/src/main/java/org/onap/policy/drools/lifecycle/LifecycleStateRunning.java
index a4a5d66d..dacb6058 100644
--- a/feature-lifecycle/src/main/java/org/onap/policy/drools/lifecycle/LifecycleStateRunning.java
+++ b/feature-lifecycle/src/main/java/org/onap/policy/drools/lifecycle/LifecycleStateRunning.java
@@ -122,6 +122,9 @@ public abstract class LifecycleStateRunning extends LifecycleStateDefault {
 
     @Override
     public boolean update(@NonNull PdpUpdate update) {
+        // UPDATE messages contain the complete universe of active policies in this PDP-D
+        // regardless if they were already functioning in the PDP-D or new.
+
         synchronized (fsm) {
             if (update.getPdpHeartbeatIntervalMs() != null
                     && !fsm.setStatusIntervalAction(update.getPdpHeartbeatIntervalMs() / 1000)) {
@@ -132,7 +135,7 @@ public abstract class LifecycleStateRunning extends LifecycleStateDefault {
 
             // update subgroup if applicable per update message
 
-            fsm.setSubGroupAction(update.getPdpSubgroup());
+            fsm.setSubGroup(update.getPdpSubgroup());
 
             // snapshot the active policies previous to apply the new set of active
             // policies as given by the PAP in the update message
@@ -168,9 +171,12 @@ public abstract class LifecycleStateRunning extends LifecycleStateDefault {
 
             failedPolicies.addAll(reApplyNonNativePolicies(activePoliciesPreUpdateMap));
 
-            return fsm.statusAction(response(update.getRequestId(),
-                            (failedPolicies.isEmpty()) ? PdpResponseStatus.SUCCESS : PdpResponseStatus.FAIL,
-                            fsm.getPolicyIdsMessage(failedPolicies))) && failedPolicies.isEmpty();
+            PdpResponseDetails response =
+                response(update.getRequestId(),
+                        (failedPolicies.isEmpty()) ? PdpResponseStatus.SUCCESS : PdpResponseStatus.FAIL,
+                        fsm.getPolicyIdsMessage(failedPolicies));
+
+            return fsm.statusAction(response) && failedPolicies.isEmpty();
         }
     }
 
@@ -195,12 +201,13 @@ public abstract class LifecycleStateRunning extends LifecycleStateDefault {
         List<ToscaPolicy> failedUndeployPolicies = undeployPolicies(policies);
         if (!failedUndeployPolicies.isEmpty()) {
             logger.warn("update-policies: undeployment failures: {}", fsm.getPolicyIdsMessage(failedUndeployPolicies));
-            failedUndeployPolicies.forEach(fsm::undeployedPolicyAction);
+            failedUndeployPolicies.forEach(fsm::failedUndeployPolicyAction);
         }
 
         List<ToscaPolicy> failedDeployPolicies = deployPolicies(policies);
         if (!failedDeployPolicies.isEmpty()) {
             logger.warn("update-policies: deployment failures: {}", fsm.getPolicyIdsMessage(failedDeployPolicies));
+            failedDeployPolicies.forEach(fsm::failedDeployPolicyAction);
         }
 
         return Pair.of(failedUndeployPolicies, failedDeployPolicies);
diff --git a/feature-lifecycle/src/main/java/org/onap/policy/drools/server/restful/RestLifecycleManager.java b/feature-lifecycle/src/main/java/org/onap/policy/drools/server/restful/RestLifecycleManager.java
index c1b0aab0..e9969094 100644
--- a/feature-lifecycle/src/main/java/org/onap/policy/drools/server/restful/RestLifecycleManager.java
+++ b/feature-lifecycle/src/main/java/org/onap/policy/drools/server/restful/RestLifecycleManager.java
@@ -101,7 +101,7 @@ public class RestLifecycleManager {
     @ApiOperation(value = "Retrieves the Lifecycle subgroup",
             notes = "Lifecycle Subgroup", response = String.class)
     public Response subgroup() {
-        return Response.status(Response.Status.OK).entity(LifecycleFeature.getFsm().getSubgroup()).build();
+        return Response.status(Response.Status.OK).entity(LifecycleFeature.getFsm().getSubGroup()).build();
     }
 
     /**
@@ -114,8 +114,8 @@ public class RestLifecycleManager {
             notes = "Lifecycle Subgroup", response = String.class)
     public Response subgroup(
         @ApiParam(value = "Subgroup", required = true) @PathParam("subgroup") String subgroup) {
-        LifecycleFeature.getFsm().setSubgroup(subgroup);
-        return Response.status(Response.Status.OK).entity(LifecycleFeature.getFsm().getSubgroup()).build();
+        LifecycleFeature.getFsm().setSubGroup(subgroup);
+        return Response.status(Response.Status.OK).entity(LifecycleFeature.getFsm().getSubGroup()).build();
     }
 
     /**
@@ -153,7 +153,7 @@ public class RestLifecycleManager {
 
         PdpStateChange change = new PdpStateChange();
         change.setPdpGroup(LifecycleFeature.getFsm().getGroup());
-        change.setPdpSubgroup(LifecycleFeature.getFsm().getSubgroup());
+        change.setPdpSubgroup(LifecycleFeature.getFsm().getSubGroup());
         change.setState(PdpState.valueOf(state));
         change.setName(LifecycleFeature.getFsm().getName());
 
@@ -440,7 +440,7 @@ public class RestLifecycleManager {
         PdpUpdate update = new PdpUpdate();
         update.setName(LifecycleFeature.getFsm().getName());
         update.setPdpGroup(LifecycleFeature.getFsm().getGroup());
-        update.setPdpSubgroup(LifecycleFeature.getFsm().getSubgroup());
+        update.setPdpSubgroup(LifecycleFeature.getFsm().getSubGroup());
         update.setPolicies(policies);
         return update;
     }
diff --git a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleFsmTest.java b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleFsmTest.java
index a77b03f5..9dda01df 100644
--- a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleFsmTest.java
+++ b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleFsmTest.java
@@ -21,6 +21,7 @@
 package org.onap.policy.drools.lifecycle;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -33,11 +34,14 @@ import org.junit.Before;
 import org.junit.Test;
 import org.onap.policy.common.utils.coder.CoderException;
 import org.onap.policy.common.utils.coder.StandardCoder;
+import org.onap.policy.common.utils.network.NetworkUtil;
 import org.onap.policy.common.utils.resources.ResourceUtils;
 import org.onap.policy.common.utils.time.PseudoScheduledExecutorService;
 import org.onap.policy.common.utils.time.TestTimeMulti;
 import org.onap.policy.drools.persistence.SystemPersistenceConstants;
 import org.onap.policy.drools.utils.logging.LoggerUtil;
+import org.onap.policy.models.pdp.concepts.PdpStatus;
+import org.onap.policy.models.pdp.enums.PdpState;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate;
 
@@ -159,6 +163,54 @@ public class LifecycleFsmTest {
         assertEquals(List.of(artifactPolicy, artifact2Policy), fsm.getNativeArtifactPolicies(deployedPolicies));
     }
 
+    @Test
+    public void testSetGroup() {
+        fsm.setGroup("bar");
+        assertEquals("bar", fsm.getGroup());
+        assertEquals("bar", fsm.getStats().getPdpGroupName());
+    }
+
+    @Test
+    public void testSetSubGroup() {
+        fsm.setSubGroup("foo");
+        assertEquals("foo", fsm.getSubGroup());
+        assertEquals("foo", fsm.getStats().getPdpSubGroupName());
+    }
+
+    @Test
+    public void testDeployedPolicyAction() {
+        fsm.deployedPolicyAction(controllerPolicy);
+        assertEquals(1, fsm.getStats().getPolicyDeploySuccessCount());
+
+        fsm.undeployedPolicyAction(controllerPolicy);
+        assertEquals(2, fsm.getStats().getPolicyDeploySuccessCount());
+    }
+
+    @Test
+    public void testRestDeployCountsAction() {
+        deployAllPolicies();
+        assertEquals(8, fsm.getStats().getPolicyDeploySuccessCount());
+
+        fsm.resetDeployCountsAction();
+        assertEquals(0, fsm.getStats().getPolicyDeploySuccessCount());
+        assertEquals(0, fsm.getStats().getPolicyDeployFailCount());
+        assertEquals(0, fsm.getStats().getPolicyDeployCount());
+    }
+
+    @Test
+    public void testStatusPayload() {
+        fsm.updateDeployCountsAction(8L, 6L, 2L);
+        PdpStatus status = fsm.statusPayload(PdpState.ACTIVE);
+
+        assertEquals(fsm.getGroup(), status.getStatistics().getPdpGroupName());
+        assertEquals(fsm.getSubGroup(), status.getStatistics().getPdpSubGroupName());
+        assertEquals(NetworkUtil.getHostname(), status.getStatistics().getPdpInstanceId());
+        assertEquals(6, status.getStatistics().getPolicyDeploySuccessCount());
+        assertEquals(2, status.getStatistics().getPolicyDeployFailCount());
+        assertEquals(8, status.getStatistics().getPolicyDeployCount());
+        assertNotNull(status.getStatistics().getTimeStamp());
+    }
+
     protected void deployAllPolicies() {
         fsm.deployedPolicyAction(controllerPolicy);
         fsm.deployedPolicyAction(controller2Policy);
diff --git a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleFsmUpdateTest.java b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleFsmUpdateTest.java
index 01500576..9ae43e21 100644
--- a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleFsmUpdateTest.java
+++ b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleFsmUpdateTest.java
@@ -186,7 +186,7 @@ public class LifecycleFsmUpdateTest {
         change.setState(PdpState.ACTIVE);
         change.setName(fsm.getName());
 
-        fsm.setSubGroupAction("a");
+        fsm.setSubGroup("a");
         fsm.source.offer(new StandardCoder().encode(change));
 
         assertEquals(0, fsm.getPoliciesMap().size());
@@ -212,21 +212,25 @@ public class LifecycleFsmUpdateTest {
         assertEquals(0, fsm.getPoliciesMap().size());
         assertEquals("ACTIVE", fsm.state().toString());
         assertEquals(0, PolicyControllerConstants.getFactory().inventory().size());
+        verifyDeployStats(0, 0, 0);
 
         // native controller policy - deploy
         // Delta: +controllerPolicy
         assertTrue(fsm.update(getPdpUpdate(List.of(controllerPolicy))));
         verifyActivePolicies(List.of(controllerPolicy));
+        verifyDeployStats(1, 1, 0);
 
         // no policies - undeploy
         // Delta: []
         assertTrue(fsm.update(getPdpUpdate(Collections.emptyList())));
         verifyActivePolicies(Collections.emptyList());
+        verifyDeployStats(2, 2, 0);
 
         // native controller + artifact policy (out of order) - deploy
         // Delta: +artifactPolicy, +controllerPolicy
         assertTrue(fsm.update(getPdpUpdate(List.of(artifactPolicy, controllerPolicy))));
         verifyActivePolicies(List.of(artifactPolicy, controllerPolicy));
+        verifyDeployStats(4, 4, 0);
 
         // attempt to deploy opPolicy but invalid controller
         // Delta: +opPolicy
@@ -235,55 +239,65 @@ public class LifecycleFsmUpdateTest {
         assertFalse(fsm.getActivePolicies().contains(opPolicy));
         verifyExists(false, "lifecycle", List.of(opPolicy));
         verifyActivePolicies(List.of(artifactPolicy, controllerPolicy));
+        verifyDeployStats(5, 4, 1);
 
         // Delta: +opPolicy
         opPolicy.getProperties().remove("controllerName");
         assertTrue(fsm.update(getPdpUpdate(List.of(artifactPolicy, opPolicy, controllerPolicy))));
         verifyExists(true, "lifecycle", List.of(opPolicy));
         verifyActivePolicies(List.of(opPolicy, artifactPolicy, controllerPolicy));
+        verifyDeployStats(6, 5, 1);
 
         // Delta: -opPolicy
         assertTrue(fsm.update(getPdpUpdate(List.of(artifactPolicy, controllerPolicy))));
         assertFalse(PolicyControllerConstants.getFactory().get("lifecycle").getDrools().exists(opPolicy));
         verifyActivePolicies(List.of(controllerPolicy, artifactPolicy));
+        verifyDeployStats(7, 6, 1);
 
         // Delta: -artifactPolicy
         assertTrue(fsm.update(getPdpUpdate(List.of(controllerPolicy))));
         assertFalse(PolicyControllerConstants.getFactory().get("lifecycle").getDrools().isBrained());
         verifyActivePolicies(List.of(controllerPolicy));
+        verifyDeployStats(8, 7, 1);
 
         // Delta: -controllerPolicy
         assertTrue(fsm.update(getPdpUpdate(Collections.emptyList())));
         assertThatIllegalArgumentException().isThrownBy(() -> PolicyControllerConstants.getFactory().get("lifecycle"));
         assertEquals(0, PolicyControllerConstants.getFactory().inventory().size());
         verifyActivePolicies(Collections.emptyList());
+        verifyDeployStats(9, 8, 1);
 
         // Delta: +controllerPolicy, +artifactPolicy, and +opPolicy
         assertTrue(fsm.update(getPdpUpdate(List.of(opPolicy, artifactPolicy, controllerPolicy))));
         verifyExists(true, "lifecycle", List.of(opPolicy));
         verifyActivePolicies(List.of(opPolicy, artifactPolicy, controllerPolicy));
+        verifyDeployStats(12, 11, 1);
 
         // Delta: -artifactPolicy
         assertTrue(fsm.update(getPdpUpdate(List.of(opPolicy, controllerPolicy))));
         assertEquals(1, PolicyControllerConstants.getFactory().inventory().size());
         assertFalse(PolicyControllerConstants.getFactory().get("lifecycle").getDrools().isBrained());
+        verifyDeployStats(13, 12, 1);
 
         // Delta: +artifactPolicy
         // from deltas, all delta updates should be successfully applied
         assertTrue(fsm.update(getPdpUpdate(List.of(opPolicy, controllerPolicy, artifactPolicy))));
         verifyExists(true, "lifecycle", List.of(opPolicy));
         verifyActivePolicies(List.of(opPolicy, artifactPolicy, controllerPolicy));
+        verifyDeployStats(14, 13, 1);
 
         // Delta: -controllerPolicy
         // from deltas, all delta updates should be successfully applied
         assertTrue(fsm.update(getPdpUpdate(List.of(opPolicy, artifactPolicy))));
         assertEquals(0, PolicyControllerConstants.getFactory().inventory().size());
+        verifyDeployStats(15, 14, 1);
 
         // Delta: +controllerPolicy
         // from deltas, all delta updates should be successfully applied
         assertTrue(fsm.update(getPdpUpdate(List.of(opPolicy, controllerPolicy, artifactPolicy))));
         verifyExists(true, "lifecycle", List.of(opPolicy));
         verifyActivePolicies(List.of(opPolicy, artifactPolicy, controllerPolicy));
+        verifyDeployStats(16, 15, 1);
 
         // Delta: +op2Policy, +controller2Policy
         // from deltas, all delta updates should be successfully applied
@@ -293,6 +307,7 @@ public class LifecycleFsmUpdateTest {
         verifyExists(true, "lifecycle", List.of(opPolicy, op2Policy));
         assertFalse(PolicyControllerConstants.getFactory().get("foo").getDrools().isBrained());
         verifyActivePolicies(List.of(opPolicy, artifactPolicy, controllerPolicy, op2Policy, controller2Policy));
+        verifyDeployStats(18, 17, 1);
 
         // Delta: +artifact2policy, +valPolicy, +unvalPolicy
         // from deltas, all delta updates should be successfully applied
@@ -304,6 +319,7 @@ public class LifecycleFsmUpdateTest {
         verifyExists(false, "foo", List.of(opPolicy, op2Policy));
         verifyActivePolicies(List.of(opPolicy, artifactPolicy, controllerPolicy, op2Policy,
                 controller2Policy, valPolicy, unvalPolicy, artifact2Policy));
+        verifyDeployStats(21, 20, 1);
 
         // Delta: -artifact2Policy, +valPolicy
         // from deltas, all delta updates should be successfully applied, and unvalPolicy disabled
@@ -315,6 +331,7 @@ public class LifecycleFsmUpdateTest {
         assertFalse(PolicyControllerConstants.getFactory().get("foo").getDrools().isBrained());
         verifyActivePolicies(List.of(opPolicy, artifactPolicy, controllerPolicy, op2Policy,
                 controller2Policy, valPolicy));
+        verifyDeployStats(23, 22, 1);
 
         // Delta: +artifact2Policy
         // from deltas, all delta updates should be successfully applied, opPolicy, op2Policy and unvalPolicy
@@ -328,6 +345,7 @@ public class LifecycleFsmUpdateTest {
         verifyExists(false, "foo", List.of(opPolicy, op2Policy, unvalPolicy));
         verifyActivePolicies(List.of(opPolicy, artifactPolicy, controllerPolicy, op2Policy,
                 controller2Policy, valPolicy, artifact2Policy));
+        verifyDeployStats(24, 23, 1);
 
         // Delta: -controllerPolicy, +artifactPolicy, +unvalPolicy
         // from deltas, all delta updates should be successful
@@ -341,6 +359,7 @@ public class LifecycleFsmUpdateTest {
         verifyActivePoliciesWithDisables(
                 List.of(opPolicy, op2Policy, controller2Policy, valPolicy, artifact2Policy, unvalPolicy),
                 List.of(opPolicy.getIdentifier(), op2Policy.getIdentifier()));
+        verifyDeployStats(27, 26, 1);
 
         // Delta: -opPolicy, -op2Policy, -controller2Policy, -valPolicy, -artifact2Policy, -unvalPolicy
         // from deltas, -opPolicy and -op2Policy undeploys will fail since there is not controller with that
@@ -350,6 +369,7 @@ public class LifecycleFsmUpdateTest {
         assertThatIllegalArgumentException().isThrownBy(() -> PolicyControllerConstants.getFactory().get("foo"));
         assertEquals(0, PolicyControllerConstants.getFactory().inventory().size());
         verifyActivePolicies(Collections.emptyList());
+        verifyDeployStats(33, 30, 3);
 
         fsm.shutdown();
     }
@@ -517,6 +537,12 @@ public class LifecycleFsmUpdateTest {
                 candidateNativeArtifactPolicies.isEmpty());
     }
 
+    protected void verifyDeployStats(long count, long success, long fail) {
+        assertEquals(count, fsm.getStats().getPolicyDeployCount());
+        assertEquals(success, fsm.getStats().getPolicyDeploySuccessCount());
+        assertEquals(fail, fsm.getStats().getPolicyDeployFailCount());
+    }
+
     protected List<NativeArtifactPolicy> getNativeArtifactPoliciesBut(List<ToscaPolicy> nativePolicies,
                                                                       String controllerName) {
         return
diff --git a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStateActivePoliciesTest.java b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStateActivePoliciesTest.java
index d961dce3..2b105f91 100644
--- a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStateActivePoliciesTest.java
+++ b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStateActivePoliciesTest.java
@@ -80,7 +80,7 @@ public class LifecycleStateActivePoliciesTest extends LifecycleStateRunningTest
         change.setState(PdpState.ACTIVE);
         change.setName(fsm.getName());
 
-        fsm.setSubGroupAction("a");
+        fsm.setSubGroup("a");
         fsm.source.offer(new StandardCoder().encode(change));
         controllerSupport.getController().start();
     }
diff --git a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStateActiveTest.java b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStateActiveTest.java
index 8a5ea6e4..0ce1f821 100644
--- a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStateActiveTest.java
+++ b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStateActiveTest.java
@@ -79,7 +79,7 @@ public class LifecycleStateActiveTest extends LifecycleStateRunningTest {
         change.setState(PdpState.ACTIVE);
         change.setName(fsm.getName());
 
-        fsm.setSubGroupAction("a");
+        fsm.setSubGroup("a");
         fsm.source.offer(new StandardCoder().encode(change));
         controllerSupport.getController().start();
     }
@@ -102,7 +102,7 @@ public class LifecycleStateActiveTest extends LifecycleStateRunningTest {
     private void assertActive() {
         assertEquals(PdpState.ACTIVE, fsm.state());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertEquals("a", fsm.getSubgroup());
+        assertEquals("a", fsm.getSubGroup());
         assertTrue(fsm.isAlive());
         waitUntil(fsm.getStatusTimerSeconds() + 1, TimeUnit.SECONDS, isStatus(PdpState.ACTIVE));
     }
@@ -161,13 +161,13 @@ public class LifecycleStateActiveTest extends LifecycleStateRunningTest {
         fsm.source.offer(new StandardCoder().encode(change));
         assertEquals(PdpState.ACTIVE, fsm.state());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertNotEquals("b", fsm.getSubgroup());
+        assertNotEquals("b", fsm.getSubGroup());
 
         change.setName(fsm.getName());
         fsm.source.offer(new StandardCoder().encode(change));
         assertEquals(PdpState.ACTIVE, fsm.state());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertEquals("a", fsm.getSubgroup());
+        assertEquals("a", fsm.getSubGroup());
 
         change.setState(PdpState.SAFE);
         fsm.source.offer(new StandardCoder().encode(change));
@@ -201,7 +201,7 @@ public class LifecycleStateActiveTest extends LifecycleStateRunningTest {
 
         assertEquals(PdpState.ACTIVE, fsm.state());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertEquals("w", fsm.getSubgroup());
+        assertEquals("w", fsm.getSubGroup());
 
         ToscaPolicy toscaPolicyRestartV1 =
             getExamplesPolicy("policies/vCPE.policy.operational.input.tosca.json", "operational.restart");
diff --git a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStatePassiveTest.java b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStatePassiveTest.java
index afa89d68..6945f10a 100644
--- a/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStatePassiveTest.java
+++ b/feature-lifecycle/src/test/java/org/onap/policy/drools/lifecycle/LifecycleStatePassiveTest.java
@@ -164,7 +164,7 @@ public class LifecycleStatePassiveTest extends LifecycleStateRunningTest {
         assertEquals(PdpState.PASSIVE, fsm.state());
         assertEquals(interval, fsm.getStatusTimerSeconds());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertEquals("z", fsm.getSubgroup());
+        assertEquals("z", fsm.getSubGroup());
         assertBasicPassive();
 
         ToscaPolicy toscaPolicy =
@@ -177,7 +177,7 @@ public class LifecycleStatePassiveTest extends LifecycleStateRunningTest {
         assertEquals(PdpState.PASSIVE, fsm.state());
         assertEquals(interval, fsm.getStatusTimerSeconds());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertEquals("z", fsm.getSubgroup());
+        assertEquals("z", fsm.getSubGroup());
         assertBasicPassive();
 
         assertEquals(2, fsm.policyTypesMap.size());
@@ -191,7 +191,7 @@ public class LifecycleStatePassiveTest extends LifecycleStateRunningTest {
         assertEquals(PdpState.PASSIVE, fsm.state());
         assertEquals(interval, fsm.getStatusTimerSeconds());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertNull(fsm.getSubgroup());
+        assertNull(fsm.getSubGroup());
         assertBasicPassive();
         assertEquals(2, fsm.policyTypesMap.size());
         assertTrue(fsm.policiesMap.isEmpty());
@@ -204,7 +204,7 @@ public class LifecycleStatePassiveTest extends LifecycleStateRunningTest {
         assertEquals(PdpState.PASSIVE, fsm.state());
         assertEquals(interval, fsm.getStatusTimerSeconds());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertEquals("a", fsm.getSubgroup());
+        assertEquals("a", fsm.getSubGroup());
         assertBasicPassive();
         assertEquals(2, fsm.policyTypesMap.size());
         assertTrue(fsm.policiesMap.isEmpty());
@@ -220,7 +220,7 @@ public class LifecycleStatePassiveTest extends LifecycleStateRunningTest {
         assertEquals(PdpState.PASSIVE, fsm.state());
         assertEquals(interval, fsm.getStatusTimerSeconds());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertEquals("a", fsm.getSubgroup());
+        assertEquals("a", fsm.getSubGroup());
         assertBasicPassive();
         assertEquals(0, controllerSupport.getController().getDrools().factCount("junits"));
 
@@ -245,7 +245,7 @@ public class LifecycleStatePassiveTest extends LifecycleStateRunningTest {
         assertEquals(PdpState.PASSIVE, fsm.state());
         assertEquals(interval, fsm.getStatusTimerSeconds());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertNull(fsm.getSubgroup());
+        assertNull(fsm.getSubGroup());
         assertBasicPassive();
         assertEquals(0, controllerSupport.getController().getDrools().factCount("junits"));
 
@@ -266,7 +266,7 @@ public class LifecycleStatePassiveTest extends LifecycleStateRunningTest {
 
         assertEquals(PdpState.PASSIVE, fsm.state());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertNull(fsm.getSubgroup());
+        assertNull(fsm.getSubGroup());
 
         PdpUpdate update = new PdpUpdate();
         update.setName(NetworkUtil.getHostname());
@@ -289,7 +289,7 @@ public class LifecycleStatePassiveTest extends LifecycleStateRunningTest {
         assertEquals(fsm.policiesMap.get(toscaPolicy.getIdentifier()), toscaPolicy);
         assertEquals(PdpState.PASSIVE, fsm.state());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertEquals("a", fsm.getSubgroup());
+        assertEquals("a", fsm.getSubGroup());
         assertBasicPassive();
         assertEquals(0, controllerSupport.getController().getDrools().factCount("junits"));
 
@@ -299,7 +299,7 @@ public class LifecycleStatePassiveTest extends LifecycleStateRunningTest {
 
         assertEquals(PdpState.ACTIVE, fsm.state());
         assertEquals(LifecycleFsm.DEFAULT_PDP_GROUP, fsm.getGroup());
-        assertEquals("a", fsm.getSubgroup());
+        assertEquals("a", fsm.getSubGroup());
 
         waitUntil(5, TimeUnit.SECONDS, () -> controllerSupport.getController().getDrools().factCount("junits") == 1);
 
diff --git a/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStats.java b/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStats.java
index d6d8d500..add4c18c 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStats.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/stats/PolicyStats.java
@@ -23,10 +23,12 @@ package org.onap.policy.drools.stats;
 import static org.slf4j.LoggerFactory.getLogger;
 
 import java.time.Instant;
-import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.NonNull;
+import lombok.Setter;
+import lombok.Synchronized;
 import lombok.ToString;
 import org.onap.policy.drools.metrics.TransMetric;
 import org.slf4j.Logger;
@@ -35,7 +37,8 @@ import org.slf4j.Logger;
  * Basic policy execution statistics.
  */
 
-@Data
+@Setter(onMethod_ = @Synchronized)
+@Getter(onMethod_ = @Synchronized)
 @NoArgsConstructor
 @EqualsAndHashCode
 @ToString
@@ -86,6 +89,8 @@ public class PolicyStats {
     /**
      * add a stat transaction record.
      */
+
+    @Synchronized
     public void stat(@NonNull TransMetric trans) {
         policyExecutedCount++;
         if (trans.isSuccess()) {