Add timeout for ACM operations

User can retry the ACM state change order when ACM operations time out.

Issue-ID: POLICY-4716
Signed-off-by: zrrmmua <ramesh.murugan.iyer@est.tech>
Change-Id: Ie840fc13170ef235c6b6f3b26a47b7516f10296c
(cherry picked from commit 1d1908528e67d9763482cf9d5921456f19af31e7)

4 files changed, 69 insertions, 1 deletions

diff --git a/models/src/main/java/org/onap/policy/clamp/models/acm/concepts/StateChangeResult.java b/models/src/main/java/org/onap/policy/clamp/models/acm/concepts/StateChangeResult.java
index de4404fdd..c09b79df5 100644
--- a/models/src/main/java/org/onap/policy/clamp/models/acm/concepts/StateChangeResult.java
+++ b/models/src/main/java/org/onap/policy/clamp/models/acm/concepts/StateChangeResult.java
@@ -22,5 +22,6 @@ package org.onap.policy.clamp.models.acm.concepts;
 
 public enum StateChangeResult {
     NO_ERROR,
-    FAILED
+    FAILED,
+    TIMEOUT
 }
diff --git a/models/src/main/java/org/onap/policy/clamp/models/acm/persistence/provider/AcInstanceStateResolver.java b/models/src/main/java/org/onap/policy/clamp/models/acm/persistence/provider/AcInstanceStateResolver.java
index 659169b9f..e2f4fdf9b 100644
--- a/models/src/main/java/org/onap/policy/clamp/models/acm/persistence/provider/AcInstanceStateResolver.java
+++ b/models/src/main/java/org/onap/policy/clamp/models/acm/persistence/provider/AcInstanceStateResolver.java
@@ -50,6 +50,7 @@ public class AcInstanceStateResolver {
 
     private static final String NO_ERROR = StateChangeResult.NO_ERROR.name();
     private static final String FAILED = StateChangeResult.FAILED.name();
+    private static final String TIMEOUT = StateChangeResult.TIMEOUT.name();
 
     // list of results
     public static final String DEPLOY = DeployOrder.DEPLOY.name();
@@ -87,6 +88,22 @@ public class AcInstanceStateResolver {
 
         this.graph.put(new String[] {DEPLOY_NONE, LOCK, DEPLOYED, UNLOCKING, FAILED}, LOCK);
         this.graph.put(new String[] {DEPLOY_NONE, UNLOCK, DEPLOYED, UNLOCKING, FAILED}, UNLOCK);
+
+        // timeout
+        this.graph.put(new String[] {DEPLOY, LOCK_NONE, UNDEPLOYING, STATE_LOCKED_NONE, TIMEOUT}, DEPLOY);
+        this.graph.put(new String[] {UNDEPLOY, LOCK_NONE, UNDEPLOYING, STATE_LOCKED_NONE, TIMEOUT}, UNDEPLOY);
+        this.graph.put(new String[] {UNDEPLOY, LOCK_NONE, UPDATING, STATE_LOCKED_NONE, TIMEOUT}, UNDEPLOY);
+
+        this.graph.put(new String[] {DEPLOY, LOCK_NONE, DEPLOYING, STATE_LOCKED_NONE, TIMEOUT}, DEPLOY);
+        this.graph.put(new String[] {UNDEPLOY, LOCK_NONE, DEPLOYING, STATE_LOCKED_NONE, TIMEOUT}, UNDEPLOY);
+
+        this.graph.put(new String[] {DELETE, LOCK_NONE, DELETING, LOCK_NONE, TIMEOUT}, DELETE);
+
+        this.graph.put(new String[] {DEPLOY_NONE, UNLOCK, DEPLOYED, LOCKING, TIMEOUT}, UNLOCK);
+        this.graph.put(new String[] {DEPLOY_NONE, LOCK, DEPLOYED, LOCKING, TIMEOUT}, LOCK);
+
+        this.graph.put(new String[] {DEPLOY_NONE, LOCK, DEPLOYED, UNLOCKING, TIMEOUT}, LOCK);
+        this.graph.put(new String[] {DEPLOY_NONE, UNLOCK, DEPLOYED, UNLOCKING, TIMEOUT}, UNLOCK);
     }
 
     /**
diff --git a/runtime-acm/src/main/java/org/onap/policy/clamp/acm/runtime/supervision/SupervisionScanner.java b/runtime-acm/src/main/java/org/onap/policy/clamp/acm/runtime/supervision/SupervisionScanner.java
index 803bc86ca..41c7b1c74 100644
--- a/runtime-acm/src/main/java/org/onap/policy/clamp/acm/runtime/supervision/SupervisionScanner.java
+++ b/runtime-acm/src/main/java/org/onap/policy/clamp/acm/runtime/supervision/SupervisionScanner.java
@@ -31,6 +31,7 @@ import org.onap.policy.clamp.acm.runtime.supervision.comm.AutomationCompositionS
 import org.onap.policy.clamp.models.acm.concepts.AutomationComposition;
 import org.onap.policy.clamp.models.acm.concepts.DeployState;
 import org.onap.policy.clamp.models.acm.concepts.ParticipantUtils;
+import org.onap.policy.clamp.models.acm.concepts.StateChangeResult;
 import org.onap.policy.clamp.models.acm.persistence.provider.AcDefinitionProvider;
 import org.onap.policy.clamp.models.acm.persistence.provider.AutomationCompositionProvider;
 import org.onap.policy.clamp.models.acm.utils.AcmUtils;
@@ -111,6 +112,13 @@ public class SupervisionScanner {
             return;
         }
 
+        if (automationCompositionCounter.isFault(automationComposition.getInstanceId())
+                && StateChangeResult.NO_ERROR.equals(automationComposition.getStateChangeResult())) {
+            // retry by the user
+            LOGGER.debug("clearing fault for the ac instance");
+            clearFaultAndCounter(automationComposition);
+        }
+
         var completed = true;
         var minSpNotCompleted = 1000; // min startPhase not completed
         var maxSpNotCompleted = 0; // max startPhase not completed
@@ -196,6 +204,9 @@ public class SupervisionScanner {
             } else {
                 LOGGER.debug("report AutomationComposition fault");
                 automationCompositionCounter.setFault(instanceId);
+                LOGGER.debug("report timeout for the ac instance");
+                automationComposition.setStateChangeResult(StateChangeResult.TIMEOUT);
+                automationCompositionProvider.updateAutomationComposition(automationComposition);
             }
         }
     }
diff --git a/runtime-acm/src/test/java/org/onap/policy/clamp/acm/runtime/supervision/SupervisionScannerTest.java b/runtime-acm/src/test/java/org/onap/policy/clamp/acm/runtime/supervision/SupervisionScannerTest.java
index 8ead9a7c0..7aaa6d22c 100644
--- a/runtime-acm/src/test/java/org/onap/policy/clamp/acm/runtime/supervision/SupervisionScannerTest.java
+++ b/runtime-acm/src/test/java/org/onap/policy/clamp/acm/runtime/supervision/SupervisionScannerTest.java
@@ -30,6 +30,7 @@ import static org.mockito.Mockito.when;
 import static org.onap.policy.clamp.acm.runtime.util.CommonTestData.TOSCA_SERVICE_TEMPLATE_YAML;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.UUID;
 import org.junit.jupiter.api.BeforeAll;
@@ -40,6 +41,7 @@ import org.onap.policy.clamp.acm.runtime.supervision.comm.AutomationCompositionS
 import org.onap.policy.clamp.acm.runtime.util.CommonTestData;
 import org.onap.policy.clamp.models.acm.concepts.AutomationComposition;
 import org.onap.policy.clamp.models.acm.concepts.AutomationCompositionDefinition;
+import org.onap.policy.clamp.models.acm.concepts.AutomationCompositionElement;
 import org.onap.policy.clamp.models.acm.concepts.DeployState;
 import org.onap.policy.clamp.models.acm.concepts.LockState;
 import org.onap.policy.clamp.models.acm.persistence.provider.AcDefinitionProvider;
@@ -104,6 +106,7 @@ class SupervisionScannerTest {
         verify(automationCompositionProvider).updateAutomationComposition(any(AutomationComposition.class));
     }
 
+
     @Test
     void testScannerDelete() {
         var automationComposition = InstantiationUtils.getAutomationCompositionFromResource(AC_JSON, "Crud");
@@ -145,6 +148,42 @@ class SupervisionScannerTest {
     }
 
     @Test
+    void testScannerForCounterHandling() {
+        var automationComposition = InstantiationUtils.getAutomationCompositionFromResource(AC_JSON, "Crud");
+        automationComposition.setDeployState(DeployState.DEPLOYING);
+        automationComposition.setLockState(LockState.NONE);
+        for (Map.Entry<UUID, AutomationCompositionElement> entry : automationComposition.getElements().entrySet()) {
+            entry.getValue().setDeployState(DeployState.DEPLOYING);
+        }
+        var automationCompositionProvider = mock(AutomationCompositionProvider.class);
+        when(automationCompositionProvider.getAcInstancesByCompositionId(compositionId))
+                .thenReturn(List.of(automationComposition));
+
+        var automationCompositionDeployPublisher = mock(AutomationCompositionDeployPublisher.class);
+        var automationCompositionStateChangePublisher = mock(AutomationCompositionStateChangePublisher.class);
+        var acRuntimeParameterGroup = CommonTestData.geParameterGroup("dbScanner");
+        acRuntimeParameterGroup.getParticipantParameters().setMaxStatusWaitMs(-1);
+
+        //verify retry scenario
+        var scannerObj1 = new SupervisionScanner(automationCompositionProvider, acDefinitionProvider,
+                automationCompositionStateChangePublisher, automationCompositionDeployPublisher,
+                acRuntimeParameterGroup);
+
+        scannerObj1.run(true);
+        verify(automationCompositionProvider, times(0)).updateAutomationComposition(any(AutomationComposition.class));
+
+        //verify timeout scenario
+        acRuntimeParameterGroup.getParticipantParameters().getUpdateParameters().setMaxRetryCount(0);
+        var scannerObj2 = new SupervisionScanner(automationCompositionProvider, acDefinitionProvider,
+                automationCompositionStateChangePublisher, automationCompositionDeployPublisher,
+                acRuntimeParameterGroup);
+
+        scannerObj2.run(true);
+        verify(automationCompositionProvider, times(1)).updateAutomationComposition(any(AutomationComposition.class));
+
+    }
+
+    @Test
     void testSendAutomationCompositionMsgUpdate() {
         var automationComposition = InstantiationUtils.getAutomationCompositionFromResource(AC_JSON, "Crud");
         automationComposition.setDeployState(DeployState.DEPLOYING);