Retry Module-Sync based on from last failure

Retry algorithm for module-sync based on last failure
Poll Lock Reason, check if lock has surpassed minimum time
based on last update time and lock reason

Issue-ID: CPS-1076
Change-Id: Ifbbabd2b403f88f1bbe3fae3f125b1e9cb2091aa
Signed-off-by: DylanB95EST <dylan.byrne@est.tech>

5 files changed, 72 insertions, 12 deletions

diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/CompositeStateBuilder.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/CompositeStateBuilder.java
index 012ba5ede..91e92ea6f 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/CompositeStateBuilder.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/CompositeStateBuilder.java
@@ -111,8 +111,9 @@ public class CompositeStateBuilder {
      * @return CompositeState
      */
     public CompositeStateBuilder fromDataNode(final DataNode dataNode) {
-        this.cmHandleState =  CmHandleState.valueOf((String) dataNode.getLeaves()
-                .get("cm-handle-state"));
+        this.cmHandleState = CmHandleState.valueOf((String) dataNode.getLeaves()
+            .get("cm-handle-state"));
+        this.lastUpdatedTime = (String) dataNode.getLeaves().get("last-update-time");
         for (final DataNode stateChildNode : dataNode.getChildDataNodes()) {
             if (stateChildNode.getXpath().endsWith("/lock-reason")) {
                 this.lockReason = getLockReason(stateChildNode);
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java
index e590ca1cd..402f9f6b4 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java
@@ -74,13 +74,16 @@ public class ModuleSyncWatchdog {
      * Execute Cm Handle poll which changes the cm handle state from 'LOCKED' to 'ADVISED'.
      */
     @Scheduled(fixedDelayString = "${timers.locked-modules-sync.sleep-time-ms:300000}")
-    public void executeLockedMisbehavingCmHandlePoll() {
+    public void executeLockedCmHandlePoll() {
         final List<YangModelCmHandle> lockedMisbehavingCmHandles = syncUtils.getLockedMisbehavingYangModelCmHandles();
-        for (final YangModelCmHandle lockedMisbehavingModelCmHandle : lockedMisbehavingCmHandles) {
-            final CompositeState compositeState = lockedMisbehavingModelCmHandle.getCompositeState();
-            setCompositeStateToAdvisedAndRetainOldLockReasonDetails(compositeState);
-            log.debug("Locked misbehaving cm handle {} is being recycled", lockedMisbehavingModelCmHandle.getId());
-            inventoryPersistence.saveCmHandleState(lockedMisbehavingModelCmHandle.getId(), compositeState);
+        for (final YangModelCmHandle moduleSyncFailedCmHandle : lockedMisbehavingCmHandles) {
+            final CompositeState compositeState = moduleSyncFailedCmHandle.getCompositeState();
+            final boolean isReadyForRetry = syncUtils.isReadyForRetry(compositeState);
+            if (isReadyForRetry) {
+                setCompositeStateToAdvisedAndRetainOldLockReasonDetails(compositeState);
+                log.debug("Locked misbehaving cm handle {} is being recycled", moduleSyncFailedCmHandle.getId());
+                inventoryPersistence.saveCmHandleState(moduleSyncFailedCmHandle.getId(), compositeState);
+            }
         }
     }
 
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java
index 42edcb7ec..8b7dfe6b8 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java
@@ -24,6 +24,9 @@ package org.onap.cps.ncmp.api.inventory.sync;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
 import java.security.SecureRandom;
+import java.time.Duration;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -140,6 +143,28 @@ public class SyncUtils {
             .lockReasonCategory(lockReasonCategory).build());
     }
 
+
+    /**
+     * Check if the retry mechanism should attempt to unlock the cm handle based on the last update time.
+     *
+     * @param compositeState the composite state currently in the locked state
+     * @return if the retry mechanism should be attempted
+     */
+    public boolean isReadyForRetry(final CompositeState compositeState) {
+        int timeUntilNextAttempt = 1;
+        final OffsetDateTime time =
+            OffsetDateTime.parse(compositeState.getLastUpdateTime(),
+                DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"));
+        final Matcher matcher = retryAttemptPattern.matcher(compositeState.getLockReason().getDetails());
+        if (matcher.find()) {
+            timeUntilNextAttempt = (int) Math.pow(2, Integer.parseInt(matcher.group(1)));
+        } else {
+            log.debug("First Attempt: no current attempts found.");
+        }
+        final int timeSinceLastAttempt = (int) Duration.between(time, OffsetDateTime.now()).toMinutes();
+        return timeSinceLastAttempt > timeUntilNextAttempt;
+    }
+
     /**
      * Get the Resourece Data from Node through DMI Passthrough service.
      *
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
index 614783efd..0f89a428b 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
@@ -90,16 +90,25 @@ class ModuleSyncSpec extends Specification {
 
     }
 
-    def 'Schedule a Cm-Handle Sync for LOCKED with reason LOCKED_MISBEHAVING Cm-Handles '() {
+    def 'Schedule a Cm-Handle Sync for LOCKED with reason LOCKED_MISBEHAVING Cm-Handles with #scenario'() {
         given: 'cm handles in an locked state'
             def compositeState = new CompositeStateBuilder().withCmHandleState(CmHandleState.LOCKED)
-                    .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING, '').build()
+                    .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING, '').withLastUpdatedTimeNow().build()
             def yangModelCmHandle = new YangModelCmHandle(id: 'some-cm-handle', compositeState: compositeState)
         and: 'sync utilities return a cm handle twice'
             mockSyncUtils.getLockedMisbehavingYangModelCmHandles() >> [yangModelCmHandle, yangModelCmHandle]
+        and: 'inventory persistence returns the composite state of the cm handle'
+            mockInventoryPersistence.getCmHandleState(yangModelCmHandle.getId()) >> compositeState
+        and: 'sync utils retry locked cm handle returns #isReadyForRetry'
+            mockSyncUtils.isReadyForRetry(compositeState) >>> isReadyForRetry
         when: 'module sync poll is executed'
-            objectUnderTest.executeLockedMisbehavingCmHandlePoll()
+            objectUnderTest.executeLockedCmHandlePoll()
         then: 'the first cm handle is updated to state "ADVISED" from "READY"'
-            2 * mockInventoryPersistence.saveCmHandleState(yangModelCmHandle.id, compositeState)
+            expectedNumberOfInvocationsToSaveCmHandleState * mockInventoryPersistence.saveCmHandleState(yangModelCmHandle.id, compositeState)
+        where:
+            scenario                        | isReadyForRetry         || expectedNumberOfInvocationsToSaveCmHandleState
+            'retry locked cm handle once'   | [true, false]           || 1
+            'retry locked cm handle twice'  | [true, true]            || 2
+            'do not retry locked cm handle' | [false, false]          || 0
     }
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
index 2c45ab769..dd299146f 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
@@ -27,6 +27,7 @@ import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations
 import org.onap.cps.ncmp.api.impl.operations.DmiOperations
 import org.onap.cps.ncmp.api.inventory.CmHandleState
 import org.onap.cps.ncmp.api.inventory.CompositeState
+import org.onap.cps.ncmp.api.inventory.CompositeStateBuilder
 import org.onap.cps.ncmp.api.inventory.InventoryPersistence
 import org.onap.cps.ncmp.api.inventory.LockReasonCategory
 import org.onap.cps.ncmp.api.inventory.SyncState
@@ -38,6 +39,9 @@ import org.springframework.http.ResponseEntity
 import spock.lang.Shared
 import spock.lang.Specification
 
+import java.time.OffsetDateTime
+import java.time.format.DateTimeFormatter
+
 class SyncUtilsSpec extends Specification{
 
     def mockInventoryPersistence = Mock(InventoryPersistence)
@@ -49,6 +53,9 @@ class SyncUtilsSpec extends Specification{
     def objectUnderTest = new SyncUtils(mockInventoryPersistence, mockDmiDataOperations, jsonObjectMapper)
 
     @Shared
+    def formattedDateAndTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(OffsetDateTime.now())
+
+    @Shared
     def dataNode = new DataNode(leaves: ['id': 'cm-handle-123'])
 
     def 'Get an advised Cm-Handle where ADVISED cm handle #scenario'() {
@@ -94,6 +101,21 @@ class SyncUtilsSpec extends Specification{
             result[0].id == 'cm-handle-123'
     }
 
+    def 'Retry Locked Cm-Handle where the last update time is #scenario'() {
+        when: 'retry locked cm handle is invoked'
+            def result = objectUnderTest.isReadyForRetry(new CompositeStateBuilder()
+                .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING, details)
+                .withLastUpdatedTime(lastUpdateTime).build())
+        then: 'result returns #expectedResult'
+            result == expectedResult
+        where:
+            scenario                        | lastUpdateTime                     | details                 || expectedResult
+            'is the first attempt'          | '1900-01-01T00:00:00.000+0100'     | 'First Attempt'         || true
+            'is greater than one minute'    | '1900-01-01T00:00:00.000+0100'     | 'Attempt #1 failed:'    || true
+            'is less than eight minutes'    | formattedDateAndTime               | 'Attempt #3 failed:'    || false
+    }
+
+
     def 'Get a Cm-Handle where Operational Sync state is UnSynchronized and Cm-handle state is READY and #scenario'() {
         given: 'the inventory persistence service returns a collection of data nodes'
             mockInventoryPersistence.getCmHandlesByOperationalSyncState(SyncState.UNSYNCHRONIZED) >> unSynchronizedDataNodes