Retry mechanism (with back off algorithm) is removed with more frequent watchdog poll

- Increased watchdog frequency for locked cm handle.
- Removed retry backoff algorithm for locked cm handle.

Issue-ID: CPS-2395
Change-Id: I54d0ec8f9de53a7d181639c14aaaa93736f03e19
Signed-off-by: sourabh_sourabh <sourabh.sourabh@est.tech>

2 files changed, 62 insertions, 106 deletions

diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy
index babe8101dd..65b7ff6d6f 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy
@@ -40,12 +40,8 @@ import org.springframework.context.annotation.AnnotationConfigApplicationContext
 import org.springframework.http.HttpStatus
 import org.springframework.http.ResponseEntity
 import spock.lang.Specification
-
-import java.time.OffsetDateTime
-import java.time.format.DateTimeFormatter
 import java.util.stream.Collectors
 
-import static org.onap.cps.ncmp.impl.inventory.models.LockReasonCategory.LOCKED_MISBEHAVING
 import static org.onap.cps.ncmp.impl.inventory.models.LockReasonCategory.MODULE_SYNC_FAILED
 import static org.onap.cps.ncmp.impl.inventory.models.LockReasonCategory.MODULE_UPGRADE
 import static org.onap.cps.ncmp.impl.inventory.models.LockReasonCategory.MODULE_UPGRADE_FAILED
@@ -60,17 +56,12 @@ class ModuleOperationsUtilsSpec extends Specification{
 
     def objectUnderTest = new ModuleOperationsUtils(mockCmHandleQueries, mockDmiDataOperations, jsonObjectMapper)
 
-    def static neverUpdatedBefore = '1900-01-01T00:00:00.000+0100'
-
-    def static now = OffsetDateTime.now()
-
-    def static nowAsString = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(now)
-
     def static dataNode = new DataNode(leaves: ['id': 'cm-handle-123'])
 
     def applicationContext = new AnnotationConfigApplicationContext()
 
     def logger = (Logger) LoggerFactory.getLogger(ModuleOperationsUtils)
+
     def loggingListAppender
 
     void setup() {
@@ -103,7 +94,7 @@ class ModuleOperationsUtilsSpec extends Specification{
         given: 'A locked state'
             def compositeState = new CompositeState(lockReason: lockReason)
         when: 'update cm handle details and attempts is called'
-            objectUnderTest.updateLockReasonDetailsAndAttempts(compositeState, MODULE_SYNC_FAILED, 'new error message')
+            objectUnderTest.updateLockReasonWithAttempts(compositeState, MODULE_SYNC_FAILED, 'new error message')
         then: 'the composite state lock reason and details are updated'
             assert compositeState.lockReason.lockReasonCategory == MODULE_SYNC_FAILED
             assert compositeState.lockReason.details.contains(expectedDetails)
@@ -118,14 +109,14 @@ class ModuleOperationsUtilsSpec extends Specification{
             def compositeState = new CompositeStateBuilder().withCmHandleState(CmHandleState.LOCKED)
                 .withLockReason(MODULE_UPGRADE, "Upgrade to ModuleSetTag: " + moduleSetTag).build()
         when: 'update cm handle details'
-            objectUnderTest.updateLockReasonDetailsAndAttempts(compositeState, MODULE_UPGRADE_FAILED, 'new error message')
+            objectUnderTest.updateLockReasonWithAttempts(compositeState, MODULE_UPGRADE_FAILED, 'new error message')
         then: 'the composite state lock reason and details are updated'
             assert compositeState.lockReason.lockReasonCategory == MODULE_UPGRADE_FAILED
-            assert compositeState.lockReason.details.contains("Upgrade to ModuleSetTag: " + expectedDetails)
+            assert compositeState.lockReason.details.contains(expectedDetails)
         where:
             scenario               | moduleSetTag       || expectedDetails
-            'a module set tag'     | 'someModuleSetTag' || 'someModuleSetTag'
-            'empty module set tag' | ''                 || ''
+            'a module set tag'     | 'someModuleSetTag' || 'Upgrade to ModuleSetTag: someModuleSetTag'
+            'empty module set tag' | ''                 || 'Attempt'
     }
 
     def 'Get all locked cm-Handles where lock reasons are model sync failed or upgrade'() {
@@ -135,49 +126,9 @@ class ModuleOperationsUtilsSpec extends Specification{
         when: 'get locked Misbehaving cm handle is called'
             def result = objectUnderTest.getCmHandlesThatFailedModelSyncOrUpgrade()
         then: 'the returned cm handle collection is the correct size'
-            result.size() == 1
+            assert result.size() == 1
         and: 'the correct cm handle is returned'
-            result[0].id == 'cm-handle-123'
-    }
-
-    def 'Retry Locked Cm-Handle where the last update time is #scenario'() {
-        given: 'Last update was #lastUpdateMinutesAgo minutes ago (-1 means never)'
-            def lastUpdatedTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(now.minusMinutes(lastUpdateMinutesAgo))
-            if (lastUpdateMinutesAgo < 0 ) {
-                lastUpdatedTime = neverUpdatedBefore
-            }
-        when: 'checking to see if cm handle is ready for retry'
-         def result = objectUnderTest.needsModuleSyncRetryOrUpgrade(new CompositeStateBuilder()
-                .withLockReason(MODULE_SYNC_FAILED, lockDetails)
-                .withLastUpdatedTime(lastUpdatedTime).build())
-        then: 'retry is only attempted when expected'
-            assert result == retryExpected
-        and: 'logs contain related information'
-            def logs = loggingListAppender.list.toString()
-            assert logs.contains(logReason)
-        where: 'the following parameters are used'
-            scenario                                    | lastUpdateMinutesAgo | lockDetails                     | logReason                               || retryExpected
-            'never attempted before'                    | -1                   | 'Fist attempt:'                 | 'First Attempt:'                        || true
-            '1st attempt, last attempt > 2 minute ago'  | 3                    | 'Attempt #1 failed: some error' | 'Retry due now'                         || true
-            '2nd attempt, last attempt < 4 minutes ago' | 1                    | 'Attempt #2 failed: some error' | 'Time until next attempt is 3 minutes:' || false
-            '2nd attempt, last attempt > 4 minutes ago' | 5                    | 'Attempt #2 failed: some error' | 'Retry due now'                         || true
-    }
-
-    def 'Retry Locked Cm-Handle with lock reasons (category) #lockReasonCategory'() {
-        when: 'checking to see if cm handle is ready for retry'
-            def result = objectUnderTest.needsModuleSyncRetryOrUpgrade(new CompositeStateBuilder()
-                .withLockReason(lockReasonCategory, 'some details')
-                .withLastUpdatedTime(nowAsString).build())
-        then: 'verify retry attempts'
-            assert !result
-        and: 'logs contain related information'
-            def logs = loggingListAppender.list.toString()
-            assert logs.contains(logReason)
-        where: 'the following lock reasons occurred'
-            scenario             | lockReasonCategory    || logReason
-            'module upgrade'     | MODULE_UPGRADE_FAILED || 'First Attempt:'
-            'module sync failed' | MODULE_SYNC_FAILED    || 'First Attempt:'
-            'lock misbehaving'   | LOCKED_MISBEHAVING    || 'Locked for other reason'
+            assert result[0].id == 'cm-handle-123'
     }
 
     def 'Get a Cm-Handle where #scenario'() {
@@ -197,19 +148,25 @@ class ModuleOperationsUtilsSpec extends Specification{
             'all Cm-Handle synchronized'               | []                      | false            || 0                    | [] as Set
     }
 
-    def 'Get resource data through DMI Operations #scenario'() {
-        given: 'the inventory persistence service returns a collection of data nodes'
+    def 'Retrieve resource data from DMI operations for #scenario'() {
+        given: 'a JSON string representing the resource data'
             def jsonString = '{"stores:bookstore":{"categories":[{"code":"01"}]}}'
-            JsonNode jsonNode = jsonObjectMapper.convertToJsonNode(jsonString);
-            def responseEntity = new ResponseEntity<>(jsonNode, HttpStatus.OK)
+            JsonNode jsonNode = jsonObjectMapper.convertToJsonNode(jsonString)
+        and: 'DMI operations are mocked to return a response based on the scenario'
+            def responseEntity = new ResponseEntity<>(statusCode == HttpStatus.OK ? jsonNode : null, statusCode)
             mockDmiDataOperations.getAllResourceDataFromDmi('cm-handle-123', _) >> responseEntity
         when: 'get resource data is called'
             def result = objectUnderTest.getResourceData('cm-handle-123')
-        then: 'the returned data is correct'
-            result == jsonString
+        then: 'the returned data matches the expected result'
+            assert result == expectedResult
+        where:
+            scenario                              | statusCode                       | expectedResult
+            'successful response'                 | HttpStatus.OK                    | '{"stores:bookstore":{"categories":[{"code":"01"}]}}'
+            'response with not found status'      | HttpStatus.NOT_FOUND             | null
+            'response with internal server error' | HttpStatus.INTERNAL_SERVER_ERROR | null
     }
 
-    def 'Extract module set tag and number of attempt when lock reason contains #scenario'() {
+        def 'Extract module set tag and number of attempt when lock reason contains #scenario'() {
         expect: 'lock reason details are extracted correctly'
             def result = objectUnderTest.getLockedCompositeStateDetails(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, lockReasonDetails).build().lockReason)
         and: 'the result contains the correct moduleSetTag'
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy
index ee49f2f901..160744a7d7 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy
@@ -32,16 +32,15 @@ import org.onap.cps.ncmp.api.inventory.models.CompositeState
 import org.onap.cps.ncmp.api.inventory.models.CompositeStateBuilder
 import org.onap.cps.ncmp.impl.inventory.InventoryPersistence
 import org.onap.cps.ncmp.impl.inventory.models.CmHandleState
-import org.onap.cps.ncmp.impl.inventory.models.LockReasonCategory
 import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle
 import org.onap.cps.ncmp.impl.inventory.sync.lcm.LcmEventsCmHandleStateHandler
 import org.onap.cps.spi.model.DataNode
 import org.slf4j.LoggerFactory
 import spock.lang.Specification
-
 import java.util.concurrent.atomic.AtomicInteger
 
 import static org.onap.cps.ncmp.impl.inventory.models.LockReasonCategory.MODULE_SYNC_FAILED
+import static org.onap.cps.ncmp.impl.inventory.models.LockReasonCategory.MODULE_UPGRADE
 import static org.onap.cps.ncmp.impl.inventory.models.LockReasonCategory.MODULE_UPGRADE_FAILED
 
 class ModuleSyncTasksSpec extends Specification {
@@ -96,70 +95,52 @@ class ModuleSyncTasksSpec extends Specification {
             assert batchCount.get() == 4
     }
 
-    def 'Module Sync ADVISED cm handle with failure during sync.'() {
-        given: 'a cm handle in an ADVISED state'
-            def cmHandle = cmHandleAsDataNodeByIdAndState('cm-handle', CmHandleState.ADVISED)
-        and: 'the inventory persistence cm handle returns a ADVISED state for the cm handle'
-            def cmHandleState = new CompositeState(cmHandleState: CmHandleState.ADVISED)
-            1 * mockInventoryPersistence.getCmHandleState('cm-handle') >> cmHandleState
-        and: 'module sync service attempts to sync the cm handle and throws an exception'
-            1 * mockModuleSyncService.syncAndCreateSchemaSetAndAnchor(*_) >> { throw new Exception('some exception') }
+    def 'Handle CM handle failure during #scenario and log MODULE_UPGRADE lock reason'() {
+        given: 'a CM handle in LOCKED state with a specific lock reason'
+            def cmHandle = cmHandleAsDataNodeByIdAndState('cm-handle', CmHandleState.LOCKED)
+            def expectedCmHandleState = new CompositeState(cmHandleState: CmHandleState.LOCKED, lockReason: CompositeState
+                    .LockReason.builder().lockReasonCategory(lockReasonCategory).details(lockReasonDetails).build())
+            1 * mockInventoryPersistence.getCmHandleState('cm-handle') >> expectedCmHandleState
+        and: 'module sync service attempts to sync/upgrade the CM handle and throws an exception'
+            mockModuleSyncService.syncAndCreateSchemaSetAndAnchor(_) >> { throw new Exception('some exception') }
+            mockModuleSyncService.syncAndUpgradeSchemaSet(_) >> { throw new Exception('some exception') }
         when: 'module sync is executed'
             objectUnderTest.performModuleSync([cmHandle], batchCount)
-        then: 'update lock reason, details and attempts is invoked'
-            1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(cmHandleState, MODULE_SYNC_FAILED, 'some exception')
+        then: 'lock reason is updated with number of attempts'
+            1 * mockSyncUtils.updateLockReasonWithAttempts(expectedCmHandleState, expectedLockReasonCategory, 'some exception')
         and: 'the state handler is called to update the state to LOCKED'
             1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> { args ->
                 assertBatch(args, ['cm-handle'], CmHandleState.LOCKED)
             }
         and: 'batch count is decremented by one'
             assert batchCount.get() == 4
-    }
-
-    def 'Failed cm handle during #scenario.'() {
-        given: 'a cm handle in LOCKED state'
-            def cmHandle = cmHandleAsDataNodeByIdAndState('cm-handle', CmHandleState.LOCKED)
-        and: 'the inventory persistence cm handle returns a LOCKED state with reason for the cm handle'
-            def expectedCmHandleState = new CompositeState(cmHandleState: cmHandleState, lockReason: CompositeState
-                .LockReason.builder().lockReasonCategory(lockReasonCategory).details(lockReasonDetails).build())
-            1 * mockInventoryPersistence.getCmHandleState('cm-handle') >> expectedCmHandleState
-        and: 'module sync service attempts to sync/upgrade the cm handle and throws an exception'
-            mockModuleSyncService.syncAndCreateSchemaSetAndAnchor(*_) >> { throw new Exception('some exception') }
-            mockModuleSyncService.syncAndUpgradeSchemaSet(*_) >> { throw new Exception('some exception') }
-        when: 'module sync is executed'
-            objectUnderTest.performModuleSync([cmHandle], batchCount)
-        then: 'update lock reason, details and attempts is invoked'
-            1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(expectedCmHandleState, expectedLockReasonCategory, 'some exception')
         where:
-            scenario         | cmHandleState        | lockReasonCategory    | lockReasonDetails                              || expectedLockReasonCategory
-            'module upgrade' | CmHandleState.LOCKED | MODULE_UPGRADE_FAILED | 'Upgrade to ModuleSetTag: some-module-set-tag' || MODULE_UPGRADE_FAILED
-            'module sync'    | CmHandleState.LOCKED | MODULE_SYNC_FAILED    | 'some lock details'                            || MODULE_SYNC_FAILED
+            scenario         | lockReasonCategory    | lockReasonDetails                              || expectedLockReasonCategory
+            'module sync'    | MODULE_SYNC_FAILED    | 'some lock details'                            || MODULE_SYNC_FAILED
+            'module upgrade' | MODULE_UPGRADE_FAILED | 'Upgrade to ModuleSetTag: some-module-set-tag' || MODULE_UPGRADE_FAILED
+            'module upgrade' | MODULE_UPGRADE        | 'Upgrade in progress'                          || MODULE_UPGRADE_FAILED
     }
 
+
     def 'Reset failed CM Handles #scenario.'() {
         given: 'cm handles in an locked state'
             def lockedState = new CompositeStateBuilder().withCmHandleState(CmHandleState.LOCKED)
-                    .withLockReason(LockReasonCategory.MODULE_SYNC_FAILED, '').withLastUpdatedTimeNow().build()
+                .withLockReason(MODULE_SYNC_FAILED, '').withLastUpdatedTimeNow().build()
             def yangModelCmHandle1 = new YangModelCmHandle(id: 'cm-handle-1', compositeState: lockedState)
             def yangModelCmHandle2 = new YangModelCmHandle(id: 'cm-handle-2', compositeState: lockedState)
-            def expectedCmHandleStatePerCmHandle = [(yangModelCmHandle1): CmHandleState.ADVISED]
+            def expectedCmHandleStatePerCmHandle
+                    = [(yangModelCmHandle1): CmHandleState.ADVISED, (yangModelCmHandle2): CmHandleState.ADVISED]
         and: 'clear in progress map'
             resetModuleSyncStartedOnCmHandles(moduleSyncStartedOnCmHandles)
         and: 'add cm handle entry into progress map'
             moduleSyncStartedOnCmHandles.put('cm-handle-1', 'started')
             moduleSyncStartedOnCmHandles.put('cm-handle-2', 'started')
-        and: 'sync utils retry locked cm handle returns #isReadyForRetry'
-            mockSyncUtils.needsModuleSyncRetryOrUpgrade(lockedState) >>> isReadyForRetry
         when: 'resetting failed cm handles'
             objectUnderTest.resetFailedCmHandles([yangModelCmHandle1, yangModelCmHandle2])
         then: 'updated to state "ADVISED" from "READY" is called as often as there are cm handles ready for retry'
-            expectedNumberOfInvocationsToUpdateCmHandleState * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(expectedCmHandleStatePerCmHandle)
-        and: 'after reset performed size of in progress map'
-            assert moduleSyncStartedOnCmHandles.size() == inProgressMapSize
-        where:
-            scenario                        | isReadyForRetry | inProgressMapSize || expectedNumberOfInvocationsToUpdateCmHandleState
-            'retry locked cm handle'        | [true, false]   | 1                 || 1
-            'do not retry locked cm handle' | [false, false]  | 2                 || 0
+            1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(expectedCmHandleStatePerCmHandle)
+        and: 'after reset performed progress map is empty'
+            assert moduleSyncStartedOnCmHandles.size() == 0
     }
 
     def 'Module Sync ADVISED cm handle without entry in progress map.'() {
@@ -189,6 +170,24 @@ class ModuleSyncTasksSpec extends Specification {
             assert loggingEvent.formattedMessage == 'ch-1 removed from in progress map'
     }
 
+    def 'Sync and upgrade CM handle if in upgrade state for #scenario'() {
+        given: 'a CM handle in an upgrade state'
+            def cmHandle = cmHandleAsDataNodeByIdAndState('cm-handle', CmHandleState.LOCKED)
+            def compositeState = new CompositeState(lockReason: CompositeState.LockReason.builder().lockReasonCategory(lockReasonCategory).build())
+            1 * mockInventoryPersistence.getCmHandleState('cm-handle') >> compositeState
+        when: 'module sync is executed'
+            objectUnderTest.performModuleSync([cmHandle], batchCount)
+        then: 'the module sync service should attempt to sync and upgrade the CM handle'
+            1 * mockModuleSyncService.syncAndUpgradeSchemaSet(_) >> { args ->
+                assert args[0].id == 'cm-handle'
+            }
+        where: 'the following lock reasons are used'
+            scenario                | lockReasonCategory
+            'module upgrade'        | MODULE_UPGRADE
+            'module upgrade failed' | MODULE_UPGRADE_FAILED
+    }
+
+
     def 'Remove non-existing cm handle id from hazelcast map'() {
         given: 'hazelcast map does not contains cm handle id'
             def result = moduleSyncStartedOnCmHandles.get('non-existing-cm-handle')