Retry CM-Handles that are LOCKED, Failed-to-Sync

Watchdog added to fetched locked cm handles by Lock reason
and operation sync state
Tests updated

Issue-ID: CPS-878
Signed-off-by: puthuparambil.aditya <aditya.puthuparambil@bell.ca>
Change-Id: I0ecc519e93b097ec3d309db7f686fb23a0fc59b2

7 files changed, 100 insertions, 18 deletions

diff --git a/cps-ncmp-service/pom.xml b/cps-ncmp-service/pom.xml
index 99a024e65d..502e9824d7 100644
--- a/cps-ncmp-service/pom.xml
+++ b/cps-ncmp-service/pom.xml
@@ -3,6 +3,7 @@
   ============LICENSE_START=======================================================
   Copyright (C) 2021-2022 Nordix Foundation
   Modifications Copyright (C) 2021 Pantheon.tech
+ Modifications Copyright (C) 2022 Bell Canada
   ================================================================================
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -31,10 +32,10 @@
     </parent>
 
     <artifactId>cps-ncmp-service</artifactId>
+
     <properties>
-    <minimum-coverage>0.93</minimum-coverage>
+        <minimum-coverage>0.96</minimum-coverage>
     </properties>
-
     <dependencies>
         <dependency>
             <groupId>${project.groupId}</groupId>
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistence.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistence.java
index 2fc2dc5c1a..ce34154b9b 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistence.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistence.java
@@ -1,6 +1,7 @@
 /*
  *  ============LICENSE_START=======================================================
  *  Copyright (C) 2022 Nordix Foundation
+ *  Modifications Copyright (C) 2022 Bell Canada
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -92,6 +93,17 @@ public class InventoryPersistence {
     }
 
     /**
+     * Method to return cm handles from the cps path.
+     *
+     * @param cpsPath cps path for which the cmHandle is requested
+     * @return a list of cm handles
+     */
+    public List<DataNode> getCmHandlesByCpsPath(final String cpsPath) {
+        return cpsDataPersistenceService.queryDataNodes(
+            NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, cpsPath, FetchDescendantsOption.OMIT_DESCENDANTS);
+    }
+
+    /**
      * This method retrieves DMI service name, DMI properties and the state for a given cm handle.
      * @param cmHandleId the id of the cm handle
      * @return yang model cm handle
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java
index bcc7daa39d..dbc7dd4f2c 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java
@@ -1,6 +1,7 @@
 /*
- *  ============LICENSE_START=======================================================
+ * ============LICENSE_START=======================================================
  *  Copyright (C) 2022 Nordix Foundation
+ *  Modifications Copyright (C) 2022 Bell Canada
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -20,11 +21,13 @@
 
 package org.onap.cps.ncmp.api.inventory.sync;
 
+import java.util.List;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
 import org.onap.cps.ncmp.api.inventory.CmHandleState;
 import org.onap.cps.ncmp.api.inventory.CompositeState;
+import org.onap.cps.ncmp.api.inventory.CompositeState.LockReason;
 import org.onap.cps.ncmp.api.inventory.InventoryPersistence;
 import org.onap.cps.ncmp.api.inventory.LockReasonCategory;
 import org.springframework.scheduling.annotation.Scheduled;
@@ -44,7 +47,7 @@ public class ModuleSyncWatchdog {
     /**
      * Execute Cm Handle poll which changes the cm handle state from 'ADVISED' to 'READY'.
      */
-    @Scheduled(fixedDelayString = "${timers.advised-modules-sync.sleep-time-ms}")
+    @Scheduled(fixedDelayString = "${timers.advised-modules-sync.sleep-time-ms:30000}")
     public void executeAdvisedCmHandlePoll() {
         YangModelCmHandle advisedCmHandle = syncUtils.getAnAdvisedCmHandle();
         while (advisedCmHandle != null) {
@@ -68,4 +71,20 @@ public class ModuleSyncWatchdog {
         log.debug("No Cm-Handles currently found in an ADVISED state");
     }
 
+    /**
+     * Execute Cm Handle poll which changes the cm handle state from 'LOCKED' to 'ADVISED'.
+     */
+    @Scheduled(fixedDelayString = "${timers.locked-modules-sync.sleep-time-ms:300000}")
+    public void executeLockedMisbehavingCmHandlePoll() {
+        final List<YangModelCmHandle> lockedMisbehavingCmHandles = syncUtils.getLockedMisbehavingCmHandles();
+        for (final YangModelCmHandle lockedMisbehavingModelCmHandle: lockedMisbehavingCmHandles) {
+            final CompositeState updatedCompositeState = lockedMisbehavingModelCmHandle.getCompositeState();
+            updatedCompositeState.setCmHandleState(CmHandleState.ADVISED);
+            updatedCompositeState.setLastUpdateTimeNow();
+            updatedCompositeState.setLockReason(LockReason.builder()
+                .details(updatedCompositeState.getLockReason().getDetails()).build());
+            log.debug("Locked misbehaving cm handle {} is being recycled", lockedMisbehavingModelCmHandle.getId());
+            inventoryPersistence.saveCmHandleState(lockedMisbehavingModelCmHandle.getId(), updatedCompositeState);
+        }
+    }
 }
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java
index a4f29de3e8..22eeabb0df 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java
@@ -1,6 +1,7 @@
 /*
  *  ============LICENSE_START=======================================================
  *  Copyright (C) 2022 Nordix Foundation
+ *  Modifications Copyright (C) 2022 Bell Canada
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -24,8 +25,10 @@ import java.security.SecureRandom;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.onap.cps.ncmp.api.impl.utils.YangDataConverter;
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
 import org.onap.cps.ncmp.api.inventory.CmHandleState;
 import org.onap.cps.ncmp.api.inventory.CompositeState;
@@ -41,7 +44,6 @@ public class SyncUtils {
 
     private static final SecureRandom secureRandom = new SecureRandom();
 
-
     private final InventoryPersistence inventoryPersistence;
 
     private static final Pattern retryAttemptPattern = Pattern.compile("^Attempt #(\\d+) failed:");
@@ -64,6 +66,19 @@ public class SyncUtils {
 
 
     /**
+     * Query data nodes for cm handles with an "LOCKED" cm handle state with reason LOCKED_MISBEHAVING".
+     *
+     * @return a random yang model cm handle with an ADVISED state, return null if not found
+     */
+    public List<YangModelCmHandle> getLockedMisbehavingCmHandles() {
+        final List<DataNode> lockedCmHandleAsDataNodeList = inventoryPersistence.getCmHandlesByCpsPath(
+            "//lock-reason[@reason=\"LOCKED_MISBEHAVING\"]/ancestor::cm-handles");
+        return lockedCmHandleAsDataNodeList.stream()
+            .map(cmHandle -> YangDataConverter.convertCmHandleToYangModel(cmHandle,
+                cmHandle.getLeaves().get("id").toString())).collect(Collectors.toList());
+    }
+
+    /**
      * Update Composite State attempts counter and set new lock reason and details.
      *
      * @param lockReasonCategory lock reason category
@@ -84,5 +99,4 @@ public class SyncUtils {
             .lockReasonCategory(lockReasonCategory).build());
     }
 
-
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy
index a2ebcb5d81..e6346cb024 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy
@@ -1,6 +1,7 @@
 /*
  *  ============LICENSE_START=======================================================
  *  Copyright (C) 2022 Nordix Foundation
+ *  Modifications Copyright (C) 2022 Bell Canada
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -27,6 +28,7 @@ import org.onap.cps.spi.CpsDataPersistenceService
 import org.onap.cps.spi.FetchDescendantsOption
 import org.onap.cps.spi.exceptions.DataValidationException
 import org.onap.cps.spi.model.DataNode
+import org.onap.cps.spi.model.DataNodeBuilder
 import org.onap.cps.utils.JsonObjectMapper
 import spock.lang.Shared
 import spock.lang.Specification
@@ -50,7 +52,7 @@ class InventoryPersistenceSpec extends Specification {
     def objectUnderTest = new InventoryPersistence(spiedJsonObjectMapper, mockCpsDataService, mockCpsDataPersistenceService)
 
     def formattedDateAndTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
-        .format(OffsetDateTime.of(2022, 12, 31, 20, 30, 40, 1, ZoneOffset.UTC))
+            .format(OffsetDateTime.of(2022, 12, 31, 20, 30, 40, 1, ZoneOffset.UTC))
 
     def cmHandleId = 'some-cm-handle'
     def leaves = ["dmi-service-name":"common service name","dmi-data-service-name":"data service name","dmi-model-service-name":"model service name"]
@@ -88,7 +90,7 @@ class InventoryPersistenceSpec extends Specification {
         where: 'the following parameters are used'
             scenario                    | childDataNodes                                || expectedDmiProperties                               || expectedPublicProperties                              || expectedCompositeState
             'no properties'             | []                                            || []                                                  || []                                                    || null
-            'DMI and public properties' | childDataNodesForCmHandleWithAllProperties    || [new YangModelCmHandle.Property("name1", "value1")] || [new YangModelCmHandle.Property("name2", "value2")]   || null
+            'DMI and public properties' | childDataNodesForCmHandleWithAllProperties    || [new YangModelCmHandle.Property("name1", "value1")] || [new YangModelCmHandle.Property("name2", "value2")] || null
             'just DMI properties'       | childDataNodesForCmHandleWithDMIProperties    || [new YangModelCmHandle.Property("name1", "value1")] || []                                                    || null
             'just public properties'    | childDataNodesForCmHandleWithPublicProperties || []                                                  || [new YangModelCmHandle.Property("name2", "value2")]   || null
             'with state details'        | childDataNodesForCmHandleWithState            || []                                                  || []                                                    || CmHandleState.ADVISED
@@ -105,7 +107,7 @@ class InventoryPersistenceSpec extends Specification {
 
     def "Handling missing service names as null CPS-1043."() {
         given: 'the cps data service returns a data node from the DMI registry with empty child and leaf attributes'
-            def dataNode = new DataNode(childDataNodes:[], leaves: ["cm-handle-state":"ADVISED"])
+            def dataNode = new DataNode(childDataNodes:[], leaves: [:])
             mockCpsDataService.getDataNode('NCMP-Admin', 'ncmp-dmi-registry', xpath, INCLUDE_ALL_DESCENDANTS) >> dataNode
         when: 'retrieving the yang modelled cm handle'
             def result = objectUnderTest.getYangModelCmHandle(cmHandleId)
@@ -121,7 +123,7 @@ class InventoryPersistenceSpec extends Specification {
             def dataNode = new DataNode(leaves: ['cm-handle-state': 'ADVISED'])
         and: 'cps data service returns a valid data node'
             mockCpsDataService.getDataNode('NCMP-Admin', 'ncmp-dmi-registry',
-                '/dmi-registry/cm-handles[@id=\'Some-Cm-Handle\']/state', FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) >> dataNode
+                    '/dmi-registry/cm-handles[@id=\'Some-Cm-Handle\']/state', FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) >> dataNode
         when: 'get cm handle state is invoked'
             def result = objectUnderTest.getCmHandleState(cmHandleId)
         then: 'result has returned the correct cm handle state'
@@ -137,7 +139,7 @@ class InventoryPersistenceSpec extends Specification {
         then: 'update node leaves is invoked with the correct params'
             1 * mockCpsDataService.replaceNodeTree('NCMP-Admin', 'ncmp-dmi-registry', '/dmi-registry/cm-handles[@id=\'Some-Cm-Handle\']', expectedJsonData, _ as OffsetDateTime)
         where: 'the following states are used'
-             scenario | cmHandleState        || expectedJsonData
+            scenario | cmHandleState        || expectedJsonData
             'READY'   | CmHandleState.READY  || '{"state":{"cm-handle-state":"READY","last-update-time":"2022-12-31T20:30:40.000+0000"}}'
             'LOCKED'  | CmHandleState.LOCKED || '{"state":{"cm-handle-state":"LOCKED","last-update-time":"2022-12-31T20:30:40.000+0000"}}'
     }
@@ -148,11 +150,25 @@ class InventoryPersistenceSpec extends Specification {
         and: 'cps data service returns a list of data nodes'
             def dataNodes = [new DataNode()]
             mockCpsDataPersistenceService.queryDataNodes('NCMP-Admin', 'ncmp-dmi-registry',
-                '//state[@cm-handle-state="ADVISED"]/ancestor::cm-handles', OMIT_DESCENDANTS) >> dataNodes
+                    '//state[@cm-handle-state="ADVISED"]/ancestor::cm-handles', OMIT_DESCENDANTS) >> dataNodes
         when: 'get cm handles by state is invoked'
             def result = objectUnderTest.getCmHandlesByState(cmHandleState)
         then: 'the returned result is a list of data nodes returned by cps data service'
             assert result == dataNodes
     }
 
+    def 'Retrieve cm handle by cps path '() {
+        given: 'a cm handle state to query based on the cps path'
+            def cmHandleDataNode = new DataNode(xpath: 'xpath', leaves: ['cm-handle-state': 'LOCKED'])
+            def cpsPath = '//cps-path'
+        and: 'cps data service returns a valid data node'
+            mockCpsDataPersistenceService.queryDataNodes('NCMP-Admin', 'ncmp-dmi-registry',
+                    cpsPath, OMIT_DESCENDANTS)
+                    >> Arrays.asList(cmHandleDataNode)
+        when: 'get cm handles by cps path is invoked'
+            def result = objectUnderTest.getCmHandlesByCpsPath(cpsPath)
+        then: 'the returned result is a list of data nodes returned by cps data service'
+            assert result.contains(cmHandleDataNode)
+    }
+
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
index 97bea096a3..544f739c06 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
@@ -1,6 +1,7 @@
 /*
  *  ============LICENSE_START=======================================================
  *  Copyright (C) 2022 Nordix Foundation
+ *  Modifications Copyright (C) 2022 Bell Canada
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -25,6 +26,7 @@ import org.onap.cps.ncmp.api.inventory.CmHandleState
 import org.onap.cps.ncmp.api.inventory.CompositeState
 import org.onap.cps.ncmp.api.inventory.InventoryPersistence
 import org.onap.cps.ncmp.api.inventory.LockReasonCategory
+import org.onap.cps.ncmp.api.inventory.CompositeStateBuilder
 import spock.lang.Specification
 
 class ModuleSyncSpec extends Specification {
@@ -88,4 +90,16 @@ class ModuleSyncSpec extends Specification {
 
     }
 
+    def 'Schedule a Cm-Handle Sync for LOCKED with reason LOCKED_MISBEHAVING Cm-Handles '() {
+        given: 'cm handles in an locked state'
+            def compositeState = new CompositeStateBuilder().withCmHandleState(CmHandleState.LOCKED)
+                    .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING, '').build()
+            def yangModelCmHandle = new YangModelCmHandle(id: 'some-cm-handle', compositeState: compositeState)
+        and: 'sync utilities return a cm handle twice'
+            mockSyncUtils.getLockedMisbehavingCmHandles() >> [yangModelCmHandle, yangModelCmHandle]
+        when: 'module sync poll is executed'
+            objectUnderTest.executeLockedMisbehavingCmHandlePoll()
+        then: 'the first cm handle is updated to state "ADVISED" from "READY"'
+            2 * mockInventoryPersistence.saveCmHandleState(yangModelCmHandle.id, compositeState)
+    }
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
index 7d67acccc3..15d1efe068 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
@@ -1,6 +1,7 @@
 /*
  * ============LICENSE_START=======================================================
  *  Copyright (C) 2022 Nordix Foundation
+ *  Modifications Copyright (C) 2022 Bell Canada
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -24,15 +25,12 @@ import org.onap.cps.ncmp.api.inventory.CmHandleState
 import org.onap.cps.ncmp.api.inventory.CompositeState
 import org.onap.cps.ncmp.api.inventory.InventoryPersistence
 import org.onap.cps.ncmp.api.inventory.LockReasonCategory
-import org.onap.cps.spi.CpsDataPersistenceService
-import org.onap.cps.spi.FetchDescendantsOption
 import org.onap.cps.spi.model.DataNode
 import spock.lang.Shared
 import spock.lang.Specification
 
 class SyncUtilsSpec extends Specification{
 
-    def mockCpsDataPersistenceService = Mock(CpsDataPersistenceService)
     def mockInventoryPersistence = Mock(InventoryPersistence)
 
     def objectUnderTest = new SyncUtils(mockInventoryPersistence)
@@ -40,8 +38,6 @@ class SyncUtilsSpec extends Specification{
     @Shared
     def dataNode = new DataNode(leaves: ['id': 'cm-handle-123'])
 
-
-
     def 'Get an advised Cm-Handle where ADVISED cm handle #scenario'() {
         given: 'the inventory persistence service returns a collection of data nodes'
             mockInventoryPersistence.getCmHandlesByState(CmHandleState.ADVISED) >> dataNodeCollection
@@ -71,5 +67,15 @@ class SyncUtilsSpec extends Specification{
             'does not exist' | null                                                                                         || 'Attempt #1 failed: new error message'
             'exists'         | CompositeState.LockReason.builder().details("Attempt #2 failed: some error message").build() || 'Attempt #3 failed: new error message'
     }
-
+    def 'Get all locked Cm-Handle where Lock Reason is LOCKED_MISBEHAVING cm handle #scenario'() {
+        given: 'the cps (persistence service) returns a collection of data nodes'
+            mockInventoryPersistence.getCmHandlesByCpsPath(
+                    '//lock-reason[@reason="LOCKED_MISBEHAVING"]/ancestor::cm-handles') >> [dataNode ]
+        when: 'get locked Misbehaving cm handle is called'
+            def result = objectUnderTest.getLockedMisbehavingCmHandles()
+        then: 'the returned cm handle collection is the correct size'
+            result.size() == 1
+        and: 'the correct cm handle is returned'
+            result[0].id == 'cm-handle-123'
+    }
 }