Retry for yang module upgrade operation

- Added retry code for yang module upgrade failed

Issue-ID: CPS-1802

Signed-off-by: sourabh_sourabh <sourabh.sourabh@est.tech>
Change-Id: Ida68d45f926ed48b92f4b5a3b82e641d778303ce
Signed-off-by: sourabh_sourabh <sourabh.sourabh@est.tech>

6 files changed, 102 insertions, 57 deletions

diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java
index db7b12cbc..f0336900f 100755
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java
@@ -60,6 +60,7 @@ import org.onap.cps.ncmp.api.impl.inventory.CompositeStateBuilder;
 import org.onap.cps.ncmp.api.impl.inventory.CompositeStateUtils;
 import org.onap.cps.ncmp.api.impl.inventory.DataStoreSyncState;
 import org.onap.cps.ncmp.api.impl.inventory.InventoryPersistence;
+import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleOperationsUtils;
 import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations;
 import org.onap.cps.ncmp.api.impl.operations.OperationType;
 import org.onap.cps.ncmp.api.impl.trustlevel.TrustLevel;
@@ -409,7 +410,8 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService
         final NcmpServiceCmHandle ncmpServiceCmHandle = new NcmpServiceCmHandle();
         ncmpServiceCmHandle.setCmHandleId(cmHandleId);
         final String moduleSetTag = dmiPluginRegistration.getUpgradedCmHandles().getModuleSetTag();
-        final String lockReasonWithModuleSetTag = MessageFormat.format("ModuleSetTag: {0}", moduleSetTag);
+        final String lockReasonWithModuleSetTag = MessageFormat.format(
+                ModuleOperationsUtils.MODULE_SET_TAG_MESSAGE_FORMAT, moduleSetTag);
         ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withCmHandleState(CmHandleState.READY)
                 .withLockReason(MODULE_UPGRADE, lockReasonWithModuleSetTag).build());
         return YangModelCmHandle.toYangModelCmHandle(dmiPluginRegistration.getDmiPlugin(),
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdog.java
index 49804adc1..6f089a57f 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdog.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdog.java
@@ -48,7 +48,7 @@ public class DataSyncWatchdog {
 
     private final CpsDataService cpsDataService;
 
-    private final SyncUtils syncUtils;
+    private final ModuleOperationsUtils moduleOperationsUtils;
 
     private final IMap<String, Boolean> dataSyncSemaphores;
 
@@ -58,13 +58,13 @@ public class DataSyncWatchdog {
      */
     @Scheduled(fixedDelayString = "${ncmp.timers.cm-handle-data-sync.sleep-time-ms:30000}")
     public void executeUnSynchronizedReadyCmHandlePoll() {
-        syncUtils.getUnsynchronizedReadyCmHandles().forEach(unSynchronizedReadyCmHandle -> {
+        moduleOperationsUtils.getUnsynchronizedReadyCmHandles().forEach(unSynchronizedReadyCmHandle -> {
             final String cmHandleId = unSynchronizedReadyCmHandle.getId();
             if (hasPushedIntoSemaphoreMap(cmHandleId)) {
                 log.debug("Executing data sync on {}", cmHandleId);
                 final CompositeState compositeState = inventoryPersistence
                         .getCmHandleState(cmHandleId);
-                final String resourceData = syncUtils.getResourceData(cmHandleId);
+                final String resourceData = moduleOperationsUtils.getResourceData(cmHandleId);
                 if (resourceData == null) {
                     log.debug("Error retrieving resource data for Cm-Handle: {}", cmHandleId);
                 } else {
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtils.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtils.java
index ab85c2127..25ea3bc1e 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtils.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtils.java
@@ -29,6 +29,7 @@ import java.time.OffsetDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -54,12 +55,18 @@ import org.springframework.stereotype.Service;
 @Slf4j
 @Service
 @RequiredArgsConstructor
-public class SyncUtils {
+public class ModuleOperationsUtils {
 
     private final CmHandleQueries cmHandleQueries;
     private final DmiDataOperations dmiDataOperations;
     private final JsonObjectMapper jsonObjectMapper;
-    private static final Pattern retryAttemptPattern = Pattern.compile("^Attempt #(\\d+) failed:");
+    private static final String RETRY_ATTEMPT_KEY = "attempt";
+    public static final String MODULE_SET_TAG_KEY = "moduleSetTag";
+    public static final String MODULE_SET_TAG_MESSAGE_FORMAT = "Upgrade to ModuleSetTag: {0}";
+    private static final String UPGRADE_FORMAT = "Upgrade to ModuleSetTag: %s";
+    private static final String UPGRADE_FAILED_FORMAT = UPGRADE_FORMAT + " Attempt #%d failed: %s";
+    private static final Pattern retryAttemptPattern = Pattern.compile("Attempt #(\\d+) failed:.+");
+    private static final Pattern moduleSetTagPattern = Pattern.compile("Upgrade to ModuleSetTag: (\\S+)");
 
     /**
      * Query data nodes for cm handles with an "ADVISED" cm handle state.
@@ -87,14 +94,11 @@ public class SyncUtils {
         for (final DataNode unsynchronizedCmHandle : unsynchronizedCmHandles) {
             final String cmHandleId = unsynchronizedCmHandle.getLeaves().get("id").toString();
             if (cmHandleQueries.cmHandleHasState(cmHandleId, CmHandleState.READY)) {
-                yangModelCmHandles.addAll(
-                        convertCmHandlesDataNodesToYangModelCmHandles(
+                yangModelCmHandles.addAll(convertCmHandlesDataNodesToYangModelCmHandles(
                                 Collections.singletonList(unsynchronizedCmHandle)));
             }
         }
-
         Collections.shuffle(yangModelCmHandles);
-
         return yangModelCmHandles;
     }
 
@@ -121,17 +125,43 @@ public class SyncUtils {
                                                    final LockReasonCategory lockReasonCategory,
                                                    final String errorMessage) {
         int attempt = 1;
-        if (compositeState.getLockReason() != null) {
-            final Matcher matcher = retryAttemptPattern.matcher(compositeState.getLockReason().getDetails());
-            if (matcher.find()) {
-                attempt = 1 + Integer.parseInt(matcher.group(1));
-            }
+        final Map<String, String> compositeStateDetails
+                = getLockedCompositeStateDetails(compositeState.getLockReason());
+        if (!compositeStateDetails.isEmpty()) {
+            attempt = 1 + Integer.parseInt(compositeStateDetails.get(RETRY_ATTEMPT_KEY));
         }
         compositeState.setLockReason(CompositeState.LockReason.builder()
-                .details(String.format("Attempt #%d failed: %s", attempt, errorMessage))
+                .details(String.format(UPGRADE_FAILED_FORMAT,
+                        compositeStateDetails.get(MODULE_SET_TAG_KEY), attempt, errorMessage))
                 .lockReasonCategory(lockReasonCategory).build());
     }
 
+    /**
+     * Extract lock reason details as key-value pair.
+     *
+     * @param compositeStateLockReason lock reason having all the details
+     * @return a map of lock reason details
+     */
+    public static Map<String, String> getLockedCompositeStateDetails(final CompositeState.LockReason
+                                                                             compositeStateLockReason) {
+        if (compositeStateLockReason != null) {
+            final Map<String, String> compositeStateDetails = new HashMap<>(2);
+            final String lockedCompositeStateReasonDetails = compositeStateLockReason.getDetails();
+            final Matcher retryAttemptMatcher = retryAttemptPattern.matcher(lockedCompositeStateReasonDetails);
+            if (retryAttemptMatcher.find()) {
+                final int attemptsRegexGroupId = 1;
+                compositeStateDetails.put(RETRY_ATTEMPT_KEY, retryAttemptMatcher.group(attemptsRegexGroupId));
+            }
+            final Matcher moduleSetTagMatcher = moduleSetTagPattern.matcher(lockedCompositeStateReasonDetails);
+            if (moduleSetTagMatcher.find()) {
+                final int moduleSetTagRegexGroupId = 1;
+                compositeStateDetails.put(MODULE_SET_TAG_KEY, moduleSetTagMatcher.group(moduleSetTagRegexGroupId));
+            }
+            return compositeStateDetails;
+        }
+        return Collections.emptyMap();
+    }
+
 
     /**
      * Check if a module sync retry is needed.
@@ -146,29 +176,12 @@ public class SyncUtils {
 
         final boolean failedDuringModuleSync = LockReasonCategory.MODULE_SYNC_FAILED
                 == lockReason.getLockReasonCategory();
-        final boolean moduleUpgrade = LockReasonCategory.MODULE_UPGRADE
+        final boolean failedDuringModuleUpgrade = LockReasonCategory.MODULE_UPGRADE_FAILED
                 == lockReason.getLockReasonCategory();
 
-        if (failedDuringModuleSync) {
-            final int timeInMinutesUntilNextAttempt;
-            final Matcher matcher = retryAttemptPattern.matcher(lockReason.getDetails());
-            if (matcher.find()) {
-                timeInMinutesUntilNextAttempt = (int) Math.pow(2, Integer.parseInt(matcher.group(1)));
-            } else {
-                timeInMinutesUntilNextAttempt = 1;
-                log.info("First Attempt: no current attempts found.");
-            }
-            final int timeSinceLastAttempt = (int) Duration.between(time, OffsetDateTime.now()).toMinutes();
-            if (timeInMinutesUntilNextAttempt >= timeSinceLastAttempt) {
-                log.info("Time until next attempt is {} minutes: ",
-                        timeInMinutesUntilNextAttempt - timeSinceLastAttempt);
-                return false;
-            }
-            log.info("Retry due now");
-            return true;
-        } else if (moduleUpgrade) {
-            log.info("Locked for module upgrade.");
-            return true;
+        if (failedDuringModuleSync || failedDuringModuleUpgrade) {
+            log.info("Locked for module {}.", failedDuringModuleSync ? "sync" : "upgrade");
+            return isRetryDue(lockReason, time);
         }
         log.info("Locked for other reason");
         return false;
@@ -191,6 +204,19 @@ public class SyncUtils {
         return null;
     }
 
+    /**
+     * Checks if cm handle state module is in upgrade or upgrade failed.
+     *
+     * @param compositeState current lock reason of  cm handle
+     * @return true or false based on lock reason category
+     */
+    public static boolean isInUpgradeOrUpgradeFailed(final CompositeState compositeState) {
+        return compositeState.getLockReason() != null
+                && (LockReasonCategory.MODULE_UPGRADE.equals(compositeState.getLockReason().getLockReasonCategory())
+                || LockReasonCategory.MODULE_UPGRADE_FAILED.equals(compositeState.getLockReason()
+                .getLockReasonCategory()));
+    }
+
     private String getFirstResource(final Object responseBody) {
         final String jsonObjectAsString = jsonObjectMapper.asJsonString(responseBody);
         final JsonNode overallJsonNode = jsonObjectMapper.convertToJsonNode(jsonObjectAsString);
@@ -199,10 +225,29 @@ public class SyncUtils {
         return jsonObjectMapper.asJsonString(Map.of(firstElement.getKey(), firstElement.getValue()));
     }
 
-    private static List<YangModelCmHandle> convertCmHandlesDataNodesToYangModelCmHandles(
+    private List<YangModelCmHandle> convertCmHandlesDataNodesToYangModelCmHandles(
             final List<DataNode> cmHandlesAsDataNodeList) {
         return cmHandlesAsDataNodeList.stream()
                 .map(cmHandle -> YangDataConverter.convertCmHandleToYangModel(cmHandle,
                         cmHandle.getLeaves().get("id").toString())).toList();
     }
+
+    private boolean isRetryDue(final CompositeState.LockReason compositeStateLockReason, final OffsetDateTime time) {
+        final int timeInMinutesUntilNextAttempt;
+        final Map<String, String> compositeStateDetails = getLockedCompositeStateDetails(compositeStateLockReason);
+        if (compositeStateDetails.isEmpty()) {
+            timeInMinutesUntilNextAttempt = 1;
+            log.info("First Attempt: no current attempts found.");
+        } else {
+            timeInMinutesUntilNextAttempt = (int) Math.pow(2, Integer.parseInt(compositeStateDetails
+                    .get(RETRY_ATTEMPT_KEY)));
+        }
+        final int timeSinceLastAttempt = (int) Duration.between(time, OffsetDateTime.now()).toMinutes();
+        if (timeInMinutesUntilNextAttempt >= timeSinceLastAttempt) {
+            log.info("Time until next attempt is {} minutes: ", timeInMinutesUntilNextAttempt - timeSinceLastAttempt);
+            return false;
+        }
+        log.info("Retry due now");
+        return true;
+    }
 }
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncService.java
index d191a5467..841368c0d 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncService.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncService.java
@@ -41,7 +41,6 @@ import org.onap.cps.api.CpsModuleService;
 import org.onap.cps.ncmp.api.impl.inventory.CmHandleQueries;
 import org.onap.cps.ncmp.api.impl.inventory.CmHandleState;
 import org.onap.cps.ncmp.api.impl.inventory.CompositeState;
-import org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory;
 import org.onap.cps.ncmp.api.impl.operations.DmiModelOperations;
 import org.onap.cps.ncmp.api.impl.utils.YangDataConverter;
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
@@ -77,10 +76,11 @@ public class ModuleSyncService {
         final String moduleSetTag;
         final String cmHandleId = yangModelCmHandle.getId();
         final CompositeState compositeState = yangModelCmHandle.getCompositeState();
-        final boolean inUpgrade = isInUpgrade(compositeState);
+        final boolean inUpgrade = ModuleOperationsUtils.isInUpgradeOrUpgradeFailed(compositeState);
 
         if (inUpgrade) {
-            moduleSetTag = extractModuleSetTag(compositeState);
+            moduleSetTag = ModuleOperationsUtils.getLockedCompositeStateDetails(compositeState.getLockReason())
+                    .get(ModuleOperationsUtils.MODULE_SET_TAG_KEY);
         } else {
             moduleSetTag = yangModelCmHandle.getModuleSetTag();
         }
@@ -174,12 +174,4 @@ public class ModuleSyncService {
         moduleSetTagCache.put(moduleSetTag, moduleReferencesFromExistingCmHandle);
     }
 
-    private static String extractModuleSetTag(final CompositeState compositeState) {
-        return compositeState.getLockReason().getDetails().split(":")[1].trim();
-    }
-
-    private static boolean isInUpgrade(final CompositeState compositeState) {
-        return compositeState.getLockReason() != null && LockReasonCategory.MODULE_UPGRADE.equals(
-                compositeState.getLockReason().getLockReasonCategory());
-    }
 }
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasks.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasks.java
index facaf155f..896316a49 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasks.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasks.java
@@ -44,7 +44,7 @@ import org.springframework.stereotype.Component;
 @Slf4j
 public class ModuleSyncTasks {
     private final InventoryPersistence inventoryPersistence;
-    private final SyncUtils syncUtils;
+    private final ModuleOperationsUtils moduleOperationsUtils;
     private final ModuleSyncService moduleSyncService;
     private final LcmEventsCmHandleStateHandler lcmEventsCmHandleStateHandler;
     private final IMap<String, Object> moduleSyncStartedOnCmHandles;
@@ -73,9 +73,14 @@ public class ModuleSyncTasks {
                     yangModelCmHandle.getCompositeState().setLockReason(null);
                     cmHandelStatePerCmHandle.put(yangModelCmHandle, CmHandleState.READY);
                 } catch (final Exception e) {
-                    log.warn("Processing of {} module sync failed due to reason {}.", cmHandleId, e.getMessage());
-                    syncUtils.updateLockReasonDetailsAndAttempts(compositeState, LockReasonCategory.MODULE_SYNC_FAILED,
-                            e.getMessage());
+                    log.warn("Processing of {} module failed due to reason {}.", cmHandleId, e.getMessage());
+                    if (ModuleOperationsUtils.isInUpgradeOrUpgradeFailed(compositeState)) {
+                        moduleOperationsUtils.updateLockReasonDetailsAndAttempts(compositeState,
+                                LockReasonCategory.MODULE_UPGRADE_FAILED, e.getMessage());
+                    } else {
+                        moduleOperationsUtils.updateLockReasonDetailsAndAttempts(compositeState,
+                                LockReasonCategory.MODULE_SYNC_FAILED, e.getMessage());
+                    }
                     setCmHandleStateLocked(yangModelCmHandle, compositeState.getLockReason());
                     cmHandelStatePerCmHandle.put(yangModelCmHandle, CmHandleState.LOCKED);
                 }
@@ -98,7 +103,7 @@ public class ModuleSyncTasks {
         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(failedCmHandles.size());
         for (final YangModelCmHandle failedCmHandle : failedCmHandles) {
             final CompositeState compositeState = failedCmHandle.getCompositeState();
-            final boolean isReadyForRetry = syncUtils.needsModuleSyncRetryOrUpgrade(compositeState);
+            final boolean isReadyForRetry = moduleOperationsUtils.needsModuleSyncRetryOrUpgrade(compositeState);
             log.info("Retry for cmHandleId : {} is {}", failedCmHandle.getId(), isReadyForRetry);
             if (isReadyForRetry) {
                 final String resetCmHandleId = failedCmHandle.getId();
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdog.java
index bf005051f..249232d23 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdog.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdog.java
@@ -43,7 +43,7 @@ import org.springframework.stereotype.Service;
 @Service
 public class ModuleSyncWatchdog {
 
-    private final SyncUtils syncUtils;
+    private final ModuleOperationsUtils moduleOperationsUtils;
     private final BlockingQueue<DataNode> moduleSyncWorkQueue;
     private final IMap<String, Object> moduleSyncStartedOnCmHandles;
     private final ModuleSyncTasks moduleSyncTasks;
@@ -88,7 +88,8 @@ public class ModuleSyncWatchdog {
     @Scheduled(fixedDelayString = "${ncmp.timers.locked-modules-sync.sleep-time-ms:300000}")
     public void resetPreviouslyFailedCmHandles() {
         log.info("Processing module sync retry-watchdog waking up.");
-        final List<YangModelCmHandle> failedCmHandles = syncUtils.getCmHandlesThatFailedModelSyncOrUpgrade();
+        final List<YangModelCmHandle> failedCmHandles
+                = moduleOperationsUtils.getCmHandlesThatFailedModelSyncOrUpgrade();
         log.info("Retrying {} cmHandles", failedCmHandles.size());
         moduleSyncTasks.resetFailedCmHandles(failedCmHandles);
     }
@@ -104,7 +105,7 @@ public class ModuleSyncWatchdog {
 
     private void populateWorkQueueIfNeeded() {
         if (moduleSyncWorkQueue.isEmpty()) {
-            final List<DataNode> advisedCmHandles = syncUtils.getAdvisedCmHandles();
+            final List<DataNode> advisedCmHandles = moduleOperationsUtils.getAdvisedCmHandles();
             log.info("Processing module sync fetched {} advised cm handles from DB", advisedCmHandles.size());
             for (final DataNode advisedCmHandle : advisedCmHandles) {
                 if (!moduleSyncWorkQueue.offer(advisedCmHandle)) {