From 13b8698611fcf3da3629d67360ba84d48f6049d7 Mon Sep 17 00:00:00 2001 From: sourabh_sourabh Date: Wed, 8 Nov 2023 09:39:35 +0000 Subject: Retry for yang module upgrade operation - Added retry code for yang module upgrade failed Issue-ID: CPS-1802 Signed-off-by: sourabh_sourabh Change-Id: Ida68d45f926ed48b92f4b5a3b82e641d778303ce Signed-off-by: sourabh_sourabh --- .../api/impl/NetworkCmProxyDataServiceImpl.java | 4 +- .../api/impl/inventory/sync/DataSyncWatchdog.java | 6 +- .../impl/inventory/sync/ModuleOperationsUtils.java | 253 +++++++++++++++++++++ .../api/impl/inventory/sync/ModuleSyncService.java | 14 +- .../api/impl/inventory/sync/ModuleSyncTasks.java | 15 +- .../impl/inventory/sync/ModuleSyncWatchdog.java | 7 +- .../ncmp/api/impl/inventory/sync/SyncUtils.java | 208 ----------------- 7 files changed, 276 insertions(+), 231 deletions(-) create mode 100644 cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtils.java delete mode 100644 cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtils.java (limited to 'cps-ncmp-service/src/main/java/org') diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java index db7b12cbc8..f0336900f9 100755 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java @@ -60,6 +60,7 @@ import org.onap.cps.ncmp.api.impl.inventory.CompositeStateBuilder; import org.onap.cps.ncmp.api.impl.inventory.CompositeStateUtils; import org.onap.cps.ncmp.api.impl.inventory.DataStoreSyncState; import org.onap.cps.ncmp.api.impl.inventory.InventoryPersistence; +import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleOperationsUtils; import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations; import org.onap.cps.ncmp.api.impl.operations.OperationType; import org.onap.cps.ncmp.api.impl.trustlevel.TrustLevel; @@ -409,7 +410,8 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService final NcmpServiceCmHandle ncmpServiceCmHandle = new NcmpServiceCmHandle(); ncmpServiceCmHandle.setCmHandleId(cmHandleId); final String moduleSetTag = dmiPluginRegistration.getUpgradedCmHandles().getModuleSetTag(); - final String lockReasonWithModuleSetTag = MessageFormat.format("ModuleSetTag: {0}", moduleSetTag); + final String lockReasonWithModuleSetTag = MessageFormat.format( + ModuleOperationsUtils.MODULE_SET_TAG_MESSAGE_FORMAT, moduleSetTag); ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withCmHandleState(CmHandleState.READY) .withLockReason(MODULE_UPGRADE, lockReasonWithModuleSetTag).build()); return YangModelCmHandle.toYangModelCmHandle(dmiPluginRegistration.getDmiPlugin(), diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdog.java index 49804adc13..6f089a57fb 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdog.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdog.java @@ -48,7 +48,7 @@ public class DataSyncWatchdog { private final CpsDataService cpsDataService; - private final SyncUtils syncUtils; + private final ModuleOperationsUtils moduleOperationsUtils; private final IMap dataSyncSemaphores; @@ -58,13 +58,13 @@ public class DataSyncWatchdog { */ @Scheduled(fixedDelayString = "${ncmp.timers.cm-handle-data-sync.sleep-time-ms:30000}") public void executeUnSynchronizedReadyCmHandlePoll() { - syncUtils.getUnsynchronizedReadyCmHandles().forEach(unSynchronizedReadyCmHandle -> { + moduleOperationsUtils.getUnsynchronizedReadyCmHandles().forEach(unSynchronizedReadyCmHandle -> { final String cmHandleId = unSynchronizedReadyCmHandle.getId(); if (hasPushedIntoSemaphoreMap(cmHandleId)) { log.debug("Executing data sync on {}", cmHandleId); final CompositeState compositeState = inventoryPersistence .getCmHandleState(cmHandleId); - final String resourceData = syncUtils.getResourceData(cmHandleId); + final String resourceData = moduleOperationsUtils.getResourceData(cmHandleId); if (resourceData == null) { log.debug("Error retrieving resource data for Cm-Handle: {}", cmHandleId); } else { diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtils.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtils.java new file mode 100644 index 0000000000..25ea3bc1e6 --- /dev/null +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtils.java @@ -0,0 +1,253 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2022-2023 Nordix Foundation + * Modifications Copyright (C) 2022 Bell Canada + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.api.impl.inventory.sync; + +import static org.onap.cps.ncmp.api.impl.operations.DatastoreType.PASSTHROUGH_OPERATIONAL; + +import com.fasterxml.jackson.databind.JsonNode; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.onap.cps.ncmp.api.impl.inventory.CmHandleQueries; +import org.onap.cps.ncmp.api.impl.inventory.CmHandleState; +import org.onap.cps.ncmp.api.impl.inventory.CompositeState; +import org.onap.cps.ncmp.api.impl.inventory.DataStoreSyncState; +import org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory; +import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations; +import org.onap.cps.ncmp.api.impl.utils.YangDataConverter; +import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle; +import org.onap.cps.spi.FetchDescendantsOption; +import org.onap.cps.spi.model.DataNode; +import org.onap.cps.utils.JsonObjectMapper; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ModuleOperationsUtils { + + private final CmHandleQueries cmHandleQueries; + private final DmiDataOperations dmiDataOperations; + private final JsonObjectMapper jsonObjectMapper; + private static final String RETRY_ATTEMPT_KEY = "attempt"; + public static final String MODULE_SET_TAG_KEY = "moduleSetTag"; + public static final String MODULE_SET_TAG_MESSAGE_FORMAT = "Upgrade to ModuleSetTag: {0}"; + private static final String UPGRADE_FORMAT = "Upgrade to ModuleSetTag: %s"; + private static final String UPGRADE_FAILED_FORMAT = UPGRADE_FORMAT + " Attempt #%d failed: %s"; + private static final Pattern retryAttemptPattern = Pattern.compile("Attempt #(\\d+) failed:.+"); + private static final Pattern moduleSetTagPattern = Pattern.compile("Upgrade to ModuleSetTag: (\\S+)"); + + /** + * Query data nodes for cm handles with an "ADVISED" cm handle state. + * + * @return cm handles (data nodes) in ADVISED state (empty list if none found) + */ + public List getAdvisedCmHandles() { + final List advisedCmHandlesAsDataNodes = cmHandleQueries.queryCmHandlesByState(CmHandleState.ADVISED); + log.debug("Total number of fetched advised cm handle(s) is (are) {}", advisedCmHandlesAsDataNodes.size()); + return advisedCmHandlesAsDataNodes; + } + + /** + * First query data nodes for cm handles with CM Handle Operational Sync State in "UNSYNCHRONIZED" and + * randomly select a CM Handle and query the data nodes for CM Handle State in "READY". + * + * @return a randomized yang model cm handle list with State in READY and Operation Sync State in "UNSYNCHRONIZED", + * return empty list if not found + */ + public List getUnsynchronizedReadyCmHandles() { + final List unsynchronizedCmHandles = cmHandleQueries + .queryCmHandlesByOperationalSyncState(DataStoreSyncState.UNSYNCHRONIZED); + + final List yangModelCmHandles = new ArrayList<>(); + for (final DataNode unsynchronizedCmHandle : unsynchronizedCmHandles) { + final String cmHandleId = unsynchronizedCmHandle.getLeaves().get("id").toString(); + if (cmHandleQueries.cmHandleHasState(cmHandleId, CmHandleState.READY)) { + yangModelCmHandles.addAll(convertCmHandlesDataNodesToYangModelCmHandles( + Collections.singletonList(unsynchronizedCmHandle))); + } + } + Collections.shuffle(yangModelCmHandles); + return yangModelCmHandles; + } + + /** + * Query data nodes for cm handles with an "LOCKED" cm handle state with reason. + * + * @return a random LOCKED yang model cm handle, return null if not found + */ + public List getCmHandlesThatFailedModelSyncOrUpgrade() { + final List lockedCmHandlesAsDataNodeList + = cmHandleQueries.queryCmHandleAncestorsByCpsPath( + "//lock-reason[@reason=\"MODULE_SYNC_FAILED\" or @reason=\"MODULE_UPGRADE\"]", + FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS); + return convertCmHandlesDataNodesToYangModelCmHandles(lockedCmHandlesAsDataNodeList); + } + + /** + * Update Composite State attempts counter and set new lock reason and details. + * + * @param lockReasonCategory lock reason category + * @param errorMessage error message + */ + public void updateLockReasonDetailsAndAttempts(final CompositeState compositeState, + final LockReasonCategory lockReasonCategory, + final String errorMessage) { + int attempt = 1; + final Map compositeStateDetails + = getLockedCompositeStateDetails(compositeState.getLockReason()); + if (!compositeStateDetails.isEmpty()) { + attempt = 1 + Integer.parseInt(compositeStateDetails.get(RETRY_ATTEMPT_KEY)); + } + compositeState.setLockReason(CompositeState.LockReason.builder() + .details(String.format(UPGRADE_FAILED_FORMAT, + compositeStateDetails.get(MODULE_SET_TAG_KEY), attempt, errorMessage)) + .lockReasonCategory(lockReasonCategory).build()); + } + + /** + * Extract lock reason details as key-value pair. + * + * @param compositeStateLockReason lock reason having all the details + * @return a map of lock reason details + */ + public static Map getLockedCompositeStateDetails(final CompositeState.LockReason + compositeStateLockReason) { + if (compositeStateLockReason != null) { + final Map compositeStateDetails = new HashMap<>(2); + final String lockedCompositeStateReasonDetails = compositeStateLockReason.getDetails(); + final Matcher retryAttemptMatcher = retryAttemptPattern.matcher(lockedCompositeStateReasonDetails); + if (retryAttemptMatcher.find()) { + final int attemptsRegexGroupId = 1; + compositeStateDetails.put(RETRY_ATTEMPT_KEY, retryAttemptMatcher.group(attemptsRegexGroupId)); + } + final Matcher moduleSetTagMatcher = moduleSetTagPattern.matcher(lockedCompositeStateReasonDetails); + if (moduleSetTagMatcher.find()) { + final int moduleSetTagRegexGroupId = 1; + compositeStateDetails.put(MODULE_SET_TAG_KEY, moduleSetTagMatcher.group(moduleSetTagRegexGroupId)); + } + return compositeStateDetails; + } + return Collections.emptyMap(); + } + + + /** + * Check if a module sync retry is needed. + * + * @param compositeState the composite state currently in the locked state + * @return if the retry mechanism should be attempted + */ + public boolean needsModuleSyncRetryOrUpgrade(final CompositeState compositeState) { + final OffsetDateTime time = OffsetDateTime.parse(compositeState.getLastUpdateTime(), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ")); + final CompositeState.LockReason lockReason = compositeState.getLockReason(); + + final boolean failedDuringModuleSync = LockReasonCategory.MODULE_SYNC_FAILED + == lockReason.getLockReasonCategory(); + final boolean failedDuringModuleUpgrade = LockReasonCategory.MODULE_UPGRADE_FAILED + == lockReason.getLockReasonCategory(); + + if (failedDuringModuleSync || failedDuringModuleUpgrade) { + log.info("Locked for module {}.", failedDuringModuleSync ? "sync" : "upgrade"); + return isRetryDue(lockReason, time); + } + log.info("Locked for other reason"); + return false; + } + + /** + * Get the Resourece Data from Node through DMI Passthrough service. + * + * @param cmHandleId cm handle id + * @return optional string containing the resource data + */ + public String getResourceData(final String cmHandleId) { + final ResponseEntity resourceDataResponseEntity = dmiDataOperations.getResourceDataFromDmi( + PASSTHROUGH_OPERATIONAL.getDatastoreName(), + cmHandleId, + UUID.randomUUID().toString()); + if (resourceDataResponseEntity.getStatusCode().is2xxSuccessful()) { + return getFirstResource(resourceDataResponseEntity.getBody()); + } + return null; + } + + /** + * Checks if cm handle state module is in upgrade or upgrade failed. + * + * @param compositeState current lock reason of cm handle + * @return true or false based on lock reason category + */ + public static boolean isInUpgradeOrUpgradeFailed(final CompositeState compositeState) { + return compositeState.getLockReason() != null + && (LockReasonCategory.MODULE_UPGRADE.equals(compositeState.getLockReason().getLockReasonCategory()) + || LockReasonCategory.MODULE_UPGRADE_FAILED.equals(compositeState.getLockReason() + .getLockReasonCategory())); + } + + private String getFirstResource(final Object responseBody) { + final String jsonObjectAsString = jsonObjectMapper.asJsonString(responseBody); + final JsonNode overallJsonNode = jsonObjectMapper.convertToJsonNode(jsonObjectAsString); + final Iterator> overallJsonTreeMap = overallJsonNode.fields(); + final Map.Entry firstElement = overallJsonTreeMap.next(); + return jsonObjectMapper.asJsonString(Map.of(firstElement.getKey(), firstElement.getValue())); + } + + private List convertCmHandlesDataNodesToYangModelCmHandles( + final List cmHandlesAsDataNodeList) { + return cmHandlesAsDataNodeList.stream() + .map(cmHandle -> YangDataConverter.convertCmHandleToYangModel(cmHandle, + cmHandle.getLeaves().get("id").toString())).toList(); + } + + private boolean isRetryDue(final CompositeState.LockReason compositeStateLockReason, final OffsetDateTime time) { + final int timeInMinutesUntilNextAttempt; + final Map compositeStateDetails = getLockedCompositeStateDetails(compositeStateLockReason); + if (compositeStateDetails.isEmpty()) { + timeInMinutesUntilNextAttempt = 1; + log.info("First Attempt: no current attempts found."); + } else { + timeInMinutesUntilNextAttempt = (int) Math.pow(2, Integer.parseInt(compositeStateDetails + .get(RETRY_ATTEMPT_KEY))); + } + final int timeSinceLastAttempt = (int) Duration.between(time, OffsetDateTime.now()).toMinutes(); + if (timeInMinutesUntilNextAttempt >= timeSinceLastAttempt) { + log.info("Time until next attempt is {} minutes: ", timeInMinutesUntilNextAttempt - timeSinceLastAttempt); + return false; + } + log.info("Retry due now"); + return true; + } +} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncService.java index d191a54677..841368c0db 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncService.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncService.java @@ -41,7 +41,6 @@ import org.onap.cps.api.CpsModuleService; import org.onap.cps.ncmp.api.impl.inventory.CmHandleQueries; import org.onap.cps.ncmp.api.impl.inventory.CmHandleState; import org.onap.cps.ncmp.api.impl.inventory.CompositeState; -import org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory; import org.onap.cps.ncmp.api.impl.operations.DmiModelOperations; import org.onap.cps.ncmp.api.impl.utils.YangDataConverter; import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle; @@ -77,10 +76,11 @@ public class ModuleSyncService { final String moduleSetTag; final String cmHandleId = yangModelCmHandle.getId(); final CompositeState compositeState = yangModelCmHandle.getCompositeState(); - final boolean inUpgrade = isInUpgrade(compositeState); + final boolean inUpgrade = ModuleOperationsUtils.isInUpgradeOrUpgradeFailed(compositeState); if (inUpgrade) { - moduleSetTag = extractModuleSetTag(compositeState); + moduleSetTag = ModuleOperationsUtils.getLockedCompositeStateDetails(compositeState.getLockReason()) + .get(ModuleOperationsUtils.MODULE_SET_TAG_KEY); } else { moduleSetTag = yangModelCmHandle.getModuleSetTag(); } @@ -174,12 +174,4 @@ public class ModuleSyncService { moduleSetTagCache.put(moduleSetTag, moduleReferencesFromExistingCmHandle); } - private static String extractModuleSetTag(final CompositeState compositeState) { - return compositeState.getLockReason().getDetails().split(":")[1].trim(); - } - - private static boolean isInUpgrade(final CompositeState compositeState) { - return compositeState.getLockReason() != null && LockReasonCategory.MODULE_UPGRADE.equals( - compositeState.getLockReason().getLockReasonCategory()); - } } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasks.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasks.java index facaf155fe..896316a496 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasks.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasks.java @@ -44,7 +44,7 @@ import org.springframework.stereotype.Component; @Slf4j public class ModuleSyncTasks { private final InventoryPersistence inventoryPersistence; - private final SyncUtils syncUtils; + private final ModuleOperationsUtils moduleOperationsUtils; private final ModuleSyncService moduleSyncService; private final LcmEventsCmHandleStateHandler lcmEventsCmHandleStateHandler; private final IMap moduleSyncStartedOnCmHandles; @@ -73,9 +73,14 @@ public class ModuleSyncTasks { yangModelCmHandle.getCompositeState().setLockReason(null); cmHandelStatePerCmHandle.put(yangModelCmHandle, CmHandleState.READY); } catch (final Exception e) { - log.warn("Processing of {} module sync failed due to reason {}.", cmHandleId, e.getMessage()); - syncUtils.updateLockReasonDetailsAndAttempts(compositeState, LockReasonCategory.MODULE_SYNC_FAILED, - e.getMessage()); + log.warn("Processing of {} module failed due to reason {}.", cmHandleId, e.getMessage()); + if (ModuleOperationsUtils.isInUpgradeOrUpgradeFailed(compositeState)) { + moduleOperationsUtils.updateLockReasonDetailsAndAttempts(compositeState, + LockReasonCategory.MODULE_UPGRADE_FAILED, e.getMessage()); + } else { + moduleOperationsUtils.updateLockReasonDetailsAndAttempts(compositeState, + LockReasonCategory.MODULE_SYNC_FAILED, e.getMessage()); + } setCmHandleStateLocked(yangModelCmHandle, compositeState.getLockReason()); cmHandelStatePerCmHandle.put(yangModelCmHandle, CmHandleState.LOCKED); } @@ -98,7 +103,7 @@ public class ModuleSyncTasks { final Map cmHandleStatePerCmHandle = new HashMap<>(failedCmHandles.size()); for (final YangModelCmHandle failedCmHandle : failedCmHandles) { final CompositeState compositeState = failedCmHandle.getCompositeState(); - final boolean isReadyForRetry = syncUtils.needsModuleSyncRetryOrUpgrade(compositeState); + final boolean isReadyForRetry = moduleOperationsUtils.needsModuleSyncRetryOrUpgrade(compositeState); log.info("Retry for cmHandleId : {} is {}", failedCmHandle.getId(), isReadyForRetry); if (isReadyForRetry) { final String resetCmHandleId = failedCmHandle.getId(); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdog.java index bf005051f9..249232d230 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdog.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdog.java @@ -43,7 +43,7 @@ import org.springframework.stereotype.Service; @Service public class ModuleSyncWatchdog { - private final SyncUtils syncUtils; + private final ModuleOperationsUtils moduleOperationsUtils; private final BlockingQueue moduleSyncWorkQueue; private final IMap moduleSyncStartedOnCmHandles; private final ModuleSyncTasks moduleSyncTasks; @@ -88,7 +88,8 @@ public class ModuleSyncWatchdog { @Scheduled(fixedDelayString = "${ncmp.timers.locked-modules-sync.sleep-time-ms:300000}") public void resetPreviouslyFailedCmHandles() { log.info("Processing module sync retry-watchdog waking up."); - final List failedCmHandles = syncUtils.getCmHandlesThatFailedModelSyncOrUpgrade(); + final List failedCmHandles + = moduleOperationsUtils.getCmHandlesThatFailedModelSyncOrUpgrade(); log.info("Retrying {} cmHandles", failedCmHandles.size()); moduleSyncTasks.resetFailedCmHandles(failedCmHandles); } @@ -104,7 +105,7 @@ public class ModuleSyncWatchdog { private void populateWorkQueueIfNeeded() { if (moduleSyncWorkQueue.isEmpty()) { - final List advisedCmHandles = syncUtils.getAdvisedCmHandles(); + final List advisedCmHandles = moduleOperationsUtils.getAdvisedCmHandles(); log.info("Processing module sync fetched {} advised cm handles from DB", advisedCmHandles.size()); for (final DataNode advisedCmHandle : advisedCmHandles) { if (!moduleSyncWorkQueue.offer(advisedCmHandle)) { diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtils.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtils.java deleted file mode 100644 index ab85c2127e..0000000000 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtils.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation - * Modifications Copyright (C) 2022 Bell Canada - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ncmp.api.impl.inventory.sync; - -import static org.onap.cps.ncmp.api.impl.operations.DatastoreType.PASSTHROUGH_OPERATIONAL; - -import com.fasterxml.jackson.databind.JsonNode; -import java.time.Duration; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.onap.cps.ncmp.api.impl.inventory.CmHandleQueries; -import org.onap.cps.ncmp.api.impl.inventory.CmHandleState; -import org.onap.cps.ncmp.api.impl.inventory.CompositeState; -import org.onap.cps.ncmp.api.impl.inventory.DataStoreSyncState; -import org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory; -import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations; -import org.onap.cps.ncmp.api.impl.utils.YangDataConverter; -import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle; -import org.onap.cps.spi.FetchDescendantsOption; -import org.onap.cps.spi.model.DataNode; -import org.onap.cps.utils.JsonObjectMapper; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -@RequiredArgsConstructor -public class SyncUtils { - - private final CmHandleQueries cmHandleQueries; - private final DmiDataOperations dmiDataOperations; - private final JsonObjectMapper jsonObjectMapper; - private static final Pattern retryAttemptPattern = Pattern.compile("^Attempt #(\\d+) failed:"); - - /** - * Query data nodes for cm handles with an "ADVISED" cm handle state. - * - * @return cm handles (data nodes) in ADVISED state (empty list if none found) - */ - public List getAdvisedCmHandles() { - final List advisedCmHandlesAsDataNodes = cmHandleQueries.queryCmHandlesByState(CmHandleState.ADVISED); - log.debug("Total number of fetched advised cm handle(s) is (are) {}", advisedCmHandlesAsDataNodes.size()); - return advisedCmHandlesAsDataNodes; - } - - /** - * First query data nodes for cm handles with CM Handle Operational Sync State in "UNSYNCHRONIZED" and - * randomly select a CM Handle and query the data nodes for CM Handle State in "READY". - * - * @return a randomized yang model cm handle list with State in READY and Operation Sync State in "UNSYNCHRONIZED", - * return empty list if not found - */ - public List getUnsynchronizedReadyCmHandles() { - final List unsynchronizedCmHandles = cmHandleQueries - .queryCmHandlesByOperationalSyncState(DataStoreSyncState.UNSYNCHRONIZED); - - final List yangModelCmHandles = new ArrayList<>(); - for (final DataNode unsynchronizedCmHandle : unsynchronizedCmHandles) { - final String cmHandleId = unsynchronizedCmHandle.getLeaves().get("id").toString(); - if (cmHandleQueries.cmHandleHasState(cmHandleId, CmHandleState.READY)) { - yangModelCmHandles.addAll( - convertCmHandlesDataNodesToYangModelCmHandles( - Collections.singletonList(unsynchronizedCmHandle))); - } - } - - Collections.shuffle(yangModelCmHandles); - - return yangModelCmHandles; - } - - /** - * Query data nodes for cm handles with an "LOCKED" cm handle state with reason. - * - * @return a random LOCKED yang model cm handle, return null if not found - */ - public List getCmHandlesThatFailedModelSyncOrUpgrade() { - final List lockedCmHandlesAsDataNodeList - = cmHandleQueries.queryCmHandleAncestorsByCpsPath( - "//lock-reason[@reason=\"MODULE_SYNC_FAILED\" or @reason=\"MODULE_UPGRADE\"]", - FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS); - return convertCmHandlesDataNodesToYangModelCmHandles(lockedCmHandlesAsDataNodeList); - } - - /** - * Update Composite State attempts counter and set new lock reason and details. - * - * @param lockReasonCategory lock reason category - * @param errorMessage error message - */ - public void updateLockReasonDetailsAndAttempts(final CompositeState compositeState, - final LockReasonCategory lockReasonCategory, - final String errorMessage) { - int attempt = 1; - if (compositeState.getLockReason() != null) { - final Matcher matcher = retryAttemptPattern.matcher(compositeState.getLockReason().getDetails()); - if (matcher.find()) { - attempt = 1 + Integer.parseInt(matcher.group(1)); - } - } - compositeState.setLockReason(CompositeState.LockReason.builder() - .details(String.format("Attempt #%d failed: %s", attempt, errorMessage)) - .lockReasonCategory(lockReasonCategory).build()); - } - - - /** - * Check if a module sync retry is needed. - * - * @param compositeState the composite state currently in the locked state - * @return if the retry mechanism should be attempted - */ - public boolean needsModuleSyncRetryOrUpgrade(final CompositeState compositeState) { - final OffsetDateTime time = OffsetDateTime.parse(compositeState.getLastUpdateTime(), - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ")); - final CompositeState.LockReason lockReason = compositeState.getLockReason(); - - final boolean failedDuringModuleSync = LockReasonCategory.MODULE_SYNC_FAILED - == lockReason.getLockReasonCategory(); - final boolean moduleUpgrade = LockReasonCategory.MODULE_UPGRADE - == lockReason.getLockReasonCategory(); - - if (failedDuringModuleSync) { - final int timeInMinutesUntilNextAttempt; - final Matcher matcher = retryAttemptPattern.matcher(lockReason.getDetails()); - if (matcher.find()) { - timeInMinutesUntilNextAttempt = (int) Math.pow(2, Integer.parseInt(matcher.group(1))); - } else { - timeInMinutesUntilNextAttempt = 1; - log.info("First Attempt: no current attempts found."); - } - final int timeSinceLastAttempt = (int) Duration.between(time, OffsetDateTime.now()).toMinutes(); - if (timeInMinutesUntilNextAttempt >= timeSinceLastAttempt) { - log.info("Time until next attempt is {} minutes: ", - timeInMinutesUntilNextAttempt - timeSinceLastAttempt); - return false; - } - log.info("Retry due now"); - return true; - } else if (moduleUpgrade) { - log.info("Locked for module upgrade."); - return true; - } - log.info("Locked for other reason"); - return false; - } - - /** - * Get the Resourece Data from Node through DMI Passthrough service. - * - * @param cmHandleId cm handle id - * @return optional string containing the resource data - */ - public String getResourceData(final String cmHandleId) { - final ResponseEntity resourceDataResponseEntity = dmiDataOperations.getResourceDataFromDmi( - PASSTHROUGH_OPERATIONAL.getDatastoreName(), - cmHandleId, - UUID.randomUUID().toString()); - if (resourceDataResponseEntity.getStatusCode().is2xxSuccessful()) { - return getFirstResource(resourceDataResponseEntity.getBody()); - } - return null; - } - - private String getFirstResource(final Object responseBody) { - final String jsonObjectAsString = jsonObjectMapper.asJsonString(responseBody); - final JsonNode overallJsonNode = jsonObjectMapper.convertToJsonNode(jsonObjectAsString); - final Iterator> overallJsonTreeMap = overallJsonNode.fields(); - final Map.Entry firstElement = overallJsonTreeMap.next(); - return jsonObjectMapper.asJsonString(Map.of(firstElement.getKey(), firstElement.getValue())); - } - - private static List convertCmHandlesDataNodesToYangModelCmHandles( - final List cmHandlesAsDataNodeList) { - return cmHandlesAsDataNodeList.stream() - .map(cmHandle -> YangDataConverter.convertCmHandleToYangModel(cmHandle, - cmHandle.getLeaves().get("id").toString())).toList(); - } -} -- cgit