From 13b8698611fcf3da3629d67360ba84d48f6049d7 Mon Sep 17 00:00:00 2001 From: sourabh_sourabh Date: Wed, 8 Nov 2023 09:39:35 +0000 Subject: Retry for yang module upgrade operation - Added retry code for yang module upgrade failed Issue-ID: CPS-1802 Signed-off-by: sourabh_sourabh Change-Id: Ida68d45f926ed48b92f4b5a3b82e641d778303ce Signed-off-by: sourabh_sourabh --- .../inventory/sync/DataSyncWatchdogSpec.groovy | 4 +- .../sync/ModuleOperationsUtilsSpec.groovy | 211 +++++++++++++++++++++ .../inventory/sync/ModuleSyncServiceSpec.groovy | 13 +- .../impl/inventory/sync/ModuleSyncTasksSpec.groovy | 42 ++-- .../inventory/sync/ModuleSyncWatchdogSpec.groovy | 5 +- .../api/impl/inventory/sync/SyncUtilsSpec.groovy | 198 ------------------- 6 files changed, 247 insertions(+), 226 deletions(-) create mode 100644 cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy delete mode 100644 cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtilsSpec.groovy (limited to 'cps-ncmp-service/src/test/groovy/org') diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdogSpec.groovy index 65dfc05791..0ffb567916 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdogSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdogSpec.groovy @@ -24,8 +24,6 @@ import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NFP_OPE import com.hazelcast.map.IMap import org.onap.cps.api.CpsDataService -import org.onap.cps.ncmp.api.impl.inventory.sync.DataSyncWatchdog -import org.onap.cps.ncmp.api.impl.inventory.sync.SyncUtils import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle import org.onap.cps.ncmp.api.impl.inventory.CmHandleState import org.onap.cps.ncmp.api.impl.inventory.CompositeState @@ -39,7 +37,7 @@ class DataSyncWatchdogSpec extends Specification { def mockCpsDataService = Mock(CpsDataService) - def mockSyncUtils = Mock(SyncUtils) + def mockSyncUtils = Mock(ModuleOperationsUtils) def mockDataSyncSemaphores = Mock(IMap) diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy new file mode 100644 index 0000000000..099fc5ac18 --- /dev/null +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy @@ -0,0 +1,211 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2022-2023 Nordix Foundation + * Modifications Copyright (C) 2022 Bell Canada + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.api.impl.inventory.sync + +import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.LOCKED_MISBEHAVING +import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE +import static org.onap.cps.ncmp.api.impl.operations.DatastoreType.PASSTHROUGH_OPERATIONAL +import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_SYNC_FAILED +import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE_FAILED + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.core.read.ListAppender +import org.slf4j.LoggerFactory +import org.springframework.context.annotation.AnnotationConfigApplicationContext +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations +import org.onap.cps.ncmp.api.impl.inventory.CmHandleQueries +import org.onap.cps.ncmp.api.impl.inventory.CmHandleState +import org.onap.cps.ncmp.api.impl.inventory.CompositeState +import org.onap.cps.ncmp.api.impl.inventory.CompositeStateBuilder +import org.onap.cps.ncmp.api.impl.inventory.DataStoreSyncState +import org.onap.cps.spi.FetchDescendantsOption +import org.onap.cps.spi.model.DataNode +import org.onap.cps.utils.JsonObjectMapper +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity +import spock.lang.Specification +import java.time.OffsetDateTime +import java.time.format.DateTimeFormatter +import java.util.stream.Collectors + +class ModuleOperationsUtilsSpec extends Specification{ + + def mockCmHandleQueries = Mock(CmHandleQueries) + + def mockDmiDataOperations = Mock(DmiDataOperations) + + def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper()) + + def objectUnderTest = new ModuleOperationsUtils(mockCmHandleQueries, mockDmiDataOperations, jsonObjectMapper) + + def static neverUpdatedBefore = '1900-01-01T00:00:00.000+0100' + + def static now = OffsetDateTime.now() + + def static nowAsString = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(now) + + def static dataNode = new DataNode(leaves: ['id': 'cm-handle-123']) + + def applicationContext = new AnnotationConfigApplicationContext() + + def logger = (Logger) LoggerFactory.getLogger(ModuleOperationsUtils) + def loggingListAppender + + void setup() { + logger.setLevel(Level.DEBUG) + loggingListAppender = new ListAppender() + logger.addAppender(loggingListAppender) + loggingListAppender.start() + applicationContext.refresh() + } + + void cleanup() { + ((Logger) LoggerFactory.getLogger(ModuleOperationsUtils.class)).detachAndStopAllAppenders() + applicationContext.close() + } + + def 'Get an advised Cm-Handle where ADVISED cm handle #scenario'() { + given: 'the inventory persistence service returns a collection of data nodes' + mockCmHandleQueries.queryCmHandlesByState(CmHandleState.ADVISED) >> dataNodeCollection + when: 'get advised cm handles are fetched' + def yangModelCmHandles = objectUnderTest.getAdvisedCmHandles() + then: 'the returned data node collection is the correct size' + yangModelCmHandles.size() == expectedDataNodeSize + where: 'the following scenarios are used' + scenario | dataNodeCollection || expectedCallsToGetYangModelCmHandle | expectedDataNodeSize + 'exists' | [dataNode] || 1 | 1 + 'does not exist' | [] || 0 | 0 + } + + def 'Update Lock Reason, Details and Attempts where lock reason #scenario'() { + given: 'A locked state' + def compositeState = new CompositeState(lockReason: lockReason) + when: 'update cm handle details and attempts is called' + objectUnderTest.updateLockReasonDetailsAndAttempts(compositeState, MODULE_SYNC_FAILED, 'new error message') + then: 'the composite state lock reason and details are updated' + assert compositeState.lockReason.lockReasonCategory == MODULE_SYNC_FAILED + assert compositeState.lockReason.details.contains(expectedDetails) + where: + scenario | lockReason || expectedDetails + 'does not exist' | null || 'Attempt #1 failed: new error message' + 'exists' | CompositeState.LockReason.builder().details("Attempt #2 failed: some error message").build() || 'Attempt #3 failed: new error message' + } + + def 'Get all locked Cm-Handle where Lock Reason is MODULE_SYNC_FAILED cm handle #scenario'() { + given: 'the cps (persistence service) returns a collection of data nodes' + mockCmHandleQueries.queryCmHandleAncestorsByCpsPath( + '//lock-reason[@reason="MODULE_SYNC_FAILED" or @reason="MODULE_UPGRADE"]', + FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) >> [dataNode] + when: 'get locked Misbehaving cm handle is called' + def result = objectUnderTest.getCmHandlesThatFailedModelSyncOrUpgrade() + then: 'the returned cm handle collection is the correct size' + result.size() == 1 + and: 'the correct cm handle is returned' + result[0].id == 'cm-handle-123' + } + + def 'Retry Locked Cm-Handle where the last update time is #scenario'() { + given: 'Last update was #lastUpdateMinutesAgo minutes ago (-1 means never)' + def lastUpdatedTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(now.minusMinutes(lastUpdateMinutesAgo)) + if (lastUpdateMinutesAgo < 0 ) { + lastUpdatedTime = neverUpdatedBefore + } + when: 'checking to see if cm handle is ready for retry' + def result = objectUnderTest.needsModuleSyncRetryOrUpgrade(new CompositeStateBuilder() + .withLockReason(MODULE_SYNC_FAILED, lockDetails) + .withLastUpdatedTime(lastUpdatedTime).build()) + then: 'retry is only attempted when expected' + assert result == retryExpected + and: 'logs contain related information' + def logs = loggingListAppender.list.toString() + assert logs.contains(logReason) + where: 'the following parameters are used' + scenario | lastUpdateMinutesAgo | lockDetails | logReason || retryExpected + 'never attempted before' | -1 | 'Fist attempt:' | 'First Attempt:' || true + '1st attempt, last attempt > 2 minute ago' | 3 | 'Attempt #1 failed: some error' | 'Retry due now' || true + '2nd attempt, last attempt < 4 minutes ago' | 1 | 'Attempt #2 failed: some error' | 'Time until next attempt is 3 minutes:' || false + '2nd attempt, last attempt > 4 minutes ago' | 5 | 'Attempt #2 failed: some error' | 'Retry due now' || true + } + + def 'Retry Locked Cm-Handle with lock reasons (category) #lockReasonCategory'() { + when: 'checking to see if cm handle is ready for retry' + def result = objectUnderTest.needsModuleSyncRetryOrUpgrade(new CompositeStateBuilder() + .withLockReason(lockReasonCategory, 'some details') + .withLastUpdatedTime(nowAsString).build()) + then: 'verify retry attempts' + assert !result + and: 'logs contain related information' + def logs = loggingListAppender.list.toString() + assert logs.contains(logReason) + where: 'the following lock reasons occurred' + scenario | lockReasonCategory || logReason + 'module upgrade' | MODULE_UPGRADE_FAILED || 'First Attempt:' + 'module sync failed' | MODULE_SYNC_FAILED || 'First Attempt:' + 'lock misbehaving' | LOCKED_MISBEHAVING || 'Locked for other reason' + } + + def 'Get a Cm-Handle where #scenario'() { + given: 'the inventory persistence service returns a collection of data nodes' + mockCmHandleQueries.queryCmHandlesByOperationalSyncState(DataStoreSyncState.UNSYNCHRONIZED) >> unSynchronizedDataNodes + mockCmHandleQueries.cmHandleHasState('cm-handle-123', CmHandleState.READY) >> cmHandleHasState + when: 'get advised cm handles are fetched' + def yangModelCollection = objectUnderTest.getUnsynchronizedReadyCmHandles() + then: 'the returned data node collection is the correct size' + yangModelCollection.size() == expectedDataNodeSize + and: 'the result contains the correct data' + yangModelCollection.stream().map(yangModel -> yangModel.id).collect(Collectors.toSet()) == expectedYangModelCollectionIds + where: 'the following scenarios are used' + scenario | unSynchronizedDataNodes | cmHandleHasState || expectedDataNodeSize | expectedYangModelCollectionIds + 'a Cm-Handle unsynchronized and ready' | [dataNode] | true || 1 | ['cm-handle-123'] as Set + 'a Cm-Handle unsynchronized but not ready' | [dataNode] | false || 0 | [] as Set + 'all Cm-Handle synchronized' | [] | false || 0 | [] as Set + } + + def 'Get resource data through DMI Operations #scenario'() { + given: 'the inventory persistence service returns a collection of data nodes' + def jsonString = '{"stores:bookstore":{"categories":[{"code":"01"}]}}' + JsonNode jsonNode = jsonObjectMapper.convertToJsonNode(jsonString); + def responseEntity = new ResponseEntity<>(jsonNode, HttpStatus.OK) + mockDmiDataOperations.getResourceDataFromDmi(PASSTHROUGH_OPERATIONAL.datastoreName, 'cm-handle-123', _) >> responseEntity + when: 'get resource data is called' + def result = objectUnderTest.getResourceData('cm-handle-123') + then: 'the returned data is correct' + result == jsonString + } + + def 'Extract module set tag and number of attempt when lock reason contains #scenario'() { + expect: 'lock reason details are extracted correctly' + def result = objectUnderTest.getLockedCompositeStateDetails(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, lockReasonDetails).build().lockReason) + and: 'the result contains the correct moduleSetTag' + assert result['moduleSetTag'] == expectedModuleSetTag + and: 'the result contains the correct number of attempts' + assert result['attempt'] == expectedNumberOfAttempts + where: 'the following scenarios are used' + scenario | lockReasonDetails || expectedModuleSetTag | expectedNumberOfAttempts + 'module set tag only' | 'Upgrade to ModuleSetTag: targetModuleSetTag' || 'targetModuleSetTag' | null + 'number of attempts only' | 'Attempt #1 failed: some error' || null | '1' + 'number of attempts and module set tag both' | 'Upgrade to ModuleSetTag: targetModuleSetTag Attempt #1 failed: some error' || 'targetModuleSetTag' | '1' + } +} diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncServiceSpec.groovy index 18b87af9a1..5384f31cc1 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncServiceSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncServiceSpec.groovy @@ -20,7 +20,6 @@ package org.onap.cps.ncmp.api.impl.inventory.sync -import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.LOCKED_MISBEHAVING import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE @@ -31,7 +30,6 @@ import org.onap.cps.api.CpsAdminService import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsModuleService import org.onap.cps.spi.model.DataNodeBuilder -import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncService import org.onap.cps.ncmp.api.impl.operations.DmiModelOperations import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle import org.onap.cps.ncmp.api.impl.inventory.CmHandleQueries @@ -90,7 +88,7 @@ class ModuleSyncServiceSpec extends Specification { def 'Upgrade model for an existing cm handle with Module Set Tag where the modules are #scenario'() { given: 'a cm handle being upgraded to module set tag: tag-1' def ncmpServiceCmHandle = new NcmpServiceCmHandle() - ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, 'new moduleSetTag: tag-1').build()) + ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, 'Upgrade to ModuleSetTag: tag-1').build()) def dmiServiceName = 'some service name' ncmpServiceCmHandle.cmHandleId = 'upgraded-ch' def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(dmiServiceName, '', '', ncmpServiceCmHandle,'tag-1') @@ -98,7 +96,7 @@ class ModuleSyncServiceSpec extends Specification { def moduleReferences = [ new ModuleReference('module1','1') ] and: 'cache or DMI operations returns some module references for upgraded cm handle' if (populateCache) { - mockModuleSetTagCache.put('tag-1',moduleReferences) + mockModuleSetTagCache.put('tag-1', moduleReferences) } else { mockDmiModelOperations.getModuleReferences(yangModelCmHandle) >> moduleReferences } @@ -127,7 +125,7 @@ class ModuleSyncServiceSpec extends Specification { given: 'a cm handle that is ready but locked for upgrade' def ncmpServiceCmHandle = new NcmpServiceCmHandle() ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder() - .withLockReason(MODULE_UPGRADE, 'new moduleSetTag: targetModuleSetTag').build()) + .withLockReason(MODULE_UPGRADE, 'Upgrade to ModuleSetTag: targetModuleSetTag').build()) ncmpServiceCmHandle.setCmHandleId('cmHandleId-1') def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle('some service name', '', '', ncmpServiceCmHandle, 'targetModuleSetTag') mockCmHandleQueries.cmHandleHasState('cmHandleId-1', CmHandleState.READY) >> true @@ -170,11 +168,6 @@ class ModuleSyncServiceSpec extends Specification { result == unsupportedOperationException } - def 'Extract module set tag'() { - expect: 'the module set tag is extracted correctly' - assert 'targetModuleSetTag' == objectUnderTest.extractModuleSetTag(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, 'new moduleSetTag: targetModuleSetTag').build()) - } - def toModuleReferences(moduleReferenceAsMap) { def moduleReferences = [].withDefault { [:] } moduleReferenceAsMap.forEach(property -> diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasksSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasksSpec.groovy index 0d927551d0..3bdac1855c 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasksSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasksSpec.groovy @@ -21,6 +21,10 @@ package org.onap.cps.ncmp.api.impl.inventory.sync + +import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_SYNC_FAILED +import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE_FAILED + import ch.qos.logback.classic.Level import ch.qos.logback.classic.Logger import ch.qos.logback.classic.spi.ILoggingEvent @@ -31,9 +35,6 @@ import com.hazelcast.map.IMap import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.onap.cps.ncmp.api.impl.events.lcm.LcmEventsCmHandleStateHandler -import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncService -import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncTasks -import org.onap.cps.ncmp.api.impl.inventory.sync.SyncUtils import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle import org.onap.cps.ncmp.api.impl.inventory.CmHandleState import org.onap.cps.ncmp.api.impl.inventory.CompositeState @@ -62,7 +63,7 @@ class ModuleSyncTasksSpec extends Specification { def mockInventoryPersistence = Mock(InventoryPersistence) - def mockSyncUtils = Mock(SyncUtils) + def mockSyncUtils = Mock(ModuleOperationsUtils) def mockModuleSyncService = Mock(ModuleSyncService) @@ -79,8 +80,8 @@ class ModuleSyncTasksSpec extends Specification { def 'Module Sync ADVISED cm handles.'() { given: 'cm handles in an ADVISED state' - def cmHandle1 = advisedCmHandleAsDataNode('cm-handle-1') - def cmHandle2 = advisedCmHandleAsDataNode('cm-handle-2') + def cmHandle1 = cmHandleAsDataNodeByIdAndState('cm-handle-1', CmHandleState.ADVISED) + def cmHandle2 = cmHandleAsDataNodeByIdAndState('cm-handle-2', CmHandleState.ADVISED) and: 'the inventory persistence cm handle returns a ADVISED state for the any handle' mockInventoryPersistence.getCmHandleState(_) >> new CompositeState(cmHandleState: CmHandleState.ADVISED) when: 'module sync poll is executed' @@ -101,7 +102,7 @@ class ModuleSyncTasksSpec extends Specification { def 'Module Sync ADVISED cm handle with failure during sync.'() { given: 'a cm handle in an ADVISED state' - def cmHandle = advisedCmHandleAsDataNode('cm-handle') + def cmHandle = cmHandleAsDataNodeByIdAndState('cm-handle', CmHandleState.ADVISED) and: 'the inventory persistence cm handle returns a ADVISED state for the cm handle' def cmHandleState = new CompositeState(cmHandleState: CmHandleState.ADVISED) 1 * mockInventoryPersistence.getCmHandleState('cm-handle') >> cmHandleState @@ -110,7 +111,7 @@ class ModuleSyncTasksSpec extends Specification { when: 'module sync is executed' objectUnderTest.performModuleSync([cmHandle], batchCount) then: 'update lock reason, details and attempts is invoked' - 1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(cmHandleState, LockReasonCategory.MODULE_SYNC_FAILED, 'some exception') + 1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(cmHandleState, MODULE_SYNC_FAILED, 'some exception') and: 'the state handler is called to update the state to LOCKED' 1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> { args -> assertBatch(args, ['cm-handle'], CmHandleState.LOCKED) @@ -119,6 +120,25 @@ class ModuleSyncTasksSpec extends Specification { assert batchCount.get() == 4 } + def 'Failed cm handle during #scenario.'() { + given: 'a cm handle in LOCKED state' + def cmHandle = cmHandleAsDataNodeByIdAndState('cm-handle', CmHandleState.LOCKED) + and: 'the inventory persistence cm handle returns a LOCKED state with reason for the cm handle' + def expectedCmHandleState = new CompositeState(cmHandleState: cmHandleState, lockReason: CompositeState + .LockReason.builder().lockReasonCategory(lockReasonCategory).details(lockReasonDetails).build()) + 1 * mockInventoryPersistence.getCmHandleState('cm-handle') >> expectedCmHandleState + and: 'module sync service attempts to sync/upgrade the cm handle and throws an exception' + 1 * mockModuleSyncService.syncAndCreateOrUpgradeSchemaSetAndAnchor(*_) >> { throw new Exception('some exception') } + when: 'module sync is executed' + objectUnderTest.performModuleSync([cmHandle], batchCount) + then: 'update lock reason, details and attempts is invoked' + 1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(expectedCmHandleState, expectedLockReasonCategory, 'some exception') + where: + scenario | cmHandleState | lockReasonCategory | lockReasonDetails || expectedLockReasonCategory + 'module upgrade' | CmHandleState.LOCKED | MODULE_UPGRADE_FAILED | 'Upgrade to ModuleSetTag: some-module-set-tag' || MODULE_UPGRADE_FAILED + 'module sync' | CmHandleState.LOCKED | MODULE_SYNC_FAILED | 'some lock details' || MODULE_SYNC_FAILED + } + def 'Reset failed CM Handles #scenario.'() { given: 'cm handles in an locked state' def lockedState = new CompositeStateBuilder().withCmHandleState(CmHandleState.LOCKED) @@ -147,7 +167,7 @@ class ModuleSyncTasksSpec extends Specification { def 'Module Sync ADVISED cm handle without entry in progress map.'() { given: 'cm handles in an ADVISED state' - def cmHandle1 = advisedCmHandleAsDataNode('cm-handle-1') + def cmHandle1 = cmHandleAsDataNodeByIdAndState('cm-handle-1', CmHandleState.ADVISED) and: 'the inventory persistence cm handle returns a ADVISED state for the any handle' mockInventoryPersistence.getCmHandleState(_) >> new CompositeState(cmHandleState: CmHandleState.ADVISED) and: 'entry in progress map for other cm handle' @@ -183,8 +203,8 @@ class ModuleSyncTasksSpec extends Specification { assert loggingEvent == null } - def advisedCmHandleAsDataNode(cmHandleId) { - return new DataNode(anchorName: cmHandleId, leaves: ['id': cmHandleId, 'cm-handle-state': 'ADVISED']) + def cmHandleAsDataNodeByIdAndState(cmHandleId, cmHandleState) { + return new DataNode(anchorName: cmHandleId, leaves: ['id': cmHandleId, 'cm-handle-state': cmHandleState]) } def assertYamgModelCmHandleArgument(args, expectedCmHandleId) { diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy index 7425a52ff1..1752a17a19 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy @@ -22,9 +22,6 @@ package org.onap.cps.ncmp.api.impl.inventory.sync import com.hazelcast.map.IMap -import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncTasks -import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncWatchdog -import org.onap.cps.ncmp.api.impl.inventory.sync.SyncUtils import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle import org.onap.cps.ncmp.api.impl.inventory.sync.executor.AsyncTaskExecutor import java.util.concurrent.ArrayBlockingQueue @@ -33,7 +30,7 @@ import spock.lang.Specification class ModuleSyncWatchdogSpec extends Specification { - def mockSyncUtils = Mock(SyncUtils) + def mockSyncUtils = Mock(ModuleOperationsUtils) def static testQueueCapacity = 50 + 2 * ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtilsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtilsSpec.groovy deleted file mode 100644 index 025d9482d6..0000000000 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtilsSpec.groovy +++ /dev/null @@ -1,198 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation - * Modifications Copyright (C) 2022 Bell Canada - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ncmp.api.impl.inventory.sync - -import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.LOCKED_MISBEHAVING -import static org.onap.cps.ncmp.api.impl.operations.DatastoreType.PASSTHROUGH_OPERATIONAL -import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_SYNC_FAILED -import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE -import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE_FAILED - -import ch.qos.logback.classic.Level -import ch.qos.logback.classic.Logger -import ch.qos.logback.core.read.ListAppender -import org.onap.cps.ncmp.api.impl.inventory.sync.SyncUtils -import org.slf4j.LoggerFactory -import org.springframework.context.annotation.AnnotationConfigApplicationContext -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.ObjectMapper -import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations -import org.onap.cps.ncmp.api.impl.inventory.CmHandleQueries -import org.onap.cps.ncmp.api.impl.inventory.CmHandleState -import org.onap.cps.ncmp.api.impl.inventory.CompositeState -import org.onap.cps.ncmp.api.impl.inventory.CompositeStateBuilder -import org.onap.cps.ncmp.api.impl.inventory.DataStoreSyncState -import org.onap.cps.spi.FetchDescendantsOption -import org.onap.cps.spi.model.DataNode -import org.onap.cps.utils.JsonObjectMapper -import org.springframework.http.HttpStatus -import org.springframework.http.ResponseEntity -import spock.lang.Specification -import java.time.OffsetDateTime -import java.time.format.DateTimeFormatter -import java.util.stream.Collectors - -class SyncUtilsSpec extends Specification{ - - def mockCmHandleQueries = Mock(CmHandleQueries) - - def mockDmiDataOperations = Mock(DmiDataOperations) - - def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper()) - - def objectUnderTest = new SyncUtils(mockCmHandleQueries, mockDmiDataOperations, jsonObjectMapper) - - def static neverUpdatedBefore = '1900-01-01T00:00:00.000+0100' - - def static now = OffsetDateTime.now() - - def static nowAsString = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(now) - - def static dataNode = new DataNode(leaves: ['id': 'cm-handle-123']) - - def applicationContext = new AnnotationConfigApplicationContext() - - def logger = (Logger) LoggerFactory.getLogger(SyncUtils) - def loggingListAppender - - void setup() { - logger.setLevel(Level.DEBUG) - loggingListAppender = new ListAppender() - logger.addAppender(loggingListAppender) - loggingListAppender.start() - applicationContext.refresh() - } - - void cleanup() { - ((Logger) LoggerFactory.getLogger(SyncUtils.class)).detachAndStopAllAppenders() - applicationContext.close() - } - - def 'Get an advised Cm-Handle where ADVISED cm handle #scenario'() { - given: 'the inventory persistence service returns a collection of data nodes' - mockCmHandleQueries.queryCmHandlesByState(CmHandleState.ADVISED) >> dataNodeCollection - when: 'get advised cm handles are fetched' - def yangModelCmHandles = objectUnderTest.getAdvisedCmHandles() - then: 'the returned data node collection is the correct size' - yangModelCmHandles.size() == expectedDataNodeSize - where: 'the following scenarios are used' - scenario | dataNodeCollection || expectedCallsToGetYangModelCmHandle | expectedDataNodeSize - 'exists' | [dataNode] || 1 | 1 - 'does not exist' | [] || 0 | 0 - } - - def 'Update Lock Reason, Details and Attempts where lock reason #scenario'() { - given: 'A locked state' - def compositeState = new CompositeState(lockReason: lockReason) - when: 'update cm handle details and attempts is called' - objectUnderTest.updateLockReasonDetailsAndAttempts(compositeState, MODULE_SYNC_FAILED, 'new error message') - then: 'the composite state lock reason and details are updated' - assert compositeState.lockReason.lockReasonCategory == MODULE_SYNC_FAILED - assert compositeState.lockReason.details == expectedDetails - where: - scenario | lockReason || expectedDetails - 'does not exist' | null || 'Attempt #1 failed: new error message' - 'exists' | CompositeState.LockReason.builder().details("Attempt #2 failed: some error message").build() || 'Attempt #3 failed: new error message' - } - - def 'Get all locked Cm-Handle where Lock Reason is MODULE_SYNC_FAILED cm handle #scenario'() { - given: 'the cps (persistence service) returns a collection of data nodes' - mockCmHandleQueries.queryCmHandleAncestorsByCpsPath( - '//lock-reason[@reason="MODULE_SYNC_FAILED" or @reason="MODULE_UPGRADE"]', - FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) >> [dataNode] - when: 'get locked Misbehaving cm handle is called' - def result = objectUnderTest.getCmHandlesThatFailedModelSyncOrUpgrade() - then: 'the returned cm handle collection is the correct size' - result.size() == 1 - and: 'the correct cm handle is returned' - result[0].id == 'cm-handle-123' - } - - def 'Retry Locked Cm-Handle where the last update time is #scenario'() { - given: 'Last update was #lastUpdateMinutesAgo minutes ago (-1 means never)' - def lastUpdatedTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(now.minusMinutes(lastUpdateMinutesAgo)) - if (lastUpdateMinutesAgo < 0 ) { - lastUpdatedTime = neverUpdatedBefore - } - when: 'checking to see if cm handle is ready for retry' - def result = objectUnderTest.needsModuleSyncRetryOrUpgrade(new CompositeStateBuilder() - .withLockReason(MODULE_SYNC_FAILED, lockDetails) - .withLastUpdatedTime(lastUpdatedTime).build()) - then: 'retry is only attempted when expected' - assert result == retryExpected - and: 'logs contain related information' - def logs = loggingListAppender.list.toString() - assert logs.contains(logReason) - where: 'the following parameters are used' - scenario | lastUpdateMinutesAgo | lockDetails | logReason || retryExpected - 'never attempted before' | -1 | 'fist attempt:' | 'First Attempt:' || true - '1st attempt, last attempt > 2 minute ago' | 3 | 'Attempt #1 failed:' | 'Retry due now' || true - '2nd attempt, last attempt < 4 minutes ago' | 1 | 'Attempt #2 failed:' | 'Time until next attempt is 3 minutes:' || false - '2nd attempt, last attempt > 4 minutes ago' | 5 | 'Attempt #2 failed:' | 'Retry due now' || true - } - - def 'Retry Locked Cm-Handle with other lock reasons (category) #lockReasonCategory'() { - when: 'checking to see if cm handle is ready for retry' - def result = objectUnderTest.needsModuleSyncRetryOrUpgrade(new CompositeStateBuilder() - .withLockReason(lockReasonCategory, 'some details') - .withLastUpdatedTime(nowAsString).build()) - then: 'verify retry attempts' - assert result == retryAttempt - and: 'logs contain related information' - def logs = loggingListAppender.list.toString() - assert logs.contains(logReason) - where: 'the following lock reasons occurred' - scenario | lockReasonCategory || logReason | retryAttempt - 'module upgrade' | MODULE_UPGRADE || 'Locked for module upgrade.' | true - 'module sync failed' | MODULE_SYNC_FAILED || 'First Attempt:' | false - 'lock misbehaving' | LOCKED_MISBEHAVING || 'Locked for other reason' | false - } - - def 'Get a Cm-Handle where #scenario'() { - given: 'the inventory persistence service returns a collection of data nodes' - mockCmHandleQueries.queryCmHandlesByOperationalSyncState(DataStoreSyncState.UNSYNCHRONIZED) >> unSynchronizedDataNodes - mockCmHandleQueries.cmHandleHasState('cm-handle-123', CmHandleState.READY) >> cmHandleHasState - when: 'get advised cm handles are fetched' - def yangModelCollection = objectUnderTest.getUnsynchronizedReadyCmHandles() - then: 'the returned data node collection is the correct size' - yangModelCollection.size() == expectedDataNodeSize - and: 'the result contains the correct data' - yangModelCollection.stream().map(yangModel -> yangModel.id).collect(Collectors.toSet()) == expectedYangModelCollectionIds - where: 'the following scenarios are used' - scenario | unSynchronizedDataNodes | cmHandleHasState || expectedDataNodeSize | expectedYangModelCollectionIds - 'a Cm-Handle unsynchronized and ready' | [dataNode] | true || 1 | ['cm-handle-123'] as Set - 'a Cm-Handle unsynchronized but not ready' | [dataNode] | false || 0 | [] as Set - 'all Cm-Handle synchronized' | [] | false || 0 | [] as Set - } - - def 'Get resource data through DMI Operations #scenario'() { - given: 'the inventory persistence service returns a collection of data nodes' - def jsonString = '{"stores:bookstore":{"categories":[{"code":"01"}]}}' - JsonNode jsonNode = jsonObjectMapper.convertToJsonNode(jsonString); - def responseEntity = new ResponseEntity<>(jsonNode, HttpStatus.OK) - mockDmiDataOperations.getResourceDataFromDmi(PASSTHROUGH_OPERATIONAL.datastoreName, 'cm-handle-123', _) >> responseEntity - when: 'get resource data is called' - def result = objectUnderTest.getResourceData('cm-handle-123') - then: 'the returned data is correct' - result == jsonString - } -} -- cgit 1.2.3-korg