diff options
Diffstat (limited to 'integration-test/src')
16 files changed, 203 insertions, 166 deletions
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy index b9b1c162e7..453dbca2bd 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy @@ -22,6 +22,8 @@ package org.onap.cps.integration.base import com.hazelcast.map.IMap +import java.time.OffsetDateTime +import java.util.concurrent.BlockingQueue import okhttp3.mockwebserver.MockWebServer import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService @@ -44,6 +46,7 @@ import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncService import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.onap.cps.ncmp.impl.utils.AlternateIdMatcher import org.onap.cps.ri.repository.DataspaceRepository +import org.onap.cps.ri.repository.SchemaSetRepository import org.onap.cps.ri.utils.SessionManager import org.onap.cps.utils.JsonObjectMapper import org.springframework.beans.factory.annotation.Autowired @@ -59,10 +62,6 @@ import org.springframework.test.web.servlet.MockMvc import org.testcontainers.spock.Testcontainers import spock.lang.Shared import spock.lang.Specification -import spock.util.concurrent.PollingConditions - -import java.time.OffsetDateTime -import java.util.concurrent.BlockingQueue @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, classes = [CpsDataspaceService]) @Testcontainers @@ -102,6 +101,12 @@ abstract class CpsIntegrationSpecBase extends Specification { SessionManager sessionManager @Autowired + DataspaceRepository dataspaceRepository + + @Autowired + SchemaSetRepository schemaSetRepository + + @Autowired ParameterizedCmHandleQueryService networkCmProxyCmHandleQueryService @Autowired @@ -153,16 +158,16 @@ abstract class CpsIntegrationSpecBase extends Specification { static NO_ALTERNATE_ID = '' static GENERAL_TEST_DATASPACE = 'generalTestDataspace' static BOOKSTORE_SCHEMA_SET = 'bookstoreSchemaSet' - static MODULE_SYNC_WAIT_TIME_IN_SECONDS = 2 static initialized = false def now = OffsetDateTime.now() + enum ModuleNameStrategy { UNIQUE, OVERLAPPING } + def setup() { if (!initialized) { cpsDataspaceService.createDataspace(GENERAL_TEST_DATASPACE) createStandardBookStoreSchemaSet(GENERAL_TEST_DATASPACE) - cpsAnchorService.createAnchor(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET, 'owner-of-bookstore-schema-set-do-not-delete') initialized = true } mockDmiServer1.setDispatcher(dmiDispatcher1) @@ -182,7 +187,7 @@ abstract class CpsIntegrationSpecBase extends Specification { mockDmiServer1.shutdown() mockDmiServer2.shutdown() mockPolicyServer.shutdown() - cpsModuleService.deleteAllUnusedYangModuleData() + cpsModuleService.deleteAllUnusedYangModuleData('NFP-Operational') } def static readResourceDataFile(filename) { @@ -203,18 +208,18 @@ abstract class CpsIntegrationSpecBase extends Specification { return nodeCount } - def getBookstoreYangResourcesNameToContentMap() { + def getBookstoreyangResourceContentPerName() { def bookstoreModelFileContent = readResourceDataFile('bookstore/bookstore.yang') def bookstoreTypesFileContent = readResourceDataFile('bookstore/bookstore-types.yang') return [bookstore: bookstoreModelFileContent, bookstoreTypes: bookstoreTypesFileContent] } def createStandardBookStoreSchemaSet(targetDataspace) { - cpsModuleService.createSchemaSet(targetDataspace, BOOKSTORE_SCHEMA_SET, getBookstoreYangResourcesNameToContentMap()) + cpsModuleService.createSchemaSet(targetDataspace, BOOKSTORE_SCHEMA_SET, getBookstoreyangResourceContentPerName()) } def createStandardBookStoreSchemaSet(targetDataspace, targetSchemaSet) { - cpsModuleService.createSchemaSet(targetDataspace, targetSchemaSet, getBookstoreYangResourcesNameToContentMap()) + cpsModuleService.createSchemaSet(targetDataspace, targetSchemaSet, getBookstoreyangResourceContentPerName()) } def dataspaceExists(dataspaceName) { @@ -254,9 +259,7 @@ abstract class CpsIntegrationSpecBase extends Specification { def registerCmHandle(dmiPlugin, cmHandleId, moduleSetTag, alternateId) { registerCmHandleWithoutWaitForReady(dmiPlugin, cmHandleId, moduleSetTag, alternateId) moduleSyncWatchdog.moduleSyncAdvisedCmHandles() - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - CmHandleState.READY == networkCmProxyInventoryFacade.getCmHandleCompositeState(cmHandleId).cmHandleState - }) + CmHandleState.READY == networkCmProxyInventoryFacade.getCmHandleCompositeState(cmHandleId).cmHandleState } def registerCmHandleWithoutWaitForReady(dmiPlugin, cmHandleId, moduleSetTag, alternateId) { @@ -265,9 +268,14 @@ abstract class CpsIntegrationSpecBase extends Specification { } def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset) { + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset, ModuleNameStrategy.UNIQUE) + } + + def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset, ModuleNameStrategy moduleNameStrategy ) { def cmHandles = [] def id = offset - def moduleReferences = (1..200).collect { "${moduleSetTag}Module${it}" } + def modulePrefix = moduleNameStrategy.OVERLAPPING.equals(moduleNameStrategy) ? 'same' : moduleSetTag + def moduleReferences = (1..200).collect { "${modulePrefix}Module${it}" } (1..numberOfCmHandles).each { def ncmpServiceCmHandle = new NcmpServiceCmHandle(cmHandleId: "ch-${id}", moduleSetTag: moduleSetTag, alternateId: NO_ALTERNATE_ID) cmHandles.add(ncmpServiceCmHandle) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy index c790521627..11c0ff2743 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy @@ -29,6 +29,7 @@ import org.springframework.http.HttpStatus import org.springframework.http.MediaType import java.util.regex.Matcher +import java.util.stream.Collectors import static org.onap.cps.integration.base.CpsIntegrationSpecBase.readResourceDataFile @@ -81,7 +82,7 @@ class DmiDispatcher extends Dispatcher { // get module resources for a CM-handle case ~'^/dmi/v1/ch/(.*)/moduleResources$': def cmHandleId = Matcher.lastMatcher[0][1] - return getModuleResourcesResponse(cmHandleId) + return getModuleResourcesResponse(request, cmHandleId) // pass-through data operation for a CM-handle case ~'^/dmi/v1/ch/(.*)/data/ds/(.*)$': @@ -124,8 +125,13 @@ class DmiDispatcher extends Dispatcher { return mockResponseWithBody(HttpStatus.OK, moduleReferences) } - def getModuleResourcesResponse(cmHandleId) { - def moduleResources = '[' + getModuleNamesForCmHandle(cmHandleId).collect { + def getModuleResourcesResponse(request, cmHandleId) { + def moduleResourcesRequest = jsonSlurper.parseText(request.getBody().readUtf8()) + def requestedModuleNames = moduleResourcesRequest.get('data').get('modules').collect{it.get('name')} + def candidateModuleNames = getModuleNamesForCmHandle(cmHandleId) + def moduleNames = candidateModuleNames.stream().filter(candidate -> requestedModuleNames.contains(candidate)).toList() + + def moduleResources = '[' + moduleNames.collect { MODULE_RESOURCES_RESPONSE_TEMPLATE.replaceAll("<MODULE_NAME>", it) }.join(',') + ']' return mockResponseWithBody(HttpStatus.OK, moduleResources) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy index 2bd5a4a1be..ca321119ea 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy @@ -59,17 +59,17 @@ class AnchorServiceIntegrationSpec extends FunctionalSpecBase { and: '1 anchor with "other" schema set is created' createStandardBookStoreSchemaSet(GENERAL_TEST_DATASPACE, 'otherSchemaSet') objectUnderTest.createAnchor(GENERAL_TEST_DATASPACE, 'otherSchemaSet', 'anchor3') - then: 'there are 4 anchors in the general test database' - assert objectUnderTest.getAnchors(GENERAL_TEST_DATASPACE).size() == 4 - and: 'there are 3 anchors associated with bookstore schema set' - assert objectUnderTest.getAnchorsBySchemaSetName(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET).size() == 3 + then: 'there are 3 anchors in the general test database' + assert objectUnderTest.getAnchors(GENERAL_TEST_DATASPACE).size() == 3 + and: 'there are 2 anchors associated with bookstore schema set' + assert objectUnderTest.getAnchorsBySchemaSetName(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET).size() == 2 and: 'there is 1 anchor associated with other schema set' assert objectUnderTest.getAnchorsBySchemaSetName(GENERAL_TEST_DATASPACE, 'otherSchemaSet').size() == 1 } def 'Querying anchor(name)s (depends on previous test!).'() { - expect: 'there are now 4 anchors using the "stores" module (both schema sets use the same modules) ' - assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'bookstore-types']).size() == 4 + expect: 'there are now 3 anchors using the "stores" module (both schema sets use the same modules)' + assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'bookstore-types']).size() == 3 and: 'there are no anchors using both "stores" and a "unused-model"' assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'unused-model']).size() == 0 } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy index 178b0227ca..47a332adc9 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -32,8 +32,6 @@ class DataspaceServiceIntegrationSpec extends FunctionalSpecBase { def setup() { objectUnderTest = cpsDataspaceService } - def cleanup() { cpsModuleService.deleteAllUnusedYangModuleData() } - def 'Dataspace CRUD operations.'() { when: 'a dataspace is created' objectUnderTest.createDataspace('newDataspace') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy index d8010875c1..49201e8f22 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy @@ -53,25 +53,23 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { } """ - def newYangResourcesNameToContentMap = [:] - def moduleReferences = [] + def yangResourceContentPerName = [:] + def allModuleReferences = [] def noNewModules = [:] def bookstoreModelFileContent = readResourceDataFile('bookstore/bookstore.yang') def bookstoreTypesFileContent = readResourceDataFile('bookstore/bookstore-types.yang') def setup() { objectUnderTest = cpsModuleService } - def cleanup() { objectUnderTest.deleteAllUnusedYangModuleData() } - /* C R E A T E S C H E M A S E T U S E - C A S E S */ def 'Create new schema set from yang resources with #scenario'() { given: 'a new schema set with #numberOfModules modules' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(numberOfNewModules) + populateYangResourceContentPerNameAndAllModuleReferences(numberOfNewModules) when: 'the new schema set is created' - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', newYangResourcesNameToContentMap) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', yangResourceContentPerName) then: 'the number of module references has increased by #numberOfNewModules' def yangResourceModuleReferences = objectUnderTest.getYangResourceModuleReferences(FUNCTIONAL_TEST_DATASPACE_1) originalNumberOfModuleReferences + numberOfNewModules == yangResourceModuleReferences.size() @@ -95,39 +93,43 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Create new schema set from modules with #scenario'() { given: 'a new schema set with #numberOfNewModules modules' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(numberOfNewModules) + populateYangResourceContentPerNameAndAllModuleReferences(numberOfNewModules) and: 'add existing module references (optional)' - moduleReferences.addAll(existingModuleReferences) + allModuleReferences.addAll(existingModuleReferences) when: 'the new schema set is created' def schemaSetName = "NewSchemaWith${numberOfNewModules}Modules" - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, schemaSetName, newYangResourcesNameToContentMap, moduleReferences) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, schemaSetName, yangResourceContentPerName, allModuleReferences) and: 'associated with a new anchor' cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, schemaSetName, 'newAnchor') then: 'the new anchor has the correct number of modules' def yangResourceModuleReferences = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'newAnchor') assert expectedNumberOfModulesForAnchor == yangResourceModuleReferences.size() + and: 'the schema set has the correct number of modules too' + def dataspaceEntity = dataspaceRepository.getByName(FUNCTIONAL_TEST_DATASPACE_1) + def schemaSetEntity = schemaSetRepository.getByDataspaceAndName(dataspaceEntity, schemaSetName) + assert expectedNumberOfModulesForAnchor == schemaSetEntity.yangResources.size() cleanup: objectUnderTest.deleteSchemaSetsWithCascade(FUNCTIONAL_TEST_DATASPACE_1, [ schemaSetName.toString() ]) where: 'the following module references are provided' scenario | numberOfNewModules | existingModuleReferences || expectedNumberOfModulesForAnchor 'empty schema set' | 0 | [ ] || 0 - 'one existing module' | 0 | [bookStoreModuleReference ] || 1 + 'one existing module' | 0 | [ bookStoreModuleReference ] || 1 'two new modules' | 2 | [ ] || 2 - 'two new modules, one existing' | 2 | [bookStoreModuleReference ] || 3 + 'two new modules, one existing' | 2 | [ bookStoreModuleReference ] || 3 'over max batch size #modules' | 101 | [ ] || 101 'two valid, one invalid module' | 2 | [ new ModuleReference('NOT EXIST','IRRELEVANT') ] || 2 } def 'Duplicate schema content.'() { given: 'a map of yang resources' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(1) + populateYangResourceContentPerNameAndAllModuleReferences(1) when: 'a new schema set is created' - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema1', newYangResourcesNameToContentMap) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema1', yangResourceContentPerName) then: 'the dataspace has one new module (reference)' def numberOfModuleReferencesAfterFirstSchemaSetHasBeenAdded = objectUnderTest.getYangResourceModuleReferences(FUNCTIONAL_TEST_DATASPACE_1).size() assert numberOfModuleReferencesAfterFirstSchemaSetHasBeenAdded == originalNumberOfModuleReferences + 1 when: 'a second new schema set is created' - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema2', newYangResourcesNameToContentMap) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema2', yangResourceContentPerName) then: 'the dataspace has no additional module (reference)' assert numberOfModuleReferencesAfterFirstSchemaSetHasBeenAdded == objectUnderTest.getYangResourceModuleReferences(FUNCTIONAL_TEST_DATASPACE_1).size() cleanup: @@ -136,8 +138,8 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Attempt to create schema set, error scenario: #scenario.'() { when: 'attempt to store schema set #schemaSetName in dataspace #dataspaceName' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(0) - objectUnderTest.createSchemaSet(dataspaceName, schemaSetName, newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(0) + objectUnderTest.createSchemaSet(dataspaceName, schemaSetName, yangResourceContentPerName) then: 'an #expectedException is thrown' thrown(expectedException) where: 'the following data is used' @@ -148,7 +150,7 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Attempt to create duplicate schema set from modules.'() { when: 'attempt to store duplicate schema set from modules' - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, BOOKSTORE_SCHEMA_SET, newYangResourcesNameToContentMap, []) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, BOOKSTORE_SCHEMA_SET, yangResourceContentPerName, []) then: 'an Already Defined Exception is thrown' thrown(AlreadyDefinedException) } @@ -194,12 +196,12 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Identifying new module references with #scenario'() { when: 'identifyNewModuleReferences is called' - def result = objectUnderTest.identifyNewModuleReferences(moduleReferences) + def result = objectUnderTest.identifyNewModuleReferences(allModuleReferences) then: 'the correct module references are returned' assert result.size() == expectedResult.size() assert result.containsAll(expectedResult) where: 'the following data is used' - scenario | moduleReferences || expectedResult + scenario | allModuleReferences || expectedResult 'just new module references' | [new ModuleReference('new1', 'r1'), new ModuleReference('new2', 'r1')] || [new ModuleReference('new1', 'r1'), new ModuleReference('new2', 'r1')] 'one new module,one existing reference' | [new ModuleReference('new1', 'r1'), bookStoreModuleReference] || [new ModuleReference('new1', 'r1')] 'no new module references' | [bookStoreModuleReference] || [] @@ -218,8 +220,8 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Retrieve all schema sets.'() { given: 'an extra schema set is stored' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(1) - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema1', newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(1) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema1', yangResourceContentPerName) when: 'all schema sets are retrieved' def result = objectUnderTest.getSchemaSets(FUNCTIONAL_TEST_DATASPACE_1) then: 'the result contains all expected schema sets' @@ -235,8 +237,8 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Delete schema sets with(out) cascade.'() { given: 'a schema set' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(1) - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(1) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', yangResourceContentPerName) and: 'optionally create anchor for the schema set' if (associateWithAnchor) { cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', 'newAnchor') @@ -263,11 +265,11 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Delete schema sets with shared resources.'() { given: 'a new schema set' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(1) - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet1', newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(1) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet1', yangResourceContentPerName) and: 'another schema set which shares one yang resource (module)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(2) - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet2', newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(2) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet2', yangResourceContentPerName) when: 'all schema sets are retrieved' def moduleRevisions = objectUnderTest.getYangResourceModuleReferences(FUNCTIONAL_TEST_DATASPACE_1).revision then: 'both modules (revisions) are present' @@ -301,23 +303,23 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { U P G R A D E */ - def 'Upgrade schema set (with existing and new modules, no matching module set tag in NCMP)'() { + def 'Upgrade schema set [with existing and new modules, no matching module set tag in NCMP]'() { given: 'an anchor and schema set with 2 modules (to be upgraded)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences('original', 2) - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', newYangResourcesNameToContentMap, []) + populateYangResourceContentPerNameAndAllModuleReferences('original', 2) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', yangResourceContentPerName, allModuleReferences) cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', 'targetAnchor') def yangResourceModuleReferencesBeforeUpgrade = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'targetAnchor') assert yangResourceModuleReferencesBeforeUpgrade.size() == 2 assert yangResourceModuleReferencesBeforeUpgrade.containsAll([new ModuleReference('original_0','2000-01-01'),new ModuleReference('original_1','2001-01-01')]) and: 'two new 2 modules (from node)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences('new', 2) + populateYangResourceContentPerNameAndAllModuleReferences('new', 2) def newModuleReferences = [new ModuleReference('new_0','2000-01-01'),new ModuleReference('new_1','2001-01-01')] and: 'a list of all module references (normally retrieved from node)' - def allModuleReferences = [] - allModuleReferences.add(bookStoreModuleReference) - allModuleReferences.addAll(newModuleReferences) + def allOtherModuleReferences = [] + allOtherModuleReferences.add(bookStoreModuleReference) + allOtherModuleReferences.addAll(newModuleReferences) when: 'the schema set is upgraded' - objectUnderTest.upgradeSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', newYangResourcesNameToContentMap, allModuleReferences) + objectUnderTest.upgradeSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', yangResourceContentPerName, allOtherModuleReferences) then: 'the new anchor has the correct new and existing modules' def yangResourceModuleReferencesAfterUpgrade = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'targetAnchor') assert yangResourceModuleReferencesAfterUpgrade.size() == 3 @@ -327,18 +329,19 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { objectUnderTest.deleteSchemaSetsWithCascade(FUNCTIONAL_TEST_DATASPACE_1, ['targetSchema']) } - def 'Upgrade existing schema set from another anchor (used in NCMP for matching module set tag)'() { + def 'Upgrade existing schema set from another anchor [used in NCMP for matching module set tag]'() { given: 'an anchor and schema set with 1 module (target)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences('target', 1) - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', newYangResourcesNameToContentMap, []) + populateYangResourceContentPerNameAndAllModuleReferences('target', 1) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', yangResourceContentPerName, allModuleReferences) cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', 'targetAnchor') def moduleReferencesBeforeUpgrade = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'targetAnchor') assert moduleReferencesBeforeUpgrade.size() == 1 and: 'another anchor and schema set with 2 other modules (source for upgrade)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences('source', 2) - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'sourceSchema', newYangResourcesNameToContentMap, []) + populateYangResourceContentPerNameAndAllModuleReferences('source', 2) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'sourceSchema', yangResourceContentPerName, allModuleReferences) cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, 'sourceSchema', 'sourceAnchor') - assert objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'sourceAnchor').size() == 2 + def yangResourcesModuleReferences = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'sourceAnchor') + assert yangResourcesModuleReferences.size() == 2 when: 'the target schema is upgraded using the module references from the source anchor' def moduleReferencesFromSourceAnchor = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'sourceAnchor') objectUnderTest.upgradeSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', noNewModules, moduleReferencesFromSourceAnchor) @@ -356,17 +359,19 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { H E L P E R M E T H O D S */ - def populateNewYangResourcesNameToContentMapAndAllModuleReferences(numberOfModules) { - populateNewYangResourcesNameToContentMapAndAllModuleReferences('name', numberOfModules) + def populateYangResourceContentPerNameAndAllModuleReferences(numberOfModules) { + populateYangResourceContentPerNameAndAllModuleReferences('name', numberOfModules) } - def populateNewYangResourcesNameToContentMapAndAllModuleReferences(namePrefix, numberOfModules) { + def populateYangResourceContentPerNameAndAllModuleReferences(namePrefix, numberOfModules) { + yangResourceContentPerName.clear() + allModuleReferences.clear() numberOfModules.times { def uniqueName = namePrefix + '_' + it def uniqueRevision = String.valueOf(2000 + it) + '-01-01' - moduleReferences.add(new ModuleReference(uniqueName, uniqueRevision)) + allModuleReferences.add(new ModuleReference(uniqueName, uniqueRevision)) def uniqueContent = NEW_RESOURCE_CONTENT.replace(NEW_RESOURCE_REVISION, uniqueRevision).replace('module test_module', 'module '+uniqueName) - newYangResourcesNameToContentMap.put(uniqueRevision, uniqueContent) + yangResourceContentPerName.put(uniqueName, uniqueContent) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy index e4d75aa378..42fb964d52 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * Modifications Copyright (C) 2023-2025 TechMahindra Ltd * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); @@ -27,7 +27,6 @@ import org.onap.cps.integration.base.FunctionalSpecBase import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.parameters.PaginationOption import org.onap.cps.api.exceptions.CpsPathException -import spock.lang.Ignore import static org.onap.cps.api.parameters.FetchDescendantsOption.DIRECT_CHILDREN_ONLY import static org.onap.cps.api.parameters.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS @@ -57,7 +56,6 @@ class QueryServiceIntegrationSpec extends FunctionalSpecBase { 'the AND is used where result does not exist' | '//books[@lang="English" and @price=1000]' || 0 | [] } - @Ignore // TODO will be implemented in CPS-2416 def 'Query data leaf using CPS path for #scenario.'() { when: 'query data leaf for bookstore container' def result = objectUnderTest.queryDataLeaf(FUNCTIONAL_TEST_DATASPACE_1, BOOKSTORE_ANCHOR_1, cpsPath, Object.class) @@ -70,7 +68,6 @@ class QueryServiceIntegrationSpec extends FunctionalSpecBase { 'non-existing path' | '/non-existing/@title' || 0 } - @Ignore def 'Query data leaf with type #leafType using CPS path.'() { given: 'a cps path query for two books, returning only #leafName' def cpsPath = '//books[@title="Matilda" or @title="Good Omens"]/@' + leafName @@ -85,7 +82,6 @@ class QueryServiceIntegrationSpec extends FunctionalSpecBase { 'editions' | List.class || [[1988, 2000], [2006]] } - @Ignore def 'Query data leaf using CPS path with ancestor axis.'() { given: 'a cps path query that will return the names of the categories of two books' def cpsPath = '//books[@title="Matilda" or @title="Good Omens"]/ancestor::categories/@name' @@ -461,4 +457,17 @@ class QueryServiceIntegrationSpec extends FunctionalSpecBase { and: 'the queried nodes have expected bookstore names' assert result.anchorName.toSet() == [BOOKSTORE_ANCHOR_1, BOOKSTORE_ANCHOR_2].toSet() } + + def 'Query with a limit of #limit.' () { + when: + def result = objectUnderTest.queryDataNodes(FUNCTIONAL_TEST_DATASPACE_1, BOOKSTORE_ANCHOR_1, '/bookstore/categories', OMIT_DESCENDANTS, limit) + then: 'the expected number of nodes is returned' + assert countDataNodesInTree(result) == expectedNumberOfResults + where: 'the following parameters are used' + limit || expectedNumberOfResults + 1 || 1 + 2 || 2 + 0 || 5 + -1 || 5 + } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy index 6f063fb222..41798cb1f1 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package org.onap.cps.integration.functional.ncmp +import java.time.Duration import org.apache.kafka.clients.consumer.KafkaConsumer import org.apache.kafka.common.serialization.StringDeserializer import org.onap.cps.integration.KafkaTestContainer @@ -32,9 +33,6 @@ import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle import org.onap.cps.ncmp.events.lcm.v1.LcmEvent import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.api.inventory.models.LockReasonCategory -import spock.util.concurrent.PollingConditions - -import java.time.Duration class CmHandleCreateSpec extends CpsIntegrationSpecBase { @@ -73,9 +71,7 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'CM-handle goes to READY state after module sync' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(uniqueId).cmHandleState - }) + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(uniqueId).cmHandleState and: 'the CM-handle has expected modules' assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(uniqueId).moduleName.sort() @@ -113,11 +109,9 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'CM-handle goes to LOCKED state with reason MODULE_SYNC_FAILED' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState('ch-1') - assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED - assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_SYNC_FAILED - }) + def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState('ch-1') + assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED + assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_SYNC_FAILED and: 'CM-handle has no modules' assert objectUnderTest.getYangResourcesModuleReferences('ch-1').empty @@ -141,9 +135,7 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'the CM-handle goes to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState('ch-3').cmHandleState - }) + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState('ch-3').cmHandleState and: 'the CM-handle has expected moduleSetTag' assert objectUnderTest.getNcmpServiceCmHandle('ch-3').moduleSetTag == 'B' @@ -178,9 +170,9 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { assert dmiPluginRegistrationResponse.createdCmHandles.sort { it.cmHandle } == [ CmHandleRegistrationResponse.createSuccessResponse('ch-3'), CmHandleRegistrationResponse.createSuccessResponse('ch-4'), - CmHandleRegistrationResponse.createFailureResponse('ch-5', NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED), + CmHandleRegistrationResponse.createFailureResponse('ch-5', NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST), CmHandleRegistrationResponse.createSuccessResponse('ch-6'), - CmHandleRegistrationResponse.createFailureResponse('ch-7', NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED), + CmHandleRegistrationResponse.createFailureResponse('ch-7', NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST), ] cleanup: 'deregister CM handles' @@ -200,9 +192,7 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'CM-handles go to LOCKED state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.LOCKED - }) + assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.LOCKED when: 'DMI is available for retry' dmiDispatcher1.moduleNamesPerCmHandleId = ['ch-1': ['M1', 'M2'], 'ch-2': ['M1', 'M2']] @@ -212,11 +202,9 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } then: 'Both CM-handles go to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - ['ch-1', 'ch-2'].each { cmHandleId -> - assert objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState == CmHandleState.READY - } - }) + ['ch-1', 'ch-2'].each { cmHandleId -> + assert objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState == CmHandleState.READY + } and: 'Both CM-handles have expected modules' ['ch-1', 'ch-2'].each { cmHandleId -> diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy index f2593ce587..22bbaa81df 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -76,8 +76,8 @@ class CmHandleUpdateSpec extends CpsIntegrationSpecBase { def dmiPluginRegistrationResponse = objectUnderTest.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: DMI1_URL, updatedCmHandles: [cmHandleToUpdate])) - then: 'registration gives failure response, due to alternate ID being already associated' - assert dmiPluginRegistrationResponse.updatedCmHandles == [CmHandleRegistrationResponse.createFailureResponse('ch-1', NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED)] + then: 'registration gives failure response, due to cm-handle already existing' + assert dmiPluginRegistrationResponse.updatedCmHandles == [CmHandleRegistrationResponse.createFailureResponse('ch-1', NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST)] and: 'the CM-handle still has the old alternate ID' assert objectUnderTest.getNcmpServiceCmHandle('ch-1').alternateId == 'original' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy index 28714fd123..43540a9675 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy @@ -27,7 +27,6 @@ import org.onap.cps.ncmp.api.inventory.models.DmiPluginRegistration import org.onap.cps.ncmp.api.inventory.models.LockReasonCategory import org.onap.cps.ncmp.api.inventory.models.UpgradedCmHandles import org.onap.cps.ncmp.impl.NetworkCmProxyInventoryFacadeImpl -import spock.util.concurrent.PollingConditions class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { @@ -67,9 +66,7 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } then: 'CM-handle goes to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState - }) + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState and: 'the CM-handle has expected moduleSetTag' assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag @@ -79,7 +76,6 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { cleanup: 'deregister CM-handle and remove all associated module resources' deregisterCmHandle(DMI1_URL, cmHandleId) - cpsModuleService.deleteAllUnusedYangModuleData() where: 'following module set tags are used' initialModuleSetTag | updatedModuleSetTag @@ -112,9 +108,7 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } and: 'CM-handle goes to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState - }) + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState and: 'the CM-handle has expected moduleSetTag' assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag @@ -171,11 +165,9 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } then: 'CM-handle goes to LOCKED state with reason MODULE_UPGRADE_FAILED' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId) - assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED - assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE_FAILED - }) + def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId) + assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED + assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE_FAILED and: 'the CM-handle has same moduleSetTag as before' assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == 'oldTag' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy index 43db9b208e..8bbe0ca194 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy @@ -20,17 +20,18 @@ package org.onap.cps.integration.functional.ncmp +import com.hazelcast.map.IMap import io.micrometer.core.instrument.MeterRegistry -import spock.lang.Ignore - -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.springframework.beans.factory.annotation.Autowired import org.springframework.util.StopWatch +import spock.lang.Ignore import spock.util.concurrent.PollingConditions +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { ModuleSyncWatchdog objectUnderTest @@ -38,11 +39,15 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { @Autowired MeterRegistry meterRegistry + @Autowired + IMap<String, Integer> cmHandlesByState + def executorService = Executors.newFixedThreadPool(2) def PARALLEL_SYNC_SAMPLE_SIZE = 100 def setup() { objectUnderTest = moduleSyncWatchdog + clearCmHandleStateGauge() } def cleanup() { @@ -64,18 +69,16 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) } - @Ignore - /** this test has intermittent failures, due to timeouts. + /** this test has intermittent failures, due to race conditions * Ignored but left here as it might be valuable to further optimization investigations. **/ - + @Ignore def 'CPS-2478 Highlight (and improve) module sync inefficiencies.'() { given: 'register 250 cm handles with module set tag cps-2478-A' def numberOfTags = 2 def cmHandlesPerTag = 250 def totalCmHandles = numberOfTags * cmHandlesPerTag def offset = 1 - def minimumBatches = totalCmHandles / 100 registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-A', cmHandlesPerTag, offset) and: 'register anther 250 cm handles with module set tag cps-2478-B' offset += cmHandlesPerTag @@ -85,23 +88,21 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { when: 'sync all advised cm handles' objectUnderTest.moduleSyncAdvisedCmHandles() Thread.sleep(100) - then: 'retry until both schema sets are stored in db (1 schema set for each module set tag)' - def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.store').timer() - new PollingConditions().within(10, () -> { - objectUnderTest.moduleSyncAdvisedCmHandles() - Thread.sleep(100) - assert dbSchemaSetStorageTimer.count() == 2 - }) - then: 'wait till at least 5 batches of state updates are done (often more because of retries of locked cm handles)' - def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer() + then: 'Keep processing until there are no more LOCKED or ADVISED cm handles' new PollingConditions().within(10, () -> { - assert dbStateUpdateTimer.count() >= minimumBatches + def advised = cmHandlesByState.get('advisedCmHandlesCount') + def locked = cmHandlesByState.get('lockedCmHandlesCount') + if ( locked > 0 | advised > 0 ) { + println "CPS-2576 Need to retry ${locked} LOCKED / ${advised} ADVISED cm Handles" + objectUnderTest.moduleSyncAdvisedCmHandles() + Thread.sleep(100) + } + assert cmHandlesByState.get('lockedCmHandlesCount') + cmHandlesByState.get('advisedCmHandlesCount') == 0 }) - and: 'one call to DMI per module set tag to get module references (may be more due to parallel processing of batches)' - def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer() - assert dmiModuleRetrievalTimer.count() >= numberOfTags && dmiModuleRetrievalTimer.count() <= minimumBatches - and: 'log the relevant instrumentation' + def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer() + def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.createFromNewAndExistingModules').timer() + def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer() logInstrumentation(dmiModuleRetrievalTimer, 'get modules from DMI ') logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets ') logInstrumentation(dbStateUpdateTimer, 'batch state updates ') @@ -110,7 +111,6 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { def stopWatch = new StopWatch() stopWatch.start() deregisterSequenceOfCmHandles(DMI1_URL, totalCmHandles, 1) - cpsModuleService.deleteAllUnusedYangModuleData() stopWatch.stop() println "*** CPS-2478, Deletion of $totalCmHandles cm handles took ${stopWatch.getTotalTimeMillis()} milliseconds" } @@ -131,6 +131,28 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) } + /** this test has intermittent failures, due to race conditions + * Ignored but left here as it might be valuable to further optimization investigations. + **/ + @Ignore + def 'Schema sets with overlapping modules processed at the same time (DB constraint violation).'() { + given: 'register one batch (100) cm handles of tag A (with overlapping module names)' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'tagA', 100, 1, ModuleNameStrategy.OVERLAPPING) + and: 'register another batch cm handles of tag B (with overlapping module names)' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'tagB', 100, 101, ModuleNameStrategy.OVERLAPPING) + and: 'populate the work queue with both batches' + objectUnderTest.populateWorkQueueIfNeeded() + when: 'advised cm handles are processed on 2 threads (exactly one batch for each)' + objectUnderTest.moduleSyncAdvisedCmHandles() + executorService.execute(moduleSyncAdvisedCmHandles) + then: 'all cm handles have been processed' + assert getNumberOfProcessedCmHandles() == 200 + then: 'at least 1 cm handle is in state LOCKED' + assert cmHandlesByState.get('lockedCmHandlesCount') >= 1 + cleanup: 'remove all test cm handles' + deregisterSequenceOfCmHandles(DMI1_URL, 200, 1) + } + def 'Populate module sync work queue on two parallel threads with a slight difference in start time.'() { // This test proved that the issue in CPS-2403 did not arise if the the queue was populated and given time to be distributed given: 'the queue is empty at the start' @@ -169,4 +191,21 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { } } + def moduleSyncAdvisedCmHandles = () -> { + try { + objectUnderTest.moduleSyncAdvisedCmHandles() + } catch (InterruptedException e) { + e.printStackTrace() + } + } + + def clearCmHandleStateGauge() { + cmHandlesByState.keySet().each { cmHandlesByState.put(it, 0)} + } + + def getNumberOfProcessedCmHandles() { + return cmHandlesByState.get('readyCmHandlesCount') + cmHandlesByState.get('lockedCmHandlesCount') + } + + } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy index 7ce3cf5e17..77349fe0a5 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -29,7 +29,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.springframework.http.MediaType -import spock.util.concurrent.PollingConditions class RestApiSpec extends CpsIntegrationSpecBase { @@ -47,13 +46,11 @@ class RestApiSpec extends CpsIntegrationSpecBase { and: 'the module sync watchdog is triggered' moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'CM-handles go to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - (1..3).each { - mvc.perform(get('/ncmp/v1/ch/ch-'+it)) - .andExpect(status().isOk()) - .andExpect(jsonPath('$.state.cmHandleState').value('READY')) - } - }) + (1..3).each { + mvc.perform(get('/ncmp/v1/ch/ch-'+it)) + .andExpect(status().isOk()) + .andExpect(jsonPath('$.state.cmHandleState').value('READY')) + } } def 'Search for CM Handles by module using REST API.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy index a37bb6ad4d..d8553419ce 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy @@ -47,15 +47,15 @@ class CpsModuleServicePerfTest extends CpsPerfTestBase { def 'Store new schema set with many modules'() { when: 'a new schema set with 200 modules is stored' - def newYangResourcesNameToContentMap = [:] + def newYangResourceContentPerName = [:] (1..200).each { def year = 2000 + it def resourceName = "module${it}".toString() def moduleName = "stores${it}" def content = NEW_RESOURCE_CONTENT.replace('2020',String.valueOf(year)).replace('stores',moduleName) - newYangResourcesNameToContentMap.put(resourceName, content) + newYangResourceContentPerName.put(resourceName, content) } - objectUnderTest.createSchemaSet(CPS_PERFORMANCE_TEST_DATASPACE, 'perfSchemaSet', newYangResourcesNameToContentMap) + objectUnderTest.createSchemaSet(CPS_PERFORMANCE_TEST_DATASPACE, 'perfSchemaSet', newYangResourceContentPerName) then: 'the schema set is persisted correctly' def result = cpsModuleService.getSchemaSet(CPS_PERFORMANCE_TEST_DATASPACE, 'perfSchemaSet') result.moduleReferences.size() == 200 diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy index 613f760b0c..e52d3f819c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -100,7 +100,7 @@ class ModuleQueryPerfTest extends CpsPerfTestBase { cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, (i..i+100).collect {SCHEMA_SET_PREFIX + it}) } cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, [SCHEMA_SET_PREFIX + '0']) - cpsModuleService.deleteAllUnusedYangModuleData() + cpsModuleService.deleteAllUnusedYangModuleData(CPS_PERFORMANCE_TEST_DATASPACE) } // This makes a Yang module of approximately target length in bytes by padding the description field with many '*' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy index 364127f388..acc95cab8d 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy @@ -103,5 +103,4 @@ class QueryPerfTest extends CpsPerfTestBase { 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.11 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.34 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } - } diff --git a/integration-test/src/test/java/org/onap/cps/integration/KafkaTestContainer.java b/integration-test/src/test/java/org/onap/cps/integration/KafkaTestContainer.java index ff4aec4175..60c1637c5a 100644 --- a/integration-test/src/test/java/org/onap/cps/integration/KafkaTestContainer.java +++ b/integration-test/src/test/java/org/onap/cps/integration/KafkaTestContainer.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation. + * Copyright (C) 2024-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,8 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.StringDeserializer; -import org.testcontainers.containers.KafkaContainer; -import org.testcontainers.utility.DockerImageName; +import org.testcontainers.kafka.ConfluentKafkaContainer; /** * The Apache Kafka test container wrapper. @@ -35,14 +34,14 @@ import org.testcontainers.utility.DockerImageName; * Avoid unnecessary resource and time consumption. */ @Slf4j -public class KafkaTestContainer extends KafkaContainer { +public class KafkaTestContainer extends ConfluentKafkaContainer { - private static final String IMAGE_NAME_AND_VERSION = "registry.nordix.org/onaptest/confluentinc/cp-kafka:6.2.1"; + private static final String IMAGE_NAME_AND_VERSION = "confluentinc/cp-kafka:7.8.0"; private static volatile KafkaTestContainer kafkaTestContainer; private KafkaTestContainer() { - super(DockerImageName.parse(IMAGE_NAME_AND_VERSION).asCompatibleSubstituteFor("confluentinc/cp-kafka")); + super(IMAGE_NAME_AND_VERSION); } /** diff --git a/integration-test/src/test/resources/application.yml b/integration-test/src/test/resources/application.yml index 30598dfb90..e213a70a59 100644 --- a/integration-test/src/test/resources/application.yml +++ b/integration-test/src/test/resources/application.yml @@ -102,6 +102,7 @@ app: cm-subscription-dmi-out: ${CM_SUBSCRIPTION_DMI_OUT_TOPIC:dmi-ncmp-cm-avc-subscription} cm-subscription-ncmp-out: ${CM_SUBSCRIPTION_NCMP_OUT_TOPIC:subscription-response} cm-events-topic: ${NCMP_CM_EVENTS_TOPIC:cm-events} + inventory-events-topic: ncmp-inventory-events lcm: events: topic: ${LCM_EVENTS_TOPIC:ncmp-events} @@ -189,10 +190,6 @@ ncmp: trust-level: dmi-availability-watchdog-ms: 30000 - modules-sync-watchdog: - async-executor: - parallelism-level: 2 - model-loader: maximum-attempt-count: 20 |