diff options
55 files changed, 481 insertions, 373 deletions
diff --git a/cps-application/pom.xml b/cps-application/pom.xml index 5ac2202e85..1a46b5fa8a 100644 --- a/cps-application/pom.xml +++ b/cps-application/pom.xml @@ -78,6 +78,10 @@ <artifactId>micrometer-tracing-bridge-otel</artifactId> </dependency> <dependency> + <groupId>io.github.mweirauch</groupId> + <artifactId>micrometer-jvm-extras</artifactId> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.dataformat</groupId> <artifactId>jackson-dataformat-xml</artifactId> </dependency> diff --git a/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java b/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java index de981164f5..8481eadf1b 100644 --- a/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java +++ b/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java @@ -21,9 +21,12 @@ package org.onap.cps.config; import com.hazelcast.map.IMap; +import io.github.mweirauch.micrometer.jvm.extras.ProcessMemoryMetrics; +import io.github.mweirauch.micrometer.jvm.extras.ProcessThreadMetrics; import io.micrometer.core.aop.TimedAspect; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; import lombok.RequiredArgsConstructor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -41,6 +44,16 @@ public class MicroMeterConfig { return new TimedAspect(meterRegistry); } + @Bean + public MeterBinder processMemoryMetrics() { + return new ProcessMemoryMetrics(); + } + + @Bean + public MeterBinder processThreadMetrics() { + return new ProcessThreadMetrics(); + } + /** * Register gauge metric for cm handles with state 'advised'. * @@ -52,7 +65,7 @@ public class MicroMeterConfig { return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("advisedCmHandlesCount")) .tag(STATE_TAG, "ADVISED") - .description("Current number of cmhandles in advised state") + .description("Current number of cm handles in advised state") .register(meterRegistry); } @@ -67,7 +80,7 @@ public class MicroMeterConfig { return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("readyCmHandlesCount")) .tag(STATE_TAG, "READY") - .description("Current number of cmhandles in ready state") + .description("Current number of cm handles in ready state") .register(meterRegistry); } @@ -82,7 +95,7 @@ public class MicroMeterConfig { return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("lockedCmHandlesCount")) .tag(STATE_TAG, "LOCKED") - .description("Current number of cmhandles in locked state") + .description("Current number of cm handles in locked state") .register(meterRegistry); } @@ -97,7 +110,22 @@ public class MicroMeterConfig { return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("deletingCmHandlesCount")) .tag(STATE_TAG, "DELETING") - .description("Current number of cmhandles in deleting state") + .description("Current number of cm handles in deleting state") + .register(meterRegistry); + } + + /** + * Register gauge metric for cm handles with state 'deleted'. + * + * @param meterRegistry meter registry + * @return cm handle state gauge + */ + @Bean + public Gauge deletedCmHandles(final MeterRegistry meterRegistry) { + return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, + value -> cmHandlesByState.get("deletedCmHandlesCount")) + .tag(STATE_TAG, "DELETED") + .description("Number of cm handles that have been deleted since the application started") .register(meterRegistry); } diff --git a/cps-application/src/main/resources/application.yml b/cps-application/src/main/resources/application.yml index 27a15b6b51..0b5d59ecc9 100644 --- a/cps-application/src/main/resources/application.yml +++ b/cps-application/src/main/resources/application.yml @@ -172,7 +172,7 @@ management: endpoints: web: exposure: - include: info,health,loggers,prometheus,metrics + include: info,health,loggers,prometheus,metrics,heapdump,threaddump endpoint: health: show-details: always diff --git a/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy b/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy index da3afc6f2c..b9302ccd72 100644 --- a/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy +++ b/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy @@ -31,10 +31,17 @@ class MicroMeterConfigSpec extends Specification { def simpleMeterRegistry = new SimpleMeterRegistry() def 'Creating a timed aspect.'() { - expect: ' a timed aspect can be created' + expect: 'a timed aspect can be created' assert objectUnderTest.timedAspect(simpleMeterRegistry) != null } + def 'Creating JVM process metrics.'() { + expect: 'process memory metrics can be created' + assert objectUnderTest.processMemoryMetrics() != null + and: 'process thread metrics can be created' + assert objectUnderTest.processThreadMetrics() != null + } + def 'Creating gauges for cm handle states.'() { given: 'cache returns value for each state' cmHandlesByState.get(_) >> 1 @@ -43,10 +50,10 @@ class MicroMeterConfigSpec extends Specification { objectUnderTest.readyCmHandles(simpleMeterRegistry) objectUnderTest.lockedCmHandles(simpleMeterRegistry) objectUnderTest.deletingCmHandles(simpleMeterRegistry) + objectUnderTest.deletedCmHandles(simpleMeterRegistry) then: 'each state has the correct value when queried' - def states = ["ADVISED", "READY", "LOCKED", "DELETING"] - states.each { state -> - def gaugeValue = simpleMeterRegistry.get("cmHandlesByState").tag("state",state).gauge().value() + ['ADVISED', 'READY', 'LOCKED', 'DELETING', 'DELETED'].each { state -> + def gaugeValue = simpleMeterRegistry.get('cmHandlesByState').tag('state',state).gauge().value() assert gaugeValue == 1 } } diff --git a/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java b/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java index 1d39060024..c1d65758c7 100644 --- a/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java +++ b/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java @@ -36,6 +36,7 @@ public class ArchitectureTestBase { "lombok..", "org.apache..", "org.mapstruct..", + "org.opendaylight..", "org.slf4j..", "org.springframework..", "reactor.." diff --git a/cps-dependencies/pom.xml b/cps-dependencies/pom.xml index 5783ef7fcd..62fa154611 100644 --- a/cps-dependencies/pom.xml +++ b/cps-dependencies/pom.xml @@ -146,7 +146,7 @@ <dependency> <groupId>com.hazelcast</groupId> <artifactId>hazelcast-spring</artifactId> - <version>5.3.7</version> + <version>5.5.0</version> </dependency> <dependency> <groupId>com.squareup.okhttp3</groupId> @@ -173,6 +173,11 @@ <scope>import</scope> </dependency> <dependency> + <groupId>io.github.mweirauch</groupId> + <artifactId>micrometer-jvm-extras</artifactId> + <version>0.2.2</version> + </dependency> + <dependency> <groupId>io.gsonfire</groupId> <artifactId>gson-fire</artifactId> <version>1.9.0</version> diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java index 97fd8e5ad8..3415793478 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java @@ -49,7 +49,7 @@ import org.onap.cps.api.CpsDataService; import org.onap.cps.api.exceptions.DataNodeNotFoundException; import org.onap.cps.api.exceptions.DataValidationException; import org.onap.cps.api.model.DataNode; -import org.onap.cps.api.model.DataNodeBuilder; +import org.onap.cps.impl.DataNodeBuilder; import org.onap.cps.ncmp.api.inventory.models.CmHandleRegistrationResponse; import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java index 9534cf35b1..041daa0927 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java @@ -35,8 +35,6 @@ import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.CpsAnchorService; import org.onap.cps.api.CpsDataService; import org.onap.cps.api.CpsModuleService; -import org.onap.cps.api.exceptions.AlreadyDefinedException; -import org.onap.cps.api.exceptions.DuplicatedYangResourceException; import org.onap.cps.api.model.ModuleReference; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; import org.onap.cps.utils.ContentType; @@ -101,20 +99,14 @@ public class ModuleSyncService { private void syncAndCreateSchemaSet(final YangModelCmHandle yangModelCmHandle, final String schemaSetName) { if (isNewSchemaSet(schemaSetName)) { final ModuleDelta moduleDelta = getModuleDelta(yangModelCmHandle); - try { - log.info("Creating Schema Set {} for CM Handle {}", schemaSetName, yangModelCmHandle.getId()); - cpsModuleService.createSchemaSetFromModules( - NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, - schemaSetName, - moduleDelta.newModuleNameToContentMap, - moduleDelta.allModuleReferences - ); - log.info("Successfully created Schema Set {} for CM Handle {}", - schemaSetName, yangModelCmHandle.getId()); - } catch (final AlreadyDefinedException | DuplicatedYangResourceException exception) { - log.warn("Schema Set {} already exists, no need to (re)create it for {}", - schemaSetName, yangModelCmHandle.getId()); - } + log.info("Creating Schema Set {} for CM Handle {}", schemaSetName, yangModelCmHandle.getId()); + cpsModuleService.createSchemaSetFromModules( + NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, + schemaSetName, + moduleDelta.newModuleNameToContentMap, + moduleDelta.allModuleReferences + ); + log.info("Successfully created Schema Set {} for CM Handle {}", schemaSetName, yangModelCmHandle.getId()); } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy index 4d42e62025..8b04568239 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy @@ -24,7 +24,7 @@ package org.onap.cps.ncmp.api.inventory.models import org.onap.cps.ncmp.api.inventory.DataStoreSyncState import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import spock.lang.Specification import java.time.OffsetDateTime diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy index b600d02be5..70bd418026 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy @@ -32,7 +32,7 @@ import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle import org.onap.cps.api.exceptions.DataNodeNotFoundException import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import org.onap.cps.utils.ContentType import org.onap.cps.utils.JsonObjectMapper import org.slf4j.LoggerFactory diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy index f8adfe5578..7881375762 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy @@ -90,20 +90,21 @@ class ModuleSyncServiceSpec extends Specification { 'without' | '' } - def 'Attempt Sync models for a cm handle with existing schema set (#exception).'() { + def 'Attempt Sync models for a cm handle with existing schema set (#originalException).'() { given: 'a cm handle to be synced' def yangModelCmHandle = createAdvisedCmHandle('existing tag') and: 'dmi returns no new yang resources' mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] and: 'already defined exception occurs when creating schema (existing)' - mockCpsModuleService.createSchemaSetFromModules(*_) >> { throw exception } + mockCpsModuleService.createSchemaSetFromModules(*_) >> { throw originalException } when: 'module sync is triggered' objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) - then: 'no exception is thrown up' - noExceptionThrown() + then: 'same exception is thrown up' + def thrownException = thrown(Exception) + assert thrownException == originalException where: 'following exceptions occur' - exception << [ AlreadyDefinedException.forSchemaSet('', '', null), - new DuplicatedYangResourceException('', '', null) ] + originalException << [AlreadyDefinedException.forSchemaSet('', '', null), + new DuplicatedYangResourceException('', '', null) ] } def 'Model upgrade without using Module Set Tags (legacy) where the modules are in database.'() { diff --git a/cps-rest/docs/openapi/cpsAdmin.yml b/cps-rest/docs/openapi/cpsAdmin.yml index f394270dd5..6cfffa48f3 100644 --- a/cps-rest/docs/openapi/cpsAdmin.yml +++ b/cps-rest/docs/openapi/cpsAdmin.yml @@ -1,6 +1,6 @@ # ============LICENSE_START======================================================= # Copyright (c) 2021 Bell Canada. -# Modifications Copyright (C) 2021-2022 Nordix Foundation +# Modifications Copyright (C) 2021-2025 Nordix Foundation # Modifications Copyright (C) 2022 TechMahindra Ltd. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); @@ -230,3 +230,23 @@ adminDataspace: $ref: 'components.yml#/components/responses/Forbidden' '500': $ref: 'components.yml#/components/responses/InternalServerError' + +adminCleanDataspace: + post: + description: Clean the dataspace (remove orphaned schema sets and modules) + tags: + - cps-admin + summary: Clean the dataspace + operationId: cleanDataspace + parameters: + - $ref: 'components.yml#/components/parameters/apiVersionInPath' + - $ref: 'components.yml#/components/parameters/dataspaceNameInPath' + responses: + '204': + $ref: 'components.yml#/components/responses/NoContent' + '400': + $ref: 'components.yml#/components/responses/BadRequest' + '403': + $ref: 'components.yml#/components/responses/Forbidden' + '500': + $ref: 'components.yml#/components/responses/InternalServerError' diff --git a/cps-rest/docs/openapi/openapi.yml b/cps-rest/docs/openapi/openapi.yml index f4eab61875..c85bf7cac7 100644 --- a/cps-rest/docs/openapi/openapi.yml +++ b/cps-rest/docs/openapi/openapi.yml @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (C) 2021-2024 Nordix Foundation +# Copyright (C) 2021-2025 Nordix Foundation # Modifications Copyright (C) 2021 Pantheon.tech # Modifications Copyright (C) 2021 Bell Canada. # Modifications Copyright (C) 2022-2024 TechMahindra Ltd. @@ -61,6 +61,9 @@ paths: /{apiVersion}/admin/dataspaces/{dataspace-name}: $ref: 'cpsAdmin.yml#/adminDataspace' + /{apiVersion}/admin/dataspaces/{dataspace-name}/actions/clean: + $ref: 'cpsAdmin.yml#/adminCleanDataspace' + /v1/dataspaces/{dataspace-name}/anchors: $ref: 'cpsAdminV1Deprecated.yml#/anchorsByDataspace' diff --git a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java index 675c0eaec4..4c6bd6cdc5 100755 --- a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2023 Nordix Foundation + * Copyright (C) 2020-2025 Nordix Foundation * Modifications Copyright (C) 2020-2021 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -176,6 +176,20 @@ public class AdminRestController implements CpsAdminApi { } /** + * Clean the given dataspace of any orphaned (module) data. + * + * @param apiVersion api version + * @param dataspaceName dataspace name + * + * @return a {@Link ResponseEntity} of {@link HttpStatus} NO_CONTENT + */ + @Override + public ResponseEntity<Void> cleanDataspace(final String apiVersion, final String dataspaceName) { + cpsModuleService.deleteAllUnusedYangModuleData(dataspaceName); + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + /** * Create a new anchor. * * @param dataspaceName dataspace name diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy index 2335a5e770..0d189783fd 100755 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * Copyright (C) 2020-2021 Pantheon.tech * Modifications Copyright (C) 2020-2021 Bell Canada. - * Modifications Copyright (C) 2021-2022 Nordix Foundation + * Modifications Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2022 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,15 +23,8 @@ package org.onap.cps.rest.controller -import org.onap.cps.api.CpsAnchorService - -import static org.onap.cps.api.parameters.CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post - import org.mapstruct.factory.Mappers +import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataspaceService import org.onap.cps.api.CpsModuleService import org.onap.cps.api.exceptions.AlreadyDefinedException @@ -51,6 +44,12 @@ import org.springframework.util.LinkedMultiValueMap import org.springframework.util.MultiValueMap import spock.lang.Specification +import static org.onap.cps.api.parameters.CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post + @WebMvcTest(AdminRestController) class AdminRestControllerSpec extends Specification { @@ -79,7 +78,7 @@ class AdminRestControllerSpec extends Specification { def dataspace = new Dataspace(name: dataspaceName) def 'Create new dataspace with #scenario.'() { - when: 'post is invoked' + when: 'post is invoked on endpoint for creating a dataspace' def response = mvc.perform( post("/cps/api/${apiVersion}/dataspaces") @@ -97,7 +96,7 @@ class AdminRestControllerSpec extends Specification { } def 'Create dataspace over existing with same name.'() { - given: 'an endpoint' + given: 'the endpoint to create a dataspace' def createDataspaceEndpoint = "$basePath/v1/dataspaces" and: 'the service method throws an exception indicating the dataspace is already defined' def thrownException = new AlreadyDefinedException(dataspaceName, new RuntimeException()) @@ -115,7 +114,7 @@ class AdminRestControllerSpec extends Specification { def 'Get a dataspace.'() { given: 'service method returns a dataspace' mockCpsDataspaceService.getDataspace(dataspaceName) >> dataspace - and: 'an endpoint' + and: 'the endpoint for getting a dataspace by name' def getDataspaceEndpoint = "$basePath/v1/admin/dataspaces/$dataspaceName" when: 'get dataspace API is invoked' def response = mvc.perform(get(getDataspaceEndpoint)).andReturn().response @@ -124,6 +123,17 @@ class AdminRestControllerSpec extends Specification { response.getContentAsString().contains(dataspaceName) } + def 'Clean a dataspace.'() { + given: 'service method returns a dataspace' + mockCpsDataspaceService.getDataspace(dataspaceName) >> dataspace + and: 'the endpoint for cleaning a dataspace' + def postCleanDataspaceEndpoint = "$basePath/v1/admin/dataspaces/$dataspaceName/actions/clean" + when: 'post is invoked on the clean dataspace endpoint' + def response = mvc.perform(post(postCleanDataspaceEndpoint)).andReturn().response + then: 'no content is returned' + response.status == HttpStatus.NO_CONTENT.value() + } + def 'Get all dataspaces.'() { given: 'service method returns all dataspace' mockCpsDataspaceService.getAllDataspaces() >> [dataspace, new Dataspace(name: "dataspace-test2")] @@ -173,8 +183,7 @@ class AdminRestControllerSpec extends Specification { .param('schema-set-name', schemaSetName)) .andReturn().response then: 'associated service method is invoked with expected parameters' - 1 * mockCpsModuleService.createSchemaSet(dataspaceName, schemaSetName, _) >> - { args -> yangResourceMapCapture = args[2] } + 1 * mockCpsModuleService.createSchemaSet(dataspaceName, schemaSetName, _) >> { args -> yangResourceMapCapture = args[2] } yangResourceMapCapture['assembly.yang'] == "fake assembly content 1\n" yangResourceMapCapture['component.yang'] == "fake component content 1\n" and: 'response code indicates success' @@ -208,7 +217,7 @@ class AdminRestControllerSpec extends Specification { } def 'Create schema set from zip archive having #caseDescriptor.'() { - given: 'an endpoint' + given: 'the endpoint to create a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'zip archive having #caseDescriptor is uploaded with create schema set request' def response = @@ -228,7 +237,7 @@ class AdminRestControllerSpec extends Specification { def 'Create schema set from file with unsupported filename extension.'() { given: 'file with unsupported filename extension (.doc)' def multipartFile = createMultipartFile("filename.doc", "content") - and: 'an endpoint' + and: 'the endpoint to create a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'file uploaded with schema set create request' def response = @@ -242,7 +251,7 @@ class AdminRestControllerSpec extends Specification { } def 'Create schema set from #fileType file with IOException occurrence on processing.'() { - given: 'an endpoint' + given: 'the endpoint to create a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'file uploaded with schema set create request' def multipartFile = createMultipartFileForIOException(fileType) @@ -259,7 +268,7 @@ class AdminRestControllerSpec extends Specification { } def 'Delete schema set.'() { - given: 'an endpoint' + given: 'the endpoint for deleting a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets/$schemaSetName" when: 'delete schema set endpoint is invoked' def response = mvc.perform(delete(schemaSetEndpoint)).andReturn().response @@ -274,7 +283,7 @@ class AdminRestControllerSpec extends Specification { def thrownException = new SchemaSetInUseException(dataspaceName, schemaSetName) mockCpsModuleService.deleteSchemaSet(dataspaceName, schemaSetName, CASCADE_DELETE_PROHIBITED) >> { throw thrownException } - and: 'an endpoint' + and: 'the endpoint for deleting a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets/$schemaSetName" when: 'delete schema set endpoint is invoked' def response = mvc.perform(delete(schemaSetEndpoint)).andReturn().response @@ -286,7 +295,7 @@ class AdminRestControllerSpec extends Specification { given: 'service method returns a new schema set' mockCpsModuleService.getSchemaSet(dataspaceName, schemaSetName) >> new SchemaSet(name: schemaSetName, dataspaceName: dataspaceName) - and: 'an endpoint' + and: 'the endpoint for getting a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets/$schemaSetName" when: 'get schema set API is invoked' def response = mvc.perform(get(schemaSetEndpoint)).andReturn().response @@ -300,7 +309,7 @@ class AdminRestControllerSpec extends Specification { mockCpsModuleService.getSchemaSets(dataspaceName) >> [new SchemaSet(name: schemaSetName, dataspaceName: dataspaceName), new SchemaSet(name: "test-schemaset", dataspaceName: dataspaceName)] - and: 'an endpoint' + and: 'the endpoint for getting all schema sets' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'get schema sets API is invoked' def response = mvc.perform(get(schemaSetEndpoint)).andReturn().response @@ -315,7 +324,7 @@ class AdminRestControllerSpec extends Specification { def requestParams = new LinkedMultiValueMap<>() requestParams.add('schema-set-name', schemaSetName) requestParams.add('anchor-name', anchorName) - when: 'post is invoked' + when: 'post is invoked on the create anchors endpoint' def response = mvc.perform( post("/cps/api/${apiVersion}/dataspaces/my_dataspace/anchors") @@ -332,10 +341,10 @@ class AdminRestControllerSpec extends Specification { 'V2 API' | 'v2' || '' } - def 'Get existing anchor.'() { - given: 'service method returns a list of anchors' + def 'Get existing anchors.'() { + given: 'service method returns a list of (one) anchors' mockCpsAnchorService.getAnchors(dataspaceName) >> [anchor] - and: 'an endpoint' + and: 'the endpoint for getting all anchors' def anchorEndpoint = "$basePath/v1/dataspaces/$dataspaceName/anchors" when: 'get all anchors API is invoked' def response = mvc.perform(get(anchorEndpoint)).andReturn().response @@ -348,7 +357,7 @@ class AdminRestControllerSpec extends Specification { given: 'service method returns an anchor' mockCpsAnchorService.getAnchor(dataspaceName, anchorName) >> new Anchor(name: anchorName, dataspaceName: dataspaceName, schemaSetName: schemaSetName) - and: 'an endpoint' + and: 'the endpoint for getting an anchor' def anchorEndpoint = "$basePath/v1/dataspaces/$dataspaceName/anchors/$anchorName" when: 'get anchor API is invoked' def response = mvc.perform(get(anchorEndpoint)).andReturn().response @@ -361,7 +370,7 @@ class AdminRestControllerSpec extends Specification { } def 'Delete anchor.'() { - given: 'an endpoint' + given: 'the endpoint for deleting an anchor' def anchorEndpoint = "$basePath/v1/dataspaces/$dataspaceName/anchors/$anchorName" when: 'delete method is invoked on anchor endpoint' def response = mvc.perform(delete(anchorEndpoint)).andReturn().response @@ -372,7 +381,7 @@ class AdminRestControllerSpec extends Specification { } def 'Delete dataspace.'() { - given: 'an endpoint' + given: 'the endpoint for deleting a dataspace' def dataspaceEndpoint = "$basePath/v1/dataspaces" when: 'delete dataspace endpoint is invoked' def response = mvc.perform(delete(dataspaceEndpoint) diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy index ca89fafe83..f2f962422f 100755 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy @@ -30,8 +30,8 @@ import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder -import org.onap.cps.api.model.DeltaReportBuilder +import org.onap.cps.impl.DataNodeBuilder +import org.onap.cps.impl.DeltaReportBuilder import org.onap.cps.utils.ContentType import org.onap.cps.utils.DateTimeUtility import org.onap.cps.utils.JsonObjectMapper diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy index f29654c99f..2b5c471287 100644 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsQueryService import org.onap.cps.api.parameters.PaginationOption -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import org.onap.cps.utils.JsonObjectMapper import org.onap.cps.utils.PrefixResolver import org.spockframework.spring.SpringBean diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java index c43c8e2999..52fd7f2be1 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java @@ -50,12 +50,12 @@ import org.onap.cps.api.exceptions.CpsPathException; import org.onap.cps.api.exceptions.DataNodeNotFoundException; import org.onap.cps.api.exceptions.DataNodeNotFoundExceptionBatch; import org.onap.cps.api.model.DataNode; -import org.onap.cps.api.model.DataNodeBuilder; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.api.parameters.PaginationOption; import org.onap.cps.cpspath.parser.CpsPathQuery; import org.onap.cps.cpspath.parser.CpsPathUtil; import org.onap.cps.cpspath.parser.PathParsingException; +import org.onap.cps.impl.DataNodeBuilder; import org.onap.cps.ri.models.AnchorEntity; import org.onap.cps.ri.models.DataspaceEntity; import org.onap.cps.ri.models.FragmentEntity; diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java index cf9fb021a6..4f7492ff26 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java @@ -71,8 +71,6 @@ import org.opendaylight.yangtools.yang.parser.api.YangSyntaxErrorException; import org.opendaylight.yangtools.yang.parser.rfc7950.repo.YangModelDependencyInfo; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.retry.RetryContext; -import org.springframework.retry.annotation.Backoff; -import org.springframework.retry.annotation.Retryable; import org.springframework.retry.support.RetrySynchronizationManager; import org.springframework.stereotype.Component; @@ -154,10 +152,6 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override @Transactional - // A retry is made to store the schema set if it fails because of duplicated yang resource exception that - // can occur in case of specific concurrent requests. - @Retryable(retryFor = DuplicatedYangResourceException.class, maxAttempts = 5, backoff = - @Backoff(random = true, delay = 200, maxDelay = 2000, multiplier = 2)) public void storeSchemaSet(final String dataspaceName, final String schemaSetName, final Map<String, String> moduleReferenceNameToContentMap) { final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); @@ -189,12 +183,8 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override @Transactional - // A retry is made to store the schema set if it fails because of duplicated yang resource exception that - // can occur in case of specific concurrent requests. - @Retryable(retryFor = DuplicatedYangResourceException.class, maxAttempts = 5, backoff = - @Backoff(random = true, delay = 200, maxDelay = 2000, multiplier = 2)) @Timed(value = "cps.module.persistence.schemaset.store", - description = "Time taken to store a schemaset (list of module references") + description = "Time taken to store a schemaset (list of module references)") public void storeSchemaSetFromModules(final String dataspaceName, final String schemaSetName, final Map<String, String> newModuleNameToContentMap, final Collection<ModuleReference> allModuleReferences) { @@ -237,8 +227,9 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override @Transactional - public void deleteAllUnusedYangModuleData() { - schemaSetRepository.deleteOrphanedSchemaSets(); + public void deleteAllUnusedYangModuleData(final String dataspaceName) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + schemaSetRepository.deleteOrphanedSchemaSets(dataspaceEntity.getId()); yangResourceRepository.deleteOrphanedYangResources(); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java index b8dd7b755c..fdd72624ba 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java @@ -83,8 +83,8 @@ public interface SchemaSetRepository extends JpaRepository<SchemaSetEntity, Inte */ @Modifying @Query(value = """ - DELETE FROM schema_set WHERE NOT EXISTS - (SELECT 1 FROM anchor WHERE anchor.schema_set_id = schema_set.id) + DELETE FROM schema_set WHERE schema_set.dataspace_id = :dataspaceId AND + NOT EXISTS (SELECT 1 FROM anchor WHERE anchor.schema_set_id = schema_set.id) """, nativeQuery = true) - void deleteOrphanedSchemaSets(); + void deleteOrphanedSchemaSets(@Param("dataspaceId") final int dataspaceId); } diff --git a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy index c818f3ba1f..e927922acf 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy @@ -33,7 +33,7 @@ import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.exceptions.ConcurrencyException import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import org.onap.cps.utils.JsonObjectMapper import org.springframework.dao.DataIntegrityViolationException import spock.lang.Specification diff --git a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceConcurrencySpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceConcurrencySpec.groovy deleted file mode 100644 index 28a615b0e8..0000000000 --- a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceConcurrencySpec.groovy +++ /dev/null @@ -1,145 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2022 Bell Canada. - * Modifications Copyright (C) 2021-2023 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the 'License'); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ -package org.onap.cps.ri - -import org.hibernate.exception.ConstraintViolationException -import org.onap.cps.ri.models.DataspaceEntity -import org.onap.cps.ri.models.SchemaSetEntity -import org.onap.cps.ri.repository.DataspaceRepository -import org.onap.cps.ri.repository.ModuleReferenceRepository -import org.onap.cps.ri.repository.SchemaSetRepository -import org.onap.cps.ri.repository.YangResourceRepository -import org.onap.cps.spi.CpsAdminPersistenceService -import org.onap.cps.spi.CpsModulePersistenceService -import org.onap.cps.api.exceptions.DuplicatedYangResourceException -import org.onap.cps.api.model.ModuleReference -import org.spockframework.spring.SpringBean -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.test.context.SpringBootTest -import org.springframework.dao.DataIntegrityViolationException -import org.springframework.retry.annotation.EnableRetry -import spock.lang.Specification - -import java.sql.SQLException - -@SpringBootTest(classes=[CpsModulePersistenceServiceImpl]) -@EnableRetry -class CpsModulePersistenceServiceConcurrencySpec extends Specification { - - @Autowired - CpsModulePersistenceService objectUnderTest - - @SpringBean - DataspaceRepository dataspaceRepository = Mock() - - @SpringBean - YangResourceRepository yangResourceRepository = Mock() - - @SpringBean - SchemaSetRepository schemaSetRepository = Mock() - - @SpringBean - CpsAdminPersistenceService cpsAdminPersistenceService = Mock() - - @SpringBean - ModuleReferenceRepository moduleReferenceRepository = Mock() - - def NEW_RESOURCE_NAME = 'some new resource' - def NEW_RESOURCE_CONTENT = 'module stores {\n' + - ' yang-version 1.1;\n' + - ' namespace "org:onap:ccsdk:sample";\n' + - '}' - - def newYangResourcesNameToContentMap = [(NEW_RESOURCE_NAME):NEW_RESOURCE_CONTENT] - - def yangResourceChecksum = 'b13faef573ed1374139d02c40d8ce09c80ea1dc70e63e464c1ed61568d48d539' - - def yangResourceChecksumDbConstraint = 'yang_resource_checksum_key' - - def sqlExceptionMessage = String.format('(checksum)=(%s)', yangResourceChecksum) - - def checksumIntegrityException = new DataIntegrityViolationException("checksum integrity exception", - new ConstraintViolationException('', new SQLException(sqlExceptionMessage), yangResourceChecksumDbConstraint)) - - def 'Store new schema set, maximum retries.'() { - given: 'no pre-existing schemaset in database' - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - when: 'a new schemaset is stored' - objectUnderTest.storeSchemaSet('some dataspace', 'some new schema set', newYangResourcesNameToContentMap) - then: 'a duplicated yang resource exception is thrown ' - thrown(DuplicatedYangResourceException) - and: 'the system will attempt to save the data 5 times (because checksum integrity exception is thrown each time)' - 5 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - } - - def 'Store new schema set, succeed on third attempt.'() { - given: 'no pre-existing schemaset in database' - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - when: 'a new schemaset is stored' - objectUnderTest.storeSchemaSet('some dataspace', 'some new schema set', newYangResourcesNameToContentMap) - then: 'no exception is thrown ' - noExceptionThrown() - and: 'the system will attempt to save the data 2 times with checksum integrity exception but then succeed' - 2 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - 1 * yangResourceRepository.saveAll(_) >> [] - } - - def 'Store schema set using modules, maximum retries.'() { - given: 'map of new modules, a list of existing modules, module reference' - def mapOfNewModules = [newModule1: 'module newmodule { yang-version 1.1; revision "2021-10-12" { } }'] - def moduleReferenceForExistingModule = new ModuleReference("test","2021-10-12") - def listOfExistingModulesModuleReference = [moduleReferenceForExistingModule] - and: 'no pre-existing schemaset in database' - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - when: 'a new schemaset is stored from a module' - objectUnderTest.storeSchemaSetFromModules('some dataspace', 'some new schema set' , mapOfNewModules, listOfExistingModulesModuleReference) - then: 'a duplicated yang resource exception is thrown ' - thrown(DuplicatedYangResourceException) - and: 'the system will attempt to save the data 5 times (because checksum integrity exception is thrown each time)' - 5 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - } - - def 'Store schema set using modules, succeed on third attempt.'() { - given: 'map of new modules, a list of existing modules, module reference' - def mapOfNewModules = [newModule1: 'module newmodule { yang-version 1.1; revision "2021-10-12" { } }'] - def moduleReferenceForExistingModule = new ModuleReference("test","2021-10-12") - def listOfExistingModulesModuleReference = [moduleReferenceForExistingModule] - and: 'no pre-existing schemaset in database' - def dataspaceEntity = new DataspaceEntity() - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - yangResourceRepository.getResourceIdsByModuleReferences(_) >> [] - and: 'can retrieve schemaset details after storing it' - def schemaSetEntity = new SchemaSetEntity() - schemaSetRepository.getByDataspaceAndName(dataspaceEntity, 'new schema set') >> schemaSetEntity - when: 'a new schemaset is stored from a module' - objectUnderTest.storeSchemaSetFromModules('some dataspace', 'new schema set' , mapOfNewModules, listOfExistingModulesModuleReference) - then: 'no exception is thrown ' - noExceptionThrown() - and: 'the system will attempt to save the data 2 times with checksum integrity exception but then succeed' - 2 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - 1 * yangResourceRepository.saveAll(_) >> [] - } - -} diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java index 81b6439efc..c6b8c60ab9 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java @@ -164,8 +164,11 @@ public interface CpsModuleService { Collection<ModuleReference> identifyNewModuleReferences(Collection<ModuleReference> moduleReferencesToCheck); /** - * Remove any Yang Resource Modules and Schema Sets from the DB that are no longer referenced by any anchor. + * Remove any Yang Resource Modules and Schema Sets from the given dataspace that are no longer referenced + * by any anchor. + * + * @param dataspaceName dataspace name */ - void deleteAllUnusedYangModuleData(); + void deleteAllUnusedYangModuleData(String dataspaceName); } diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java b/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java index be80b636ad..be559709f8 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java +++ b/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java @@ -26,19 +26,18 @@ import java.io.Serializable; import java.util.Collection; import java.util.Collections; import java.util.Map; -import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -@Setter(AccessLevel.PROTECTED) +@Setter @Getter @EqualsAndHashCode public class DataNode implements Serializable { private static final long serialVersionUID = 1482619410918597467L; - DataNode() {} + public DataNode() {} private String dataspace; private String schemaSetName; diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java b/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java index df642628d0..77d8d771c1 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java +++ b/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java @@ -23,11 +23,10 @@ package org.onap.cps.api.model; import com.fasterxml.jackson.annotation.JsonInclude; import java.io.Serializable; import java.util.Map; -import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; -@Setter(AccessLevel.PROTECTED) +@Setter @Getter @JsonInclude(JsonInclude.Include.NON_NULL) public class DeltaReport { @@ -36,7 +35,7 @@ public class DeltaReport { public static final String REMOVE_ACTION = "remove"; public static final String REPLACE_ACTION = "replace"; - DeltaReport() {} + public DeltaReport() {} private String action; private String xpath; diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java index f2513173a6..653fd4803f 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java @@ -42,7 +42,6 @@ import org.onap.cps.api.CpsDeltaService; import org.onap.cps.api.exceptions.DataValidationException; import org.onap.cps.api.model.Anchor; import org.onap.cps.api.model.DataNode; -import org.onap.cps.api.model.DataNodeBuilder; import org.onap.cps.api.model.DeltaReport; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.cpspath.parser.CpsPathUtil; diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java index 7a9d142506..d532001aec 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java @@ -32,7 +32,6 @@ import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.CpsDeltaService; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.model.DeltaReport; -import org.onap.cps.api.model.DeltaReportBuilder; import org.springframework.stereotype.Service; @Slf4j diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java index 7622ba5fe2..1e92c7eb69 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java @@ -175,8 +175,9 @@ public class CpsModuleServiceImpl implements CpsModuleService { } @Override - public void deleteAllUnusedYangModuleData() { - cpsModulePersistenceService.deleteAllUnusedYangModuleData(); + public void deleteAllUnusedYangModuleData(final String dataspaceName) { + cpsValidator.validateNameCharacters(dataspaceName); + cpsModulePersistenceService.deleteAllUnusedYangModuleData(dataspaceName); } private boolean isCascadeDeleteProhibited(final CascadeDeleteAllowed cascadeDeleteAllowed) { diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DataNodeBuilder.java b/cps-service/src/main/java/org/onap/cps/impl/DataNodeBuilder.java index d509f53525..a78f3d9826 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DataNodeBuilder.java +++ b/cps-service/src/main/java/org/onap/cps/impl/DataNodeBuilder.java @@ -20,7 +20,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model; +package org.onap.cps.impl; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -33,6 +33,7 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.exceptions.DataValidationException; +import org.onap.cps.api.model.DataNode; import org.onap.cps.utils.YangUtils; import org.opendaylight.yangtools.yang.common.Ordering; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReportBuilder.java b/cps-service/src/main/java/org/onap/cps/impl/DeltaReportBuilder.java index a8e922f3df..fdc2e939d6 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReportBuilder.java +++ b/cps-service/src/main/java/org/onap/cps/impl/DeltaReportBuilder.java @@ -18,11 +18,12 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model; +package org.onap.cps.impl; import java.io.Serializable; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.onap.cps.api.model.DeltaReport; @Slf4j public class DeltaReportBuilder { diff --git a/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java b/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java index b1f8aad88f..86ad50252d 100755 --- a/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java +++ b/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java @@ -146,9 +146,11 @@ public interface CpsModulePersistenceService { String moduleName, String moduleRevision); /** - * Remove any unused Yang Resource Modules and Schema Sets. + * Remove any unused Yang Resource Modules and Schema Sets from the given dataspace. + * + * @param dataspaceName dataspace name */ - void deleteAllUnusedYangModuleData(); + void deleteAllUnusedYangModuleData(String dataspaceName); /** * Identify new module references from those returned by a node compared to what is in CPS already. diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy index 3ea859ae6d..a828d26991 100644 --- a/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy @@ -40,7 +40,6 @@ import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.api.exceptions.SessionManagerException import org.onap.cps.api.exceptions.SessionTimeoutException import org.onap.cps.api.model.Anchor -import org.onap.cps.api.model.DataNodeBuilder import org.onap.cps.utils.ContentType import org.onap.cps.utils.JsonObjectMapper import org.onap.cps.utils.PrefixResolver diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy index ce871621e5..af1859f36e 100644 --- a/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy @@ -252,11 +252,11 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsValidator.validateNameCharacters('some-dataspace-name', 'some-anchor-name') } - def 'Delete all unused yang module data.'() { + def 'Delete unused yang module data for a dataspace.'() { when: 'deleting unused yang module data' - objectUnderTest.deleteAllUnusedYangModuleData() - then: 'it is delegated to the module persistence service' - 1 * mockCpsModulePersistenceService.deleteAllUnusedYangModuleData() + objectUnderTest.deleteAllUnusedYangModuleData('some-dataspace-name') + then: 'it is delegated to the module persistence service with the correct parameters' + 1 * mockCpsModulePersistenceService.deleteAllUnusedYangModuleData('some-dataspace-name') } def 'Schema set exists.'() { diff --git a/cps-service/src/test/groovy/org/onap/cps/api/model/DataNodeBuilderSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/DataNodeBuilderSpec.groovy index 24c78864a5..1597d45761 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/model/DataNodeBuilderSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/DataNodeBuilderSpec.groovy @@ -19,10 +19,11 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model +package org.onap.cps.impl import org.onap.cps.TestUtils import org.onap.cps.api.exceptions.DataValidationException +import org.onap.cps.api.model.DataNode import org.onap.cps.utils.ContentType import org.onap.cps.utils.DataMapUtils import org.onap.cps.utils.YangParserHelper diff --git a/cps-service/src/test/groovy/org/onap/cps/api/model/DeltaReportBuilderSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/DeltaReportBuilderSpec.groovy index 94e3ed5c26..2decefff21 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/model/DeltaReportBuilderSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/DeltaReportBuilderSpec.groovy @@ -18,7 +18,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model +package org.onap.cps.impl import spock.lang.Specification diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy index bb0f5b0911..6ff41c128f 100644 --- a/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy @@ -22,7 +22,7 @@ package org.onap.cps.utils -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import spock.lang.Specification class DataMapUtilsSpec extends Specification { diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 4263329eed..c9df8b9889 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -68,7 +68,7 @@ services: ONAP_OTEL_EXPORTER_ENDPOINT: http://jaeger-service:4317 POLICY_SERVICE_ENABLED: 'false' POLICY_SERVICE_DEFAULT_DECISION: 'deny from env' - ####JAVA_TOOL_OPTIONS: "-XX:InitialRAMPercentage=75.0 -XX:MaxRAMPercentage=75.0" + JAVA_TOOL_OPTIONS: "-XX:InitialRAMPercentage=75.0 -XX:MaxRAMPercentage=75.0" ### DEBUG: Uncomment next line to enable java debugging ### JAVA_TOOL_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 restart: unless-stopped diff --git a/docs/admin-guide.rst b/docs/admin-guide.rst index 4a40f9b29c..68edc87c81 100644 --- a/docs/admin-guide.rst +++ b/docs/admin-guide.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021-2022 Nordix Foundation +.. Copyright (C) 2021-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _adminGuide: @@ -12,6 +12,40 @@ CPS Admin Guide .. toctree:: :maxdepth: 1 +Regular Maintenance +=================== +This section details tasks that an administrator of the CPS application should execute on regular basis +to ensure optimum working of CPS. + +Dataspace Clean Up +------------------ +Certain data in the CPS database might not be explicitly removed after it is no longer required ('orphaned data'). +For example, schema sets and their associated unique module resources no longer used by any anchor because of model upgrades. +This data would unnecessarily take up space and could eventually affect the performance of the DB if it is not deleted. +How often this needs to be done depends on how often schema sets are being deprecated. +Typically once per month should suffice. + +To remove orphaned data in a given dataspace use the following post request: + +.. code:: + + http://<cps-component-service-name>:<cps-port>/v2/admin/dataspaces/<dataspace-name>/actions/clean + +for example + +.. code-block:: bash + + curl --location --request POST 'http://cps:8080/admin/datsaspaces/bookstore/actions/clean' \ + --header 'Content-Type: application/json; charset=utf-8' + + Response : HTTP Status 204 + +For more details refer to the CPS-Core API: :doc:`design`. + +.. note:: + NCMP has no specific maintenance tasks but it will also build up orphaned data when CM Handles get updated and or deleted. + To delete this data execute the above procedure for the dataspace named 'NFP-Operational'. + Logging Configuration ===================== @@ -210,9 +244,8 @@ Naming Validation As part of the Kohn 3.1.0 release, CPS has added validation to the names of the following components: - Dataspace names - - Schema Set names - Anchor names - - Cm-Handle identifiers + - CM Handle identifiers The following characters along with spaces are no longer valid for naming of these components. diff --git a/docs/api/swagger/cps/openapi.yaml b/docs/api/swagger/cps/openapi.yaml index c84609b638..330c2ca7f7 100644 --- a/docs/api/swagger/cps/openapi.yaml +++ b/docs/api/swagger/cps/openapi.yaml @@ -330,6 +330,65 @@ paths: summary: Get a dataspace tags: - cps-admin + /{apiVersion}/admin/dataspaces/{dataspace-name}/actions/clean: + post: + description: Clean the dataspace (remove orphaned modules) + operationId: cleanDataspace + parameters: + - description: apiVersion + in: path + name: apiVersion + required: true + schema: + default: v2 + enum: + - v1 + - v2 + type: string + - description: dataspace-name + in: path + name: dataspace-name + required: true + schema: + example: my-dataspace + type: string + responses: + "204": + content: {} + description: No Content + "400": + content: + application/json: + example: + status: 400 + message: Bad Request + details: The provided request is not valid + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Bad Request + "403": + content: + application/json: + example: + status: 403 + message: Request Forbidden + details: This request is forbidden + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Forbidden + "500": + content: + application/json: + example: + status: 500 + message: Internal Server Error + details: Internal Server Error occurred + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Internal Server Error + summary: Clean the dataspace + tags: + - cps-admin /v1/dataspaces/{dataspace-name}/anchors: post: deprecated: true diff --git a/docs/cm-handle-lcm-events.rst b/docs/cm-handle-lcm-events.rst index 8446834c31..38339e206a 100644 --- a/docs/cm-handle-lcm-events.rst +++ b/docs/cm-handle-lcm-events.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023 Nordix Foundation +.. Copyright (C) 2023-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cmHandleLcmEvents: @@ -15,7 +15,7 @@ CM Handle Lifecycle Management (LCM) Events Introduction ============ -LCM events for CM Handles are published when a CM Handle is created, deleted or another change in the cm handle state occurs. +LCM events for CM Handles are published when a CM Handle is created, deleted or another change in the CM Handle state occurs. **3 possible event types:** @@ -55,7 +55,7 @@ Event payload varies based on the type of event. **CREATE** -Event payload for this event contains the properties of the new cm handle created. +Event payload for this event contains the properties of the new CM Handle created. *Create event payload prototype* @@ -77,7 +77,7 @@ Event payload for this event contains the properties of the new cm handle create **UPDATE** -Event payload for this event contains the difference in state and properties of the cm handle. +Event payload for this event contains the difference in state and properties of the CM Handle. *Update event payload prototype* @@ -106,7 +106,7 @@ Event payload for this event contains the difference in state and properties of **DELETE** -Event payload for this event contains the identifier of the deleted cm handle. +Event payload for this event contains the identifier of the deleted CM Handle. *Delete event payload prototype* @@ -114,4 +114,4 @@ Event payload for this event contains the identifier of the deleted cm handle. "event": { "cmHandleId" : "cmhandle-001", - }
\ No newline at end of file + } diff --git a/docs/cps-events.rst b/docs/cps-events.rst index 47aa73f12e..1097af9a12 100644 --- a/docs/cps-events.rst +++ b/docs/cps-events.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2022-2023 Nordix Foundation +.. Copyright (C) 2022-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cpsEvents: @@ -16,9 +16,9 @@ CPS Events cm-notification-subscriptions.rst .. note:: - Legacy async response on a client supplied topic for single cm handle data request are no longer supported. Click link below for the legacy specification. + Legacy async response on a client supplied topic for single CM Handle data request are no longer supported. Click link below for the legacy specification. .. toctree:: :maxdepth: 0 - ncmp-async-events.rst
\ No newline at end of file + ncmp-async-events.rst diff --git a/docs/cps-ncmp-message-status-codes.rst b/docs/cps-ncmp-message-status-codes.rst index e0a3f0308b..799838ae44 100644 --- a/docs/cps-ncmp-message-status-codes.rst +++ b/docs/cps-ncmp-message-status-codes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023-2024 Nordix Foundation +.. Copyright (C) 2023-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _dataOperationMessageStatusCodes: @@ -16,9 +16,9 @@ CPS-NCMP Message Status Codes +-----------------+------------------------------------------------------+-----------------------------------+ | 1 | ACCEPTED | CM Data Notification Subscription | +-----------------+------------------------------------------------------+-----------------------------------+ - | 100 | cm handle id(s) is(are) not found | All features | + | 100 | CM Handle id(s) is(are) not found | All features | +-----------------+------------------------------------------------------+-----------------------------------+ - | 101 | cm handle(s) not ready | Data Operation | + | 101 | CM Handle(s) not ready | Data Operation | +-----------------+------------------------------------------------------+-----------------------------------+ | 102 | dmi plugin service is not responding | Data Operation | +-----------------+------------------------------------------------------+-----------------------------------+ @@ -30,9 +30,9 @@ CPS-NCMP Message Status Codes +-----------------+------------------------------------------------------+-----------------------------------+ | 108 | Unknown error | All features | +-----------------+------------------------------------------------------+-----------------------------------+ - | 109 | cm-handle already exists | Inventory | + | 109 | CM Handle already exists | Inventory | +-----------------+------------------------------------------------------+-----------------------------------+ - | 110 | cm-handle has an invalid character(s) in id | Inventory | + | 110 | CM Handle has an invalid character(s) in id | Inventory | +-----------------+------------------------------------------------------+-----------------------------------+ | 111 | alternate id already associated | Inventory | +-----------------+------------------------------------------------------+-----------------------------------+ diff --git a/docs/cps-scheduled-processes.rst b/docs/cps-scheduled-processes.rst index c204e6ca0a..9af9a81c0c 100644 --- a/docs/cps-scheduled-processes.rst +++ b/docs/cps-scheduled-processes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2022 Nordix Foundation +.. Copyright (C) 2022-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING @@ -21,17 +21,17 @@ The following section is a list of the current scheduled processes running withi Module Sync ----------- The module sync is a user :ref:`configurable timed process<additional-cps-ncmp-customizations>`, -which is set to search for CM-Handles within CPS with an *'ADVISED'* state. -Once the CM-Handle is processed by the module sync, the CM-Handle state is then set to *'READY'*, if the process completes successfully. -If for any reason the module sync fails, the CM-Handle state will then be set to *'LOCKED'*, +which is set to search for CM Handles within CPS with an *'ADVISED'* state. +Once the CM Handle is processed by the module sync, the CM Handle state is then set to *'READY'*, if the process completes successfully. +If for any reason the module sync fails, the CM Handle state will then be set to *'LOCKED'*, and the reason for the lock will also be stored within CPS. -CM-Handles in the *'LOCKED'* state will be retried when the system has availability. CM-Handles in a *'LOCKED'* -state are processed by the retry mechanism, by setting CM-Handle state back to *'ADVISED'* so the next sync cycle will process those again. +CM Handles in the *'LOCKED'* state will be retried when the system has availability. CM Handles in a *'LOCKED'* +state are processed by the retry mechanism, by setting CM Handle state back to *'ADVISED'* so the next sync cycle will process those again. Data Sync --------- The data sync is a user :ref:`configurable timed process<additional-cps-ncmp-customizations>`, -which is set to search for CM-Handles with a sync state of *'UNSYNCHRONIZED'*. -Once the CM-Handle(s) with a sync state of *'UNSYNCHRONIZED'* is processed by the data sync, -the CM-Handle sync state is then set to *'SYNCHRONIZED'*, if the process completes successfully. -If the data sync fails, the CM-Handle sync state will remain as *'UNSYNCHRONIZED'*, and will be re-attempted. +which is set to search for CM Handles with a sync state of *'UNSYNCHRONIZED'*. +Once the CM Handle(s) with a sync state of *'UNSYNCHRONIZED'* is processed by the data sync, +the CM Handle sync state is then set to *'SYNCHRONIZED'*, if the process completes successfully. +If the data sync fails, the CM Handle sync state will remain as *'UNSYNCHRONIZED'*, and will be re-attempted. diff --git a/docs/deployment.rst b/docs/deployment.rst index 940bc50923..e17392a224 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -293,35 +293,74 @@ Any spring supported property can be configured by providing in ``config.additio Additional CPS-NCMP Customizations ================================== -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.dmiPluginUserName | User name used by cps-core to authenticate themselves for using ncmp-dmi-plugin service. | ``dmiuser`` | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.dmiPluginUserPassword | Internal password used by cps-core to connect to ncmp-dmi-plugin service. | Not defined | -| | | | -| | If not defined, the password is generated when deploying the application. | | -| | | | -| | See also :ref:`cps_common_credentials_retrieval`. | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.ncmp.timers | Specifies the delay in milliseconds in which the module sync watch dog will wake again after finishing. | ``5000`` | -| .advised-modules-sync.sleep-time-ms | | | -| | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.ncmp.timers | Specifies the delay in milliseconds in which the data sync watch dog will wake again after finishing. | ``30000`` | -| .cm-handle-data-sync.sleep-time-ms | | | -| | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the maximum time in seconds, to wait for establishing a connection for the HTTP Client. | ``30`` | -| .connectionTimeoutInSeconds | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the maximum number of connections allowed per route in the HTTP client. | ``50`` | -| .maximumConnectionsPerRoute | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the maximum total number of connections that can be held by the HTTP client. | ``100`` | -| .maximumConnectionsTotal | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the duration in seconds for the threshold, after which idle connections will be evicted | ``5`` | -| .idleConnectionEvictionThresholdInSeconds | from the connection pool by the HTTP client. | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ + ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| Property | Description | Default Value | ++=================================================+=========================================================================================================+===============+ +| config.dmiPluginUserName | User name used by cps-core to authenticate themselves for using ncmp-dmi-plugin service. | ``dmiuser`` | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.dmiPluginUserPassword | Internal password used by cps-core to connect to ncmp-dmi-plugin service. | Not defined | +| | | | +| | If not defined, the password is generated when deploying the application. | | +| | | | +| | See also :ref:`cps_common_credentials_retrieval`. | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.ncmp.timers | Specifies the delay in milliseconds in which the module sync watch dog will wake again after finishing. | ``5000`` | +| .advised-modules-sync.sleep-time-ms | | | +| | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.ncmp.timers | Specifies the delay in milliseconds in which the data sync watch dog will wake again after finishing. | ``30000`` | +| .cm-handle-data-sync.sleep-time-ms | | | +| | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.additional.ncmp | Maximum size (in MB) of the in-memory buffer for HTTP response data. | ``16`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .maximumInMemorySizeInMegabytes | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.additional.ncmp | Maximum number of simultaneous connections allowed in the connection pool. | ``100`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .maximumConnectionsTotal | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.additional.ncmp | Maximum number of pending requests when the connection pool is full. | ``50`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .pendingAcquireMaxCount | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.additional.ncmp | Specifies the maximum time in seconds, to wait for establishing a connection for the HTTP Client. | ``30`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .connectionTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.additional.ncmp | Timeout (in seconds) for reading data from the server after the connection is established. | ``30`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .readTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.additional.ncmp | Timeout (in seconds) for writing data to the server. | ``30`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .writeTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ +| config.additional.ncmp | Total timeout (in seconds) for receiving a complete response, including all processing stages. | ``60`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .responseTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------------------------+---------------+ + +.. note:: + + - [app] : can be 'policy-executor' or 'dmi'. + - [services] 'all-services' for 'policy-executor'. + - [services] 'data-services' and 'model-services' for 'dmi'. CPS-Core Docker Installation ============================ @@ -342,13 +381,13 @@ Below are the list of distributed datastructures that we have. +--------------+------------------------------------+-----------------------------------------------------------+ | Component | Data Structure Name | Use | +==============+====================================+===========================================================+ -| cps-ncmp | moduleSyncStartedOnCmHandles | Watchdog process to register cm handles. | +| cps-ncmp | moduleSyncStartedOnCmHandles | Watchdog process to register CM Handles. | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | dataSyncSemaphores | Watchdog process to sync data from the nodes. | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | moduleSyncWorkQueue | Queue used internally for workers to pick the task. | +--------------+------------------------------------+-----------------------------------------------------------+ -| cps-ncmp | trustLevelPerCmHandle | Stores the trust level per cm handle id | +| cps-ncmp | trustLevelPerCmHandle | Stores the trust level per CM Handle id | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | trustLevelPerDmiPlugin | Stores the trust level for the dmi-plugins. | +--------------+------------------------------------+-----------------------------------------------------------+ diff --git a/docs/modeling.rst b/docs/modeling.rst index 7ebf6fecd5..65e4aa97ca 100644 --- a/docs/modeling.rst +++ b/docs/modeling.rst @@ -1,7 +1,7 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 .. Copyright (C) 2021 Pantheon.tech -.. Modifications Copyright (C) 2021-2023 Nordix Foundation +.. Modifications Copyright (C) 2021-2025 Nordix Foundation .. _modeling: .. toctree:: @@ -89,31 +89,31 @@ Note: Although additional-properties are present in the model of the dmi-registr Basic Concepts -------------- -- **CM-Handle** represents an instance a modeled Network Function(node) in ONAP. +- **CM Handle** represents an instance a modeled Network Function(node) in ONAP. These are stored as Anchors within CPS-Core. - - **CM-Handle States** are used to represent the potential states in which a CM-Handle can transition between. + - **CM Handle States** are used to represent the potential states in which a CM Handle can transition between. - The 5 possible CM-Handle states are: ADVISED, READY, LOCKED, DELETING, DELETED + The 5 possible CM Handle states are: ADVISED, READY, LOCKED, DELETING, DELETED - **ADVISED** indicates that a CM-Handle has been registered successfully, and is waiting for the module synchronization process to sync the CM-Handle. + **ADVISED** indicates that a CM Handle has been registered successfully, and is waiting for the module synchronization process to sync the CM Handle. - **READY** indicates that the CM-Handle has been synced successfully. + **READY** indicates that the CM Handle has been synced successfully. - **LOCKED** indicates that the CM-Handle has not synced successfully. A retry mechanism within CPS will set the state back to ADVISED after a set time. + **LOCKED** indicates that the CM Handle has not synced successfully. A retry mechanism within CPS will set the state back to ADVISED after a set time. - **DELETING** indicates that the CM-Handle is currently being deleted. + **DELETING** indicates that the CM Handle is currently being deleted. - **DELETED** indicates that the CM-Handle has been deleted successfully. + **DELETED** indicates that the CM Handle has been deleted successfully. - - **Data-sync state** is the state of the data synchronization process of the CM-Handle + - **Data-sync state** is the state of the data synchronization process of the CM Handle There are 3 possibles states: NONE_REQUESTED, UNSYNCHRONIZED, SYNCHRONIZED **NONE_REQUESTED** indicates that the data sync is not requested by the user - **UNSYNCHRONIZED** indicates the cm-handle is waiting for the data sync watchdog operation to carry out the sync process + **UNSYNCHRONIZED** indicates the CM Handle is waiting for the data sync watchdog operation to carry out the sync process **SYNCHRONIZED** indicates the watchdog process has finished the data synchronization successfully diff --git a/docs/ncmp-data-operation.rst b/docs/ncmp-data-operation.rst index 10c3bfaca5..e0b7bb88cd 100644 --- a/docs/ncmp-data-operation.rst +++ b/docs/ncmp-data-operation.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023-2024 Nordix Foundation +.. Copyright (C) 2023-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cmHandleDataOperation: @@ -15,7 +15,7 @@ Data Operations Endpoint Introduction ============ -For all data operations on cm handle(s), we have a post endpoint: +For all data operations on CM Handle(s), we have a post endpoint: - /ncmp/v1/data?topic={client-topic-name} forward request to it's dmi plugin service. @@ -50,7 +50,7 @@ This endpoint executes data operation for given array of operations: | | | implementation. For ONAP DMI Plugin it will be RESTConf paths but it can| | | | really be anything. | +--------------------------+-------------+-------------------------------------------------------------------------+ - | targetIds | Yes | List of cm handle references | + | targetIds | Yes | List of CM Handle references | +--------------------------+-------------+-------------------------------------------------------------------------+ The status codes used in the events resulting from these operations are defined here: @@ -156,4 +156,4 @@ DMI Service 2 (POST) : `http://{dmi-host-name}:{dmi-port}/dmi/v1/data?topic=my-t Above examples are for illustration purposes only. Please refer to link below for latest schema. -:download:`Data operation event schema <schemas/data-operation-event-schema-1.0.0.json>`
\ No newline at end of file +:download:`Data operation event schema <schemas/data-operation-event-schema-1.0.0.json>` diff --git a/docs/overview.rst b/docs/overview.rst index 19ab8b4847..bc99214e2e 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -1,6 +1,8 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021 Pantheon.tech, Nordix Foundation +.. Copyright (C) 2021 Pantheon.tech +.. Modifications Copyright (C) 2021-2025 Nordix Foundation + .. _overview: CPS Overview @@ -45,7 +47,7 @@ even though CPS-Core could be deployed without the NCMP extension. NCMP-DMI-Plugin --------------- -The Data-Model-Inventory (DMI) Plugin is a rest interface used to synchronize CM-Handles data between CPS and DMI through the DMI-Plugin. +The Data-Model-Inventory (DMI) Plugin is a rest interface used to synchronize CM Handles data between CPS and DMI through the DMI-Plugin. This is built previously from the CPS-NF-Proxy component. CPS Project diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 9c825e4d35..76d75cdec5 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021-2024 Nordix Foundation +.. Copyright (C) 2021-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _release_notes: @@ -71,7 +71,7 @@ Bug Fixes 3.5.5 - `CPS-2509 <https://lf-onap.atlassian.net/browse/CPS-2509>`_ Fix module endpoints using alternate identifier. - `CPS-2517 <https://lf-onap.atlassian.net/browse/CPS-2517>`_ Make Content-Type header default to JSON for CPS APIs. - - `CPS-2530 <https://lf-onap.atlassian.net/browse/CPS-2530>`_ NCMP Modules API giving empty response on READY cm handles if two sub systems discovered in parallel. + - `CPS-2530 <https://lf-onap.atlassian.net/browse/CPS-2530>`_ NCMP Modules API giving empty response on READY CM Handles if two sub systems discovered in parallel. Features -------- @@ -82,7 +82,7 @@ Features - `CPS-2436 <https://lf-onap.atlassian.net/browse/CPS-2436>`_ CM Avc Event to publish source key to target key while forwarding. - `CPS-2445 <https://lf-onap.atlassian.net/browse/CPS-2445>`_ Expose CPS and NCMP version information using git plugin. - `CPS-2451 <https://lf-onap.atlassian.net/browse/CPS-2451>`_ Removing oparent from CPS-NCMP and ONAP DMI Plugin repository. - - `CPS-2478 <https://lf-onap.atlassian.net/browse/CPS-2478>`_ Optimized Cm Handle Registration and De-Registration use case. + - `CPS-2478 <https://lf-onap.atlassian.net/browse/CPS-2478>`_ Optimized CM Handle Registration and De-Registration use case. - `CPS-2507 <https://lf-onap.atlassian.net/browse/CPS-2507>`_ Upgrade liquibase to 4.30.0 version. Performance @@ -112,7 +112,7 @@ Release Data Bug Fixes --------- 3.5.4 - - `CPS-2403 <https://lf-onap.atlassian.net/browse/CPS-2403>`_ Improve lock handling and queue management during CM-handle Module Sync. + - `CPS-2403 <https://lf-onap.atlassian.net/browse/CPS-2403>`_ Improve lock handling and queue management during CM Handle Module Sync. Features -------- @@ -220,7 +220,7 @@ Features -------- 3.5.1 - `CPS-2121 <https://lf-onap.atlassian.net/browse/CPS-2121>`_ Enabled http client prometheus metrics and manage high cardinality using URL template. - - `CPS-2289 <https://lf-onap.atlassian.net/browse/CPS-2289>`_ Support for CPS Path Query in NCMP Inventory Cm Handle Search. + - `CPS-2289 <https://lf-onap.atlassian.net/browse/CPS-2289>`_ Support for CPS Path Query in NCMP Inventory CM Handle Search. Version: 3.5.0 ============== @@ -530,7 +530,7 @@ Known Limitations, Issues and Workarounds For upgrading, CPS uses Liquibase for database upgrades. In order to enable Hibernate write batching (`CPS-1795 <https://lf-onap.atlassian.net/browse/CPS-1795>`_), a change to the database entity ID generation is required. As such, *this release does not fully support In-Service Software Upgrade* - CPS will not store new DataNodes and -NCMP will not register new CM-handles during an upgrade with old and new versions of CPS running concurrently. +NCMP will not register new CM Handles during an upgrade with old and new versions of CPS running concurrently. Other operations (read, update, delete) are not impacted. @@ -805,7 +805,7 @@ Bug Fixes Features -------- - - `CPS-1515 <https://lf-onap.atlassian.net/browse/CPS-1515>`_ Support Multiple CM-Handles for NCMP Get Operation + - `CPS-1515 <https://lf-onap.atlassian.net/browse/CPS-1515>`_ Support Multiple CM Handles for NCMP Get Operation - `CPS-1675 <https://lf-onap.atlassian.net/browse/CPS-1675>`_ Persistence write performance improvement(s) - `CPS-1745 <https://lf-onap.atlassian.net/browse/CPS-1745>`_ Upgrade to Openapi 3.0.3 @@ -1265,7 +1265,7 @@ Features - `CPS-869 <https://lf-onap.atlassian.net/browse/CPS-869>`_ Apply Standardized logging fields to adhere to ONAP Best practice REQ-1072 - `CPS-870 <https://lf-onap.atlassian.net/browse/CPS-870>`_ Align CPS-Core output with SDN-C output (add module name) - `CPS-875 <https://lf-onap.atlassian.net/browse/CPS-875>`_ CM Handle State: Watchdog-process that syncs 'ADVISED' CM Handles - - `CPS-877 <https://lf-onap.atlassian.net/browse/CPS-877>`_ CM Handle State: Exclude any CM-Handles from queries/operations that are not in state 'READY' + - `CPS-877 <https://lf-onap.atlassian.net/browse/CPS-877>`_ CM Handle State: Exclude any CM Handles from queries/operations that are not in state 'READY' - `CPS-899 <https://lf-onap.atlassian.net/browse/CPS-899>`_ Start and stop sessions on Java API - `CPS-909 <https://lf-onap.atlassian.net/browse/CPS-909>`_ Separate NCMP endpoint for ch/{cm-handle}/properties and ch/{cm-handle}/state - `CPS-917 <https://lf-onap.atlassian.net/browse/CPS-917>`_ Structured Errors response for passthrough use-cases in NCMP @@ -1278,7 +1278,7 @@ Features - `CPS-1099 <https://lf-onap.atlassian.net/browse/CPS-1099>`_ Expose simplified 'external' lock reason enum state over REST interface - `CPS-1101 <https://lf-onap.atlassian.net/browse/CPS-1101>`_ Introducing the DELETING and DELETED Cmhandle State - `CPS-1102 <https://lf-onap.atlassian.net/browse/CPS-1102>`_ Register the Cmhandle Sends Advised State notification. - - `CPS-1133 <https://lf-onap.atlassian.net/browse/CPS-1133>`_ Enable/Disable Data Sync for Cm Handle + - `CPS-1133 <https://lf-onap.atlassian.net/browse/CPS-1133>`_ Enable/Disable Data Sync for CM Handle - `CPS-1136 <https://lf-onap.atlassian.net/browse/CPS-1136>`_ DMI Audit Support (get all CM Handles for a registered DMI) @@ -1379,27 +1379,27 @@ Features - `CPS-559 <https://lf-onap.atlassian.net/browse/CPS-559>`_ Define response objects (schemas) in cps-ncmp - `CPS-636 <https://lf-onap.atlassian.net/browse/CPS-636>`_ Update operation for datastore pass through running - `CPS-638 <https://lf-onap.atlassian.net/browse/CPS-638>`_ Delete operation for datastore pass through running - - `CPS-677 <https://lf-onap.atlassian.net/browse/CPS-677>`_ Support 'public' Cm Handle Properties - - `CPS-741 <https://lf-onap.atlassian.net/browse/CPS-741>`_ Re sync after removing cm handles + - `CPS-677 <https://lf-onap.atlassian.net/browse/CPS-677>`_ Support 'public' CM Handle Properties + - `CPS-741 <https://lf-onap.atlassian.net/browse/CPS-741>`_ Re sync after removing CM Handles - `CPS-777 <https://lf-onap.atlassian.net/browse/CPS-777>`_ Ensure all DMI operations use POST method - `CPS-780 <https://lf-onap.atlassian.net/browse/CPS-780>`_ Add examples for parameters, request and response in openapi yaml for cps-core - `CPS-789 <https://lf-onap.atlassian.net/browse/CPS-789>`_ CPS Data Updated Event Schema V2 to support delete operation - `CPS-791 <https://lf-onap.atlassian.net/browse/CPS-791>`_ CPS-Core sends delete notification event - - `CPS-817 <https://lf-onap.atlassian.net/browse/CPS-817>`_ Create Endpoint For Get Cm Handles (incl. public properties) By Name + - `CPS-817 <https://lf-onap.atlassian.net/browse/CPS-817>`_ Create Endpoint For Get CM Handles (incl. public properties) By Name - `CPS-837 <https://lf-onap.atlassian.net/browse/CPS-837>`_ Add Remove and Update properties (DMI and Public) as part of CM Handle Registration update Bug Fixes --------- - - `CPS-762 <https://lf-onap.atlassian.net/browse/CPS-762>`_ Query cm handles for module names returns incorrect cm handle identifiers + - `CPS-762 <https://lf-onap.atlassian.net/browse/CPS-762>`_ Query CM Handles for module names returns incorrect CM Handle identifiers - `CPS-788 <https://lf-onap.atlassian.net/browse/CPS-788>`_ Yang Resource formatting is incorrect - - `CPS-783 <https://lf-onap.atlassian.net/browse/CPS-783>`_ Remove cm handle does not completely remove all cm handle information + - `CPS-783 <https://lf-onap.atlassian.net/browse/CPS-783>`_ Remove CM Handle does not completely remove all CM Handle information - `CPS-841 <https://lf-onap.atlassian.net/browse/CPS-841>`_ Upgrade log4j to 2.17.1 as recommended by ONAP SECCOM - `CPS-856 <https://lf-onap.atlassian.net/browse/CPS-856>`_ Retry mechanism not working for concurrent CmHandle registration - `CPS-867 <https://lf-onap.atlassian.net/browse/CPS-867>`_ Database port made configurable through env variable DB_PORT - `CPS-886 <https://lf-onap.atlassian.net/browse/CPS-886>`_ Fragment handling decreasing performance for large number of cmHandles - `CPS-887 <https://lf-onap.atlassian.net/browse/CPS-887>`_ Increase performance of cmHandle registration for large number of schema sets in DB - - `CPS-892 <https://lf-onap.atlassian.net/browse/CPS-892>`_ Fixed the response code during CM-Handle Registration from 201 CREATED to 204 NO_CONTENT + - `CPS-892 <https://lf-onap.atlassian.net/browse/CPS-892>`_ Fixed the response code during CM Handle Registration from 201 CREATED to 204 NO_CONTENT - `CPS-893 <https://lf-onap.atlassian.net/browse/CPS-893>`_ NCMP Java API depends on NCMP-Rest-API (cyclic) through json properties on Java API Known Limitations, Issues and Workarounds @@ -1407,9 +1407,9 @@ Known Limitations, Issues and Workarounds *System Limitations* -Null can no longer be passed within the dmi plugin service names when registering a cm handle, as part of +Null can no longer be passed within the dmi plugin service names when registering a CM Handle, as part of `CPS-837 <https://lf-onap.atlassian.net/browse/CPS-837>`_ null is now used to indicate if a property should be removed as part -of cm handle registration. +of CM Handle registration. The Absolute path to list with integer key will not work. Please refer `CPS-961 <https://lf-onap.atlassian.net/browse/CPS-961>`_ for more information. diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy index f3cca801e7..75cb3cd7a2 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy @@ -158,11 +158,12 @@ abstract class CpsIntegrationSpecBase extends Specification { static initialized = false def now = OffsetDateTime.now() + enum ModuleNameStrategy { UNIQUE, OVERLAPPING } + def setup() { if (!initialized) { cpsDataspaceService.createDataspace(GENERAL_TEST_DATASPACE) createStandardBookStoreSchemaSet(GENERAL_TEST_DATASPACE) - cpsAnchorService.createAnchor(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET, 'owner-of-bookstore-schema-set-do-not-delete') initialized = true } mockDmiServer1.setDispatcher(dmiDispatcher1) @@ -182,7 +183,7 @@ abstract class CpsIntegrationSpecBase extends Specification { mockDmiServer1.shutdown() mockDmiServer2.shutdown() mockPolicyServer.shutdown() - cpsModuleService.deleteAllUnusedYangModuleData() + cpsModuleService.deleteAllUnusedYangModuleData('NFP-Operational') } def static readResourceDataFile(filename) { @@ -265,9 +266,14 @@ abstract class CpsIntegrationSpecBase extends Specification { } def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset) { + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset, ModuleNameStrategy.UNIQUE) + } + + def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset, ModuleNameStrategy moduleNameStrategy ) { def cmHandles = [] def id = offset - def moduleReferences = (1..200).collect { "${moduleSetTag}Module${it}" } + def modulePrefix = moduleNameStrategy.OVERLAPPING.equals(moduleNameStrategy) ? 'same' : moduleSetTag + def moduleReferences = (1..200).collect { "${modulePrefix}Module${it}" } (1..numberOfCmHandles).each { def ncmpServiceCmHandle = new NcmpServiceCmHandle(cmHandleId: "ch-${id}", moduleSetTag: moduleSetTag, alternateId: NO_ALTERNATE_ID) cmHandles.add(ncmpServiceCmHandle) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy index 2bd5a4a1be..ca321119ea 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy @@ -59,17 +59,17 @@ class AnchorServiceIntegrationSpec extends FunctionalSpecBase { and: '1 anchor with "other" schema set is created' createStandardBookStoreSchemaSet(GENERAL_TEST_DATASPACE, 'otherSchemaSet') objectUnderTest.createAnchor(GENERAL_TEST_DATASPACE, 'otherSchemaSet', 'anchor3') - then: 'there are 4 anchors in the general test database' - assert objectUnderTest.getAnchors(GENERAL_TEST_DATASPACE).size() == 4 - and: 'there are 3 anchors associated with bookstore schema set' - assert objectUnderTest.getAnchorsBySchemaSetName(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET).size() == 3 + then: 'there are 3 anchors in the general test database' + assert objectUnderTest.getAnchors(GENERAL_TEST_DATASPACE).size() == 3 + and: 'there are 2 anchors associated with bookstore schema set' + assert objectUnderTest.getAnchorsBySchemaSetName(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET).size() == 2 and: 'there is 1 anchor associated with other schema set' assert objectUnderTest.getAnchorsBySchemaSetName(GENERAL_TEST_DATASPACE, 'otherSchemaSet').size() == 1 } def 'Querying anchor(name)s (depends on previous test!).'() { - expect: 'there are now 4 anchors using the "stores" module (both schema sets use the same modules) ' - assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'bookstore-types']).size() == 4 + expect: 'there are now 3 anchors using the "stores" module (both schema sets use the same modules)' + assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'bookstore-types']).size() == 3 and: 'there are no anchors using both "stores" and a "unused-model"' assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'unused-model']).size() == 0 } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy index 178b0227ca..47a332adc9 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -32,8 +32,6 @@ class DataspaceServiceIntegrationSpec extends FunctionalSpecBase { def setup() { objectUnderTest = cpsDataspaceService } - def cleanup() { cpsModuleService.deleteAllUnusedYangModuleData() } - def 'Dataspace CRUD operations.'() { when: 'a dataspace is created' objectUnderTest.createDataspace('newDataspace') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy index d8010875c1..c787b4209e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy @@ -61,8 +61,6 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def setup() { objectUnderTest = cpsModuleService } - def cleanup() { objectUnderTest.deleteAllUnusedYangModuleData() } - /* C R E A T E S C H E M A S E T U S E - C A S E S */ diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy index 28714fd123..097a043556 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy @@ -79,7 +79,6 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { cleanup: 'deregister CM-handle and remove all associated module resources' deregisterCmHandle(DMI1_URL, cmHandleId) - cpsModuleService.deleteAllUnusedYangModuleData() where: 'following module set tags are used' initialModuleSetTag | updatedModuleSetTag diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy index 43db9b208e..9cb8c29e21 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy @@ -20,17 +20,18 @@ package org.onap.cps.integration.functional.ncmp +import com.hazelcast.map.IMap import io.micrometer.core.instrument.MeterRegistry -import spock.lang.Ignore - -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.springframework.beans.factory.annotation.Autowired import org.springframework.util.StopWatch +import spock.lang.Ignore import spock.util.concurrent.PollingConditions +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { ModuleSyncWatchdog objectUnderTest @@ -38,11 +39,15 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { @Autowired MeterRegistry meterRegistry + @Autowired + IMap<String, Integer> cmHandlesByState + def executorService = Executors.newFixedThreadPool(2) def PARALLEL_SYNC_SAMPLE_SIZE = 100 def setup() { objectUnderTest = moduleSyncWatchdog + clearCmHandleStateGauge() } def cleanup() { @@ -64,11 +69,10 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) } - @Ignore /** this test has intermittent failures, due to timeouts. * Ignored but left here as it might be valuable to further optimization investigations. **/ - + @Ignore def 'CPS-2478 Highlight (and improve) module sync inefficiencies.'() { given: 'register 250 cm handles with module set tag cps-2478-A' def numberOfTags = 2 @@ -110,7 +114,6 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { def stopWatch = new StopWatch() stopWatch.start() deregisterSequenceOfCmHandles(DMI1_URL, totalCmHandles, 1) - cpsModuleService.deleteAllUnusedYangModuleData() stopWatch.stop() println "*** CPS-2478, Deletion of $totalCmHandles cm handles took ${stopWatch.getTotalTimeMillis()} milliseconds" } @@ -131,6 +134,26 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) } + def 'Schema sets with overlapping modules processed at the same time (DB constraint violation).'() { + given: 'register one batch (100) cm handles of tag A (with overlapping module names)' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'tagA', 100, 1, ModuleNameStrategy.OVERLAPPING) + and: 'register another batch cm handles of tag B (with overlapping module names)' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'tagB', 100, 101, ModuleNameStrategy.OVERLAPPING) + and: 'populate the work queue with both batches' + objectUnderTest.populateWorkQueueIfNeeded() + when: 'advised cm handles are processed on 2 threads (exactly one batch for each)' + objectUnderTest.moduleSyncAdvisedCmHandles() + executorService.execute(moduleSyncAdvisedCmHandles) + then: 'wait till all cm handles have been processed' + new PollingConditions().within(10, () -> { + assert getNumberOfProcessedCmHandles() == 200 + }) + then: 'at least 1 cm handle is in state LOCKED' + assert cmHandlesByState.get('lockedCmHandlesCount') >= 1 + cleanup: 'remove all test cm handles' + deregisterSequenceOfCmHandles(DMI1_URL, 200, 1) + } + def 'Populate module sync work queue on two parallel threads with a slight difference in start time.'() { // This test proved that the issue in CPS-2403 did not arise if the the queue was populated and given time to be distributed given: 'the queue is empty at the start' @@ -169,4 +192,21 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { } } + def moduleSyncAdvisedCmHandles = () -> { + try { + objectUnderTest.moduleSyncAdvisedCmHandles() + } catch (InterruptedException e) { + e.printStackTrace() + } + } + + def clearCmHandleStateGauge() { + cmHandlesByState.keySet().each { cmHandlesByState.put(it, 0)} + } + + def getNumberOfProcessedCmHandles() { + return cmHandlesByState.get('readyCmHandlesCount') + cmHandlesByState.get('lockedCmHandlesCount') + } + + } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy index 613f760b0c..e52d3f819c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -100,7 +100,7 @@ class ModuleQueryPerfTest extends CpsPerfTestBase { cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, (i..i+100).collect {SCHEMA_SET_PREFIX + it}) } cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, [SCHEMA_SET_PREFIX + '0']) - cpsModuleService.deleteAllUnusedYangModuleData() + cpsModuleService.deleteAllUnusedYangModuleData(CPS_PERFORMANCE_TEST_DATASPACE) } // This makes a Yang module of approximately target length in bytes by padding the description field with many '*' |