diff options
10 files changed, 119 insertions, 16 deletions
diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java index 3368aee148..e5853abf39 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java @@ -230,8 +230,6 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ updateAllModuleReferences(allModuleReferences, schemaSetEntity.getId()); } - - @Override @Transactional public void deleteUnusedYangResourceModules() { diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java index bbfe496d85..304ed288f5 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java @@ -188,4 +188,9 @@ public interface CpsModuleService { final Map<String, String> parentAttributes, final Map<String, String> childAttributes); + /** + * Remove any Yang Resource Modules from the DB that are no longer referenced by any schema set. + */ + void deleteUnusedYangResourceModules(); + } diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java index 4063a7f769..9f3f2cc571 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java @@ -106,7 +106,6 @@ public class CpsModuleServiceImpl implements CpsModuleService { } cpsModulePersistenceService.deleteSchemaSet(dataspaceName, schemaSetName); yangTextSchemaSourceSetCache.removeFromCache(dataspaceName, schemaSetName); - cpsModulePersistenceService.deleteUnusedYangResourceModules(); } @Override @@ -119,7 +118,6 @@ public class CpsModuleServiceImpl implements CpsModuleService { .stream().map(Anchor::getName).collect(Collectors.toSet()); cpsAnchorService.deleteAnchors(dataspaceName, anchorNames); cpsModulePersistenceService.deleteSchemaSets(dataspaceName, schemaSetNames); - cpsModulePersistenceService.deleteUnusedYangResourceModules(); for (final String schemaSetName : schemaSetNames) { yangTextSchemaSourceSetCache.removeFromCache(dataspaceName, schemaSetName); } @@ -182,6 +180,11 @@ public class CpsModuleServiceImpl implements CpsModuleService { childAttributes); } + @Override + public void deleteUnusedYangResourceModules() { + cpsModulePersistenceService.deleteUnusedYangResourceModules(); + } + private boolean isCascadeDeleteProhibited(final CascadeDeleteAllowed cascadeDeleteAllowed) { return CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED == cascadeDeleteAllowed; } diff --git a/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java b/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java new file mode 100644 index 0000000000..6bd3e1f204 --- /dev/null +++ b/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java @@ -0,0 +1,48 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.init; + +import java.util.concurrent.TimeUnit; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.onap.cps.api.CpsModuleService; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Slf4j +@RequiredArgsConstructor +@Service +public class DbCleaner { + + private final CpsModuleService cpsModuleService; + + /** + * This method will clean up the db during application start up. + * It wil run once and currently only removes unused yang resource modules. + * + */ + @Scheduled(initialDelay = 1, timeUnit = TimeUnit.SECONDS) + public void cleanDbAtStartUp() { + log.info("CPS Application started, commencing DB clean up"); + cpsModuleService.deleteUnusedYangResourceModules(); + log.info("DB clean up completed"); + } +} diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy index 1831506563..c02b06fd80 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy @@ -142,8 +142,6 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsModulePersistenceService.deleteSchemaSet('my-dataspace', 'my-schemaset') and: 'schema set will be removed from the cache' 1 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', 'my-schemaset') - and: 'orphan yang resources are deleted' - 1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules() and: 'the CpsValidator is called on the dataspaceName and schemaSetName' 1 * mockCpsValidator.validateNameCharacters('my-dataspace', _) where: 'following parameters are used' @@ -161,8 +159,6 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsModulePersistenceService.deleteSchemaSet('my-dataspace', 'my-schemaset') and: 'schema set will be removed from the cache' 1 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', 'my-schemaset') - and: 'orphan yang resources are deleted' - 1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules() and: 'the CpsValidator is called on the dataspaceName and schemaSetName' 1 * mockCpsValidator.validateNameCharacters('my-dataspace', 'my-schemaset') } @@ -187,8 +183,6 @@ class CpsModuleServiceImplSpec extends Specification { mockCpsModulePersistenceService.deleteSchemaSets('my-dataspace', _) and: 'schema sets will be removed from the cache' 2 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', _) - and: 'orphan yang resources are deleted' - 1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules() and: 'the CpsValidator is called on the dataspaceName' 1 * mockCpsValidator.validateNameCharacters('my-dataspace') and: 'the CpsValidator is called on the schemaSetNames' @@ -276,6 +270,13 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsValidator.validateNameCharacters('some-dataspace-name', 'some-anchor-name') } + def 'Delete unused yang resource modules.'() { + when: 'deleting unused yang resource modules' + objectUnderTest.deleteUnusedYangResourceModules() + then: 'it is delegated to the module persistence service' + 1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules() + } + def getModuleReferences() { return [new ModuleReference('some module name','some revision name')] } diff --git a/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy new file mode 100644 index 0000000000..5106d29fa5 --- /dev/null +++ b/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy @@ -0,0 +1,38 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.init + +import org.onap.cps.api.CpsModuleService +import spock.lang.Specification + +class DbCleanerSpec extends Specification { + + def mockCpsModuleService = Mock(CpsModuleService) + + def objectUnderTest = new DbCleaner(mockCpsModuleService) + + def 'DB clean up.'() { + when: 'scheduled method is triggered' + objectUnderTest.cleanDbAtStartUp() + then: 'the unused yang resource modules are deleted' + 1 * mockCpsModuleService.deleteUnusedYangResourceModules() + } +} diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy index d69f6cca0c..f20e4e5cad 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy @@ -32,6 +32,8 @@ class DataspaceServiceIntegrationSpec extends FunctionalSpecBase { def setup() { objectUnderTest = cpsDataspaceService } + def cleanup() { cpsModuleService.deleteUnusedYangResourceModules() } + def 'Dataspace CRUD operations.'() { when: 'a dataspace is created' objectUnderTest.createDataspace('newDataspace') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy index 9e51d80d9e..a50a59a3d7 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy @@ -59,9 +59,9 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def bookstoreModelFileContent = readResourceDataFile('bookstore/bookstore.yang') def bookstoreTypesFileContent = readResourceDataFile('bookstore/bookstore-types.yang') - def setup() { - objectUnderTest = cpsModuleService - } + def setup() { objectUnderTest = cpsModuleService } + + def cleanup() { objectUnderTest.deleteUnusedYangResourceModules() } /* C R E A T E S C H E M A S E T U S E - C A S E S @@ -77,7 +77,7 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { originalNumberOfModuleReferences + numberOfNewModules == yangResourceModuleReferences.size() cleanup: objectUnderTest.deleteSchemaSetsWithCascade(FUNCTIONAL_TEST_DATASPACE_1, [ 'newSchemaSet' ]) - where: 'the following parameters are use' + where: 'the following parameters are used' scenario | numberOfNewModules 'two valid new modules' | 2 'empty schema set' | 0 diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy index 20fa546eba..43bcbdb4f4 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy @@ -24,6 +24,7 @@ import io.micrometer.core.instrument.MeterRegistry import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.springframework.beans.factory.annotation.Autowired +import org.springframework.util.StopWatch import spock.util.concurrent.PollingConditions import java.util.concurrent.Executors @@ -61,12 +62,13 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { assert moduleSyncWorkQueue.isEmpty() } - def 'CPS-2478 Highlight module sync inefficiencies.'() { + def 'CPS-2478 Highlight (and improve) module sync inefficiencies.'() { given: 'register 250 cm handles with module set tag cps-2478-A' def numberOfTags = 2 def cmHandlesPerTag = 250 def totalCmHandles = numberOfTags * cmHandlesPerTag def offset = 1 + def minimumBatches = totalCmHandles / 100 registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-A', cmHandlesPerTag, offset) and: 'register anther 250 cm handles with module set tag cps-2478-B' offset += cmHandlesPerTag @@ -86,7 +88,7 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { then: 'wait till at least 5 batches of state updates are done (often more because of retries of locked cm handles)' def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer() new PollingConditions().within(10, () -> { - assert dbStateUpdateTimer.count() >= 5 + assert dbStateUpdateTimer.count() >= minimumBatches }) and: 'the db has been queried for tags exactly 2 times.' def dbModuleQueriesTimer = meterRegistry.get('cps.module.service.module.reference.query.by.attribute').timer() @@ -100,7 +102,12 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets ') logInstrumentation(dbStateUpdateTimer, 'batch state updates ') cleanup: 'remove all cm handles' + // To properly measure performance the sample-size should be increased to 20,000 cm handles or higher (10,000 per tag) + def stopWatch = new StopWatch() + stopWatch.start() deregisterSequenceOfCmHandles(DMI1_URL, totalCmHandles, 1) + stopWatch.stop() + println "*** CPS-2478, Deletion of $totalCmHandles cm handles took ${stopWatch.getTotalTimeMillis()} milliseconds" } def 'Populate module sync work queue simultaneously on two parallel threads (CPS-2403).'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy index add931a1ad..914f2030cc 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy @@ -100,6 +100,7 @@ class ModuleQueryPerfTest extends CpsPerfTestBase { cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, (i..i+100).collect {SCHEMA_SET_PREFIX + it}) } cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, [SCHEMA_SET_PREFIX + '0']) + cpsModuleService.deleteUnusedYangResourceModules() } // This makes a Yang module of approximately target length in bytes by padding the description field with many '*' |