De-Registration without Orphaned Module Check

- Testware updates to  measure time spent for de-registration
- Removed orphan check from module delete methods
- Using @scheduled to run Db Cleaner method once
(see https://lf-onap.atlassian.net/wiki/spaces/DW/pages/52494359/CPS-2478+Module+Sync+Inefficiencies#De-Registration:-Test-Measurements-With-and-Without-Orphanage-removal for reasoning)
- Updated integration tests to call orphan check where need (after schema set deletion)

Issue-ID: CPS-2478
Change-Id: I513af9d8bb88486a242284b58e0363a527346dd4
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

10 files changed, 119 insertions, 16 deletions

diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java
index 3368aee148..e5853abf39 100755
--- a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java
+++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java
@@ -230,8 +230,6 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ
         updateAllModuleReferences(allModuleReferences, schemaSetEntity.getId());
     }
 
-
-
     @Override
     @Transactional
     public void deleteUnusedYangResourceModules() {
diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java
index bbfe496d85..304ed288f5 100644
--- a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java
+++ b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java
@@ -188,4 +188,9 @@ public interface CpsModuleService {
                                                                final Map<String, String> parentAttributes,
                                                                final Map<String, String> childAttributes);
 
+    /**
+     * Remove any Yang Resource Modules from the DB that are no longer referenced by any schema set.
+     */
+    void deleteUnusedYangResourceModules();
+
 }
diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java
index 4063a7f769..9f3f2cc571 100644
--- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java
+++ b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java
@@ -106,7 +106,6 @@ public class CpsModuleServiceImpl implements CpsModuleService {
         }
         cpsModulePersistenceService.deleteSchemaSet(dataspaceName, schemaSetName);
         yangTextSchemaSourceSetCache.removeFromCache(dataspaceName, schemaSetName);
-        cpsModulePersistenceService.deleteUnusedYangResourceModules();
     }
 
     @Override
@@ -119,7 +118,6 @@ public class CpsModuleServiceImpl implements CpsModuleService {
                         .stream().map(Anchor::getName).collect(Collectors.toSet());
         cpsAnchorService.deleteAnchors(dataspaceName, anchorNames);
         cpsModulePersistenceService.deleteSchemaSets(dataspaceName, schemaSetNames);
-        cpsModulePersistenceService.deleteUnusedYangResourceModules();
         for (final String schemaSetName : schemaSetNames) {
             yangTextSchemaSourceSetCache.removeFromCache(dataspaceName, schemaSetName);
         }
@@ -182,6 +180,11 @@ public class CpsModuleServiceImpl implements CpsModuleService {
                 childAttributes);
     }
 
+    @Override
+    public void deleteUnusedYangResourceModules() {
+        cpsModulePersistenceService.deleteUnusedYangResourceModules();
+    }
+
     private boolean isCascadeDeleteProhibited(final CascadeDeleteAllowed cascadeDeleteAllowed) {
         return CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED == cascadeDeleteAllowed;
     }
diff --git a/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java b/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java
new file mode 100644
index 0000000000..6bd3e1f204
--- /dev/null
+++ b/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java
@@ -0,0 +1,48 @@
+/*
+ *  ============LICENSE_START=======================================================
+ *  Copyright (C) 2024 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.init;
+
+import java.util.concurrent.TimeUnit;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.onap.cps.api.CpsModuleService;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@RequiredArgsConstructor
+@Service
+public class DbCleaner {
+
+    private final CpsModuleService cpsModuleService;
+
+    /**
+     * This method will clean up the db during application start up.
+     * It wil run once and currently only removes unused yang resource modules.
+     *
+     */
+    @Scheduled(initialDelay = 1, timeUnit = TimeUnit.SECONDS)
+    public void cleanDbAtStartUp() {
+        log.info("CPS Application started, commencing DB clean up");
+        cpsModuleService.deleteUnusedYangResourceModules();
+        log.info("DB clean up completed");
+    }
+}
diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy
index 1831506563..c02b06fd80 100644
--- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy
+++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy
@@ -142,8 +142,6 @@ class CpsModuleServiceImplSpec extends Specification {
             1 * mockCpsModulePersistenceService.deleteSchemaSet('my-dataspace', 'my-schemaset')
         and: 'schema set will be removed from the cache'
             1 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', 'my-schemaset')
-        and: 'orphan yang resources are deleted'
-            1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules()
         and: 'the CpsValidator is called on the dataspaceName and schemaSetName'
             1 * mockCpsValidator.validateNameCharacters('my-dataspace', _)
         where: 'following parameters are used'
@@ -161,8 +159,6 @@ class CpsModuleServiceImplSpec extends Specification {
             1 * mockCpsModulePersistenceService.deleteSchemaSet('my-dataspace', 'my-schemaset')
         and: 'schema set will be removed from the cache'
             1 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', 'my-schemaset')
-        and: 'orphan yang resources are deleted'
-            1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules()
         and: 'the CpsValidator is called on the dataspaceName and schemaSetName'
             1 * mockCpsValidator.validateNameCharacters('my-dataspace', 'my-schemaset')
     }
@@ -187,8 +183,6 @@ class CpsModuleServiceImplSpec extends Specification {
             mockCpsModulePersistenceService.deleteSchemaSets('my-dataspace', _)
         and: 'schema sets will be removed from the cache'
             2 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', _)
-        and: 'orphan yang resources are deleted'
-            1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules()
         and: 'the CpsValidator is called on the dataspaceName'
             1 * mockCpsValidator.validateNameCharacters('my-dataspace')
         and: 'the CpsValidator is called on the schemaSetNames'
@@ -276,6 +270,13 @@ class CpsModuleServiceImplSpec extends Specification {
             1 * mockCpsValidator.validateNameCharacters('some-dataspace-name', 'some-anchor-name')
     }
 
+    def 'Delete unused yang resource modules.'() {
+        when: 'deleting unused yang resource modules'
+            objectUnderTest.deleteUnusedYangResourceModules()
+        then: 'it is delegated to the module persistence service'
+            1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules()
+    }
+
     def getModuleReferences() {
         return [new ModuleReference('some module name','some revision name')]
     }
diff --git a/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy
new file mode 100644
index 0000000000..5106d29fa5
--- /dev/null
+++ b/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy
@@ -0,0 +1,38 @@
+/*
+ *  ============LICENSE_START=======================================================
+ *  Copyright (C) 2024 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.init
+
+import org.onap.cps.api.CpsModuleService
+import spock.lang.Specification
+
+class DbCleanerSpec extends Specification {
+
+    def mockCpsModuleService = Mock(CpsModuleService)
+
+    def objectUnderTest = new DbCleaner(mockCpsModuleService)
+
+    def 'DB clean up.'() {
+        when: 'scheduled method is triggered'
+            objectUnderTest.cleanDbAtStartUp()
+        then: 'the unused yang resource modules are deleted'
+            1 * mockCpsModuleService.deleteUnusedYangResourceModules()
+    }
+}
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy
index d69f6cca0c..f20e4e5cad 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy
@@ -32,6 +32,8 @@ class DataspaceServiceIntegrationSpec extends FunctionalSpecBase {
 
     def setup() { objectUnderTest = cpsDataspaceService }
 
+    def cleanup() { cpsModuleService.deleteUnusedYangResourceModules() }
+
     def 'Dataspace CRUD operations.'() {
         when: 'a dataspace is created'
             objectUnderTest.createDataspace('newDataspace')
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy
index 9e51d80d9e..a50a59a3d7 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy
@@ -59,9 +59,9 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase {
     def bookstoreModelFileContent = readResourceDataFile('bookstore/bookstore.yang')
     def bookstoreTypesFileContent = readResourceDataFile('bookstore/bookstore-types.yang')
 
-    def setup() {
-        objectUnderTest = cpsModuleService
-    }
+    def setup() { objectUnderTest = cpsModuleService }
+
+    def cleanup() { objectUnderTest.deleteUnusedYangResourceModules() }
 
     /*
         C R E A T E   S C H E M A   S E T   U S E - C A S E S
@@ -77,7 +77,7 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase {
             originalNumberOfModuleReferences + numberOfNewModules == yangResourceModuleReferences.size()
         cleanup:
             objectUnderTest.deleteSchemaSetsWithCascade(FUNCTIONAL_TEST_DATASPACE_1, [ 'newSchemaSet' ])
-        where: 'the following parameters are use'
+        where: 'the following parameters are used'
             scenario                       | numberOfNewModules
             'two valid new modules'        | 2
             'empty schema set'             | 0
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
index 20fa546eba..43bcbdb4f4 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
@@ -24,6 +24,7 @@ import io.micrometer.core.instrument.MeterRegistry
 import org.onap.cps.integration.base.CpsIntegrationSpecBase
 import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog
 import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.util.StopWatch
 import spock.util.concurrent.PollingConditions
 
 import java.util.concurrent.Executors
@@ -61,12 +62,13 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
             assert moduleSyncWorkQueue.isEmpty()
     }
 
-    def 'CPS-2478 Highlight module sync inefficiencies.'() {
+    def 'CPS-2478 Highlight (and improve) module sync inefficiencies.'() {
         given: 'register 250 cm handles with module set tag cps-2478-A'
             def numberOfTags = 2
             def cmHandlesPerTag = 250
             def totalCmHandles = numberOfTags * cmHandlesPerTag
             def offset = 1
+            def minimumBatches = totalCmHandles / 100
             registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-A', cmHandlesPerTag, offset)
         and: 'register anther 250 cm handles with module set tag cps-2478-B'
             offset += cmHandlesPerTag
@@ -86,7 +88,7 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
         then: 'wait till at least 5 batches of state updates are done (often more because of retries of locked cm handles)'
             def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer()
             new PollingConditions().within(10, () -> {
-                assert dbStateUpdateTimer.count() >= 5
+                assert dbStateUpdateTimer.count() >= minimumBatches
             })
         and: 'the db has been queried for tags exactly 2 times.'
             def dbModuleQueriesTimer = meterRegistry.get('cps.module.service.module.reference.query.by.attribute').timer()
@@ -100,7 +102,12 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
             logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets      ')
             logInstrumentation(dbStateUpdateTimer,      'batch state updates    ')
         cleanup: 'remove all cm handles'
+            // To properly measure performance the sample-size should be increased to 20,000 cm handles or higher (10,000 per tag)
+            def stopWatch = new StopWatch()
+            stopWatch.start()
             deregisterSequenceOfCmHandles(DMI1_URL, totalCmHandles, 1)
+            stopWatch.stop()
+            println "*** CPS-2478, Deletion of $totalCmHandles cm handles took ${stopWatch.getTotalTimeMillis()} milliseconds"
     }
 
     def 'Populate module sync work queue simultaneously on two parallel threads (CPS-2403).'() {
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy
index add931a1ad..914f2030cc 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy
@@ -100,6 +100,7 @@ class ModuleQueryPerfTest extends CpsPerfTestBase {
                 cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, (i..i+100).collect {SCHEMA_SET_PREFIX + it})
             }
             cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, [SCHEMA_SET_PREFIX + '0'])
+            cpsModuleService.deleteUnusedYangResourceModules()
     }
 
     // This makes a Yang module of approximately target length in bytes by padding the description field with many '*'