Test to highlight ModuleSetTag Inefficiencies

- Add (micrometer) instrumentation to expose inefficiencies
- Add test config for micrometer
- Add setup methods in base to create many cm handles
- Set module sync parallelism to 2 for testing
- Add clean up methods for hazelcast related tests
- added test to show inefficiencies
- POC 1 use hazelcast set to prevent multiple threads working on same ModuleSetTag
- POC 2 'cache' module set tags per thread to prevent DB looks ups
- Main inefficiency left: create schemaset for EACH cm Handled even if same tag. No easy PoC...

Change-Id: Idf46b44c475a24727dd7084bb613459f4c29be55
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

6 files changed, 165 insertions, 61 deletions

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy
index 759eccd966..02a10cfa6b 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy
@@ -21,6 +21,7 @@
 
 package org.onap.cps.integration.base
 
+import com.hazelcast.collection.ISet
 import okhttp3.mockwebserver.MockWebServer
 import org.onap.cps.api.CpsAnchorService
 import org.onap.cps.api.CpsDataService
@@ -37,13 +38,13 @@ import org.onap.cps.ncmp.impl.data.NetworkCmProxyQueryService
 import org.onap.cps.ncmp.impl.inventory.InventoryPersistence
 import org.onap.cps.ncmp.impl.inventory.ParameterizedCmHandleQueryService
 import org.onap.cps.ncmp.impl.inventory.models.CmHandleState
+import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncService
 import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog
 import org.onap.cps.ncmp.impl.utils.AlternateIdMatcher
 import org.onap.cps.ri.repository.DataspaceRepository
 import org.onap.cps.ri.utils.SessionManager
 import org.onap.cps.spi.exceptions.DataspaceNotFoundException
 import org.onap.cps.spi.model.DataNode
-import org.onap.cps.utils.ContentType
 import org.onap.cps.utils.JsonObjectMapper
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Value
@@ -61,13 +62,8 @@ import spock.lang.Specification
 import spock.util.concurrent.PollingConditions
 
 import java.time.OffsetDateTime
-import java.time.format.DateTimeFormatter
 import java.util.concurrent.BlockingQueue
 
-import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DATASPACE_NAME
-import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_ANCHOR
-import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_PARENT
-
 @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, classes = [CpsDataspaceService])
 @Testcontainers
 @EnableAutoConfiguration
@@ -121,6 +117,9 @@ abstract class CpsIntegrationSpecBase extends Specification {
     ModuleSyncWatchdog moduleSyncWatchdog
 
     @Autowired
+    ModuleSyncService moduleSyncService
+
+    @Autowired
     BlockingQueue<DataNode> moduleSyncWorkQueue
 
     @Autowired
@@ -132,6 +131,8 @@ abstract class CpsIntegrationSpecBase extends Specification {
     @Autowired
     AlternateIdMatcher alternateIdMatcher
 
+    @Autowired
+    ISet<String> moduleSetTagsBeingProcessed
 
     @Value('${ncmp.policy-executor.server.port:8080}')
     private String policyServerPort;
@@ -174,13 +175,13 @@ abstract class CpsIntegrationSpecBase extends Specification {
 
         DMI1_URL = String.format("http://%s:%s", mockDmiServer1.getHostName(), mockDmiServer1.getPort())
         DMI2_URL = String.format("http://%s:%s", mockDmiServer2.getHostName(), mockDmiServer2.getPort())
-
     }
 
     def cleanup() {
         mockDmiServer1.shutdown()
         mockDmiServer2.shutdown()
         mockPolicyServer.shutdown()
+        moduleSetTagsBeingProcessed.clear()
     }
 
     def static readResourceDataFile(filename) {
@@ -262,11 +263,16 @@ abstract class CpsIntegrationSpecBase extends Specification {
         networkCmProxyInventoryFacade.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: dmiPlugin, createdCmHandles: [cmHandleToCreate]))
     }
 
-    def registerSequenceOfCmHandlesWithoutWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles) {
+    def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset) {
         def cmHandles = []
+        def id = offset
+        def moduleReferences = (1..200).collect { moduleSetTag + '_Module_' + it.toString() }
         (1..numberOfCmHandles).each {
-            def cmHandle = new NcmpServiceCmHandle(cmHandleId: 'ch-'+it, moduleSetTag: moduleSetTag, alternateId: NO_ALTERNATE_ID)
-            cmHandles.add(cmHandle)
+            def ncmpServiceCmHandle = new NcmpServiceCmHandle(cmHandleId: 'ch-'+id, moduleSetTag: moduleSetTag, alternateId: NO_ALTERNATE_ID)
+            cmHandles.add(ncmpServiceCmHandle)
+            dmiDispatcher1.moduleNamesPerCmHandleId[ncmpServiceCmHandle.cmHandleId] = moduleReferences
+            dmiDispatcher2.moduleNamesPerCmHandleId[ncmpServiceCmHandle.cmHandleId] = moduleReferences
+            id++
         }
         networkCmProxyInventoryFacade.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: dmiPlugin, createdCmHandles: cmHandles))
     }
@@ -279,9 +285,10 @@ abstract class CpsIntegrationSpecBase extends Specification {
         networkCmProxyInventoryFacade.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: dmiPlugin, removedCmHandles: cmHandleIds))
     }
 
-    def deregisterSequenceOfCmHandles(dmiPlugin, numberOfCmHandles) {
+    def deregisterSequenceOfCmHandles(dmiPlugin, numberOfCmHandles, offset) {
         def cmHandleIds = []
-        (1..numberOfCmHandles).each { cmHandleIds.add('ch-'+it) }
+        def id = offset
+        (1..numberOfCmHandles).each { cmHandleIds.add('ch-' + id++) }
         networkCmProxyInventoryFacade.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: dmiPlugin, removedCmHandles: cmHandleIds))
     }
 
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy
index 64449371fe..a5e3daf289 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy
@@ -33,54 +33,55 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase {
 
     NetworkCmProxyInventoryFacade objectUnderTest
 
-    static final CM_HANDLE_ID = 'ch-1'
-    static final CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG = 'ch-2'
+    def cmHandleId = 'ch-1'
+    def cmHandleIdWithExistingModuleSetTag = 'ch-2'
 
     def setup() {
         objectUnderTest = networkCmProxyInventoryFacade
+        moduleSyncService.clearPrivateModuleSetCache()
     }
 
     def 'Upgrade CM-handle with new moduleSetTag or no moduleSetTag.'() {
         given: 'a CM-handle is created with expected initial modules: M1 and M2'
-            dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M2']
-            registerCmHandle(DMI1_URL, CM_HANDLE_ID, initialModuleSetTag)
-            assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort()
+            dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M2']
+            registerCmHandle(DMI1_URL, cmHandleId, initialModuleSetTag)
+            assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort()
 
         when: "the CM-handle is upgraded with given moduleSetTag '${updatedModuleSetTag}'"
-            def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: updatedModuleSetTag)
+            def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [cmHandleId], moduleSetTag: updatedModuleSetTag)
             def dmiPluginRegistrationResponse = objectUnderTest.updateDmiRegistration(
                     new DmiPluginRegistration(dmiPlugin: DMI1_URL, upgradedCmHandles: cmHandlesToUpgrade))
 
         then: 'registration gives successful response'
-            assert dmiPluginRegistrationResponse.upgradedCmHandles == [CmHandleRegistrationResponse.createSuccessResponse(CM_HANDLE_ID)]
+            assert dmiPluginRegistrationResponse.upgradedCmHandles == [CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)]
 
         and: 'CM-handle is in LOCKED state due to MODULE_UPGRADE'
-            def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID)
+            def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId)
             assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED
             assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE
             assert cmHandleCompositeState.lockReason.details == "Upgrade to ModuleSetTag: ${updatedModuleSetTag}"
 
         when: 'DMI will return different modules for upgrade: M1 and M3'
-            dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M3']
+            dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M3']
 
         and: 'the module sync watchdog is triggered twice'
             2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() }
 
         then: 'CM-handle goes to READY state'
             new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID).cmHandleState
+                assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState
             })
 
         and: 'the CM-handle has expected moduleSetTag'
-            assert objectUnderTest.getNcmpServiceCmHandle(CM_HANDLE_ID).moduleSetTag == updatedModuleSetTag
+            assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag
 
         and: 'CM-handle has expected updated modules: M1 and M3'
-            assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort()
+            assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort()
 
         cleanup: 'deregister CM-handle'
-            deregisterCmHandle(DMI1_URL, CM_HANDLE_ID)
+            deregisterCmHandle(DMI1_URL, cmHandleId)
 
-        where:
+        where: 'following module set tags are used'
             initialModuleSetTag | updatedModuleSetTag
             NO_MODULE_SET_TAG   | NO_MODULE_SET_TAG
             NO_MODULE_SET_TAG   | 'new'
@@ -90,39 +91,39 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase {
 
     def 'Upgrade CM-handle with existing moduleSetTag.'() {
         given: 'DMI will return modules for registration'
-            dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M2']
-            dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG] = ['M1', 'M3']
+            dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M2']
+            dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleIdWithExistingModuleSetTag] = ['M1', 'M3']
         and: "an existing CM-handle handle with moduleSetTag '${updatedModuleSetTag}'"
-            registerCmHandle(DMI1_URL, CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG, updatedModuleSetTag)
-            assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG).moduleName.sort()
+            registerCmHandle(DMI1_URL, cmHandleIdWithExistingModuleSetTag, updatedModuleSetTag)
+            assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleIdWithExistingModuleSetTag).moduleName.sort()
         and: "a CM-handle with moduleSetTag '${initialModuleSetTag}' which will be upgraded"
-            registerCmHandle(DMI1_URL, CM_HANDLE_ID, initialModuleSetTag)
-            assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort()
+            registerCmHandle(DMI1_URL, cmHandleId, initialModuleSetTag)
+            assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort()
 
         when: "CM-handle is upgraded to moduleSetTag '${updatedModuleSetTag}'"
-            def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: updatedModuleSetTag)
+            def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [cmHandleId], moduleSetTag: updatedModuleSetTag)
             def dmiPluginRegistrationResponse = objectUnderTest.updateDmiRegistration(
                     new DmiPluginRegistration(dmiPlugin: DMI1_URL, upgradedCmHandles: cmHandlesToUpgrade))
 
         then: 'registration gives successful response'
-            assert dmiPluginRegistrationResponse.upgradedCmHandles == [CmHandleRegistrationResponse.createSuccessResponse(CM_HANDLE_ID)]
+            assert dmiPluginRegistrationResponse.upgradedCmHandles == [CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)]
 
         and: 'the module sync watchdog is triggered twice'
             2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() }
 
         and: 'CM-handle goes to READY state'
             new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID).cmHandleState
+                assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState
             })
 
         and: 'the CM-handle has expected moduleSetTag'
-            assert objectUnderTest.getNcmpServiceCmHandle(CM_HANDLE_ID).moduleSetTag == updatedModuleSetTag
+            assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag
 
         and: 'CM-handle has expected updated modules: M1 and M3'
-            assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort()
+            assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort()
 
         cleanup: 'deregister CM-handle'
-            deregisterCmHandles(DMI1_URL, [CM_HANDLE_ID, CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG])
+            deregisterCmHandles(DMI1_URL, [cmHandleId, cmHandleIdWithExistingModuleSetTag])
 
         where:
             initialModuleSetTag | updatedModuleSetTag
@@ -132,37 +133,37 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase {
 
     def 'Skip upgrade of CM-handle with same moduleSetTag as before.'() {
         given: 'an existing CM-handle with expected initial modules: M1 and M2'
-            dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M2']
-            registerCmHandle(DMI1_URL, CM_HANDLE_ID, 'same')
-            assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort()
+            dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M2']
+            registerCmHandle(DMI1_URL, cmHandleId, 'same')
+            assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort()
 
         when: 'CM-handle is upgraded with the same moduleSetTag'
-            def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: 'same')
+            def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [cmHandleId], moduleSetTag: 'same')
             objectUnderTest.updateDmiRegistration(
                     new DmiPluginRegistration(dmiPlugin: DMI1_URL, upgradedCmHandles: cmHandlesToUpgrade))
 
         then: 'CM-handle remains in READY state'
-            assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID).cmHandleState
+            assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState
 
         and: 'the CM-handle has same moduleSetTag as before'
-            assert objectUnderTest.getNcmpServiceCmHandle(CM_HANDLE_ID).moduleSetTag == 'same'
+            assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == 'same'
 
         then: 'CM-handle has same modules as before: M1 and M2'
-            assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort()
+            assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort()
 
         cleanup: 'deregister CM-handle'
-            deregisterCmHandle(DMI1_URL, CM_HANDLE_ID)
+            deregisterCmHandle(DMI1_URL, cmHandleId)
     }
 
     def 'Upgrade of CM-handle fails due to DMI error.'() {
         given: 'a CM-handle exists'
-            dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M2']
-            registerCmHandle(DMI1_URL, CM_HANDLE_ID, 'oldTag')
+            dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M2']
+            registerCmHandle(DMI1_URL, cmHandleId, 'oldTag')
         and: 'DMI is not available for upgrade'
             dmiDispatcher1.isAvailable = false
 
         when: 'the CM-handle is upgraded'
-            def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: 'newTag')
+            def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [cmHandleId], moduleSetTag: 'newTag')
             objectUnderTest.updateDmiRegistration(
                     new DmiPluginRegistration(dmiPlugin: DMI1_URL, upgradedCmHandles: cmHandlesToUpgrade))
 
@@ -171,16 +172,16 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase {
 
         then: 'CM-handle goes to LOCKED state with reason MODULE_UPGRADE_FAILED'
             new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID)
+                def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId)
                 assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED
                 assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE_FAILED
             })
 
         and: 'the CM-handle has same moduleSetTag as before'
-            assert objectUnderTest.getNcmpServiceCmHandle(CM_HANDLE_ID).moduleSetTag == 'oldTag'
+            assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == 'oldTag'
 
         cleanup: 'deregister CM-handle'
-            deregisterCmHandle(DMI1_URL, CM_HANDLE_ID)
+            deregisterCmHandle(DMI1_URL, cmHandleId)
     }
 
 }
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
index e0bb437a7c..963bc1fe61 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
@@ -20,26 +20,32 @@
 
 package org.onap.cps.integration.functional.ncmp
 
+import io.micrometer.core.instrument.MeterRegistry
 import org.onap.cps.integration.base.CpsIntegrationSpecBase
 import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog
+import org.springframework.beans.factory.annotation.Autowired
+import spock.util.concurrent.PollingConditions
 
 import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
 
 class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
 
     ModuleSyncWatchdog objectUnderTest
 
+    @Autowired
+    MeterRegistry meterRegistry
+
     def executorService = Executors.newFixedThreadPool(2)
-    def SYNC_SAMPLE_SIZE = 100
+    def PARALLEL_SYNC_SAMPLE_SIZE = 100
 
     def setup() {
         objectUnderTest = moduleSyncWatchdog
-        registerSequenceOfCmHandlesWithoutWaitForReady(DMI1_URL, NO_MODULE_SET_TAG, SYNC_SAMPLE_SIZE)
     }
 
     def cleanup() {
         try {
-            deregisterSequenceOfCmHandles(DMI1_URL, SYNC_SAMPLE_SIZE)
+            deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1)
             moduleSyncWorkQueue.clear()
         } finally {
             executorService.shutdownNow()
@@ -47,15 +53,60 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
     }
 
     def 'Watchdog is disabled for test.'() {
+        given:
+            registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, NO_MODULE_SET_TAG, PARALLEL_SYNC_SAMPLE_SIZE, 1)
         when: 'wait a while but less then the initial delay of 10 minutes'
             Thread.sleep(3000)
         then: 'the work queue remains empty'
             assert moduleSyncWorkQueue.isEmpty()
     }
 
+    def 'CPS-2478 Highlight module sync inefficiencies.'() {
+        given: 'register 250 cm handles with module set tag cps-2478-A'
+            def numberOfTags = 2
+            def cmHandlesPerTag = 250
+            def totalCmHandles = numberOfTags * cmHandlesPerTag
+            def offset = 1
+            registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-A', cmHandlesPerTag, offset)
+        and: 'register anther 250 cm handles with module set tag cps-2478-B'
+            offset += cmHandlesPerTag
+            registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-B', cmHandlesPerTag, offset)
+        and: 'clear any previous instrumentation'
+            meterRegistry.clear()
+        when: 'sync all advised cm handles'
+            objectUnderTest.moduleSyncAdvisedCmHandles()
+            Thread.sleep(100)
+        then: 'retry until all schema sets are stored in db (1 schema set  for each cm handle)'
+            def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.store').timer()
+            new PollingConditions().within(10, () -> {
+                objectUnderTest.moduleSyncAdvisedCmHandles()
+                Thread.sleep(100)
+                assert dbSchemaSetStorageTimer.count() >= 500
+            })
+        then: 'wait till at least 5 batches of state updates are done (often more because of retries of locked cm handles)'
+            def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer()
+            new PollingConditions().within(10, () -> {
+                assert dbStateUpdateTimer.count() >= 5
+            })
+        and: 'the db has been queried for tags exactly 2 times.'
+            def dbModuleQueriesTimer = meterRegistry.get('cps.module.service.module.reference.query.by.attribute').timer()
+            assert dbModuleQueriesTimer.count() == 2
+        and: 'exactly 2 calls to DMI to get module references'
+            def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer()
+            assert dmiModuleRetrievalTimer.count() == 2
+        and: 'log the relevant instrumentation'
+            logInstrumentation(dbModuleQueriesTimer,    'query module references')
+            logInstrumentation(dmiModuleRetrievalTimer, 'get modules from DMI   ')
+            logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets      ')
+            logInstrumentation(dbStateUpdateTimer,      'batch state updates    ')
+        cleanup: 'remove all cm handles'
+            deregisterSequenceOfCmHandles(DMI1_URL, totalCmHandles, 1)
+    }
+
     def 'Populate module sync work queue simultaneously on two parallel threads (CPS-2403).'() {
         // This test failed before bug https://lf-onap.atlassian.net/browse/CPS-2403 was fixed
         given: 'the queue is empty at the start'
+            registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, NO_MODULE_SET_TAG, PARALLEL_SYNC_SAMPLE_SIZE, 1)
             assert moduleSyncWorkQueue.isEmpty()
         when: 'attempt to populate the queue on the main (test) and another parallel thread at the same time'
             objectUnderTest.populateWorkQueueIfNeeded()
@@ -63,12 +114,13 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
         and: 'wait a little (to give all threads time to complete their task)'
             Thread.sleep(50)
         then: 'the queue size is exactly the sample size'
-            assert moduleSyncWorkQueue.size() == SYNC_SAMPLE_SIZE
+            assert moduleSyncWorkQueue.size() == PARALLEL_SYNC_SAMPLE_SIZE
     }
 
     def 'Populate module sync work queue on two parallel threads with a slight difference in start time.'() {
         // This test proved that the issue in CPS-2403 did not arise if the the queue was populated and given time to be distributed
         given: 'the queue is empty at the start'
+            registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, NO_MODULE_SET_TAG, PARALLEL_SYNC_SAMPLE_SIZE, 1)
             assert moduleSyncWorkQueue.isEmpty()
         when: 'attempt to populate the queue on the main (test) and another parallel thread a little later'
             objectUnderTest.populateWorkQueueIfNeeded()
@@ -76,7 +128,12 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
         and: 'wait a little (to give all threads time to complete their task)'
             Thread.sleep(50)
         then: 'the queue size is exactly the sample size'
-            assert moduleSyncWorkQueue.size() == SYNC_SAMPLE_SIZE
+            assert moduleSyncWorkQueue.size() == PARALLEL_SYNC_SAMPLE_SIZE
+    }
+
+    def logInstrumentation(timer, description) {
+        System.out.println('*** CPS-2478, ' + description + ' : ' + timer.count()+ ' times, total ' + timer.totalTime(TimeUnit.MILLISECONDS) + ' ms')
+        return true
     }
 
     def populateQueueWithoutDelay = () -> {
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/PolicyExecutorIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/PolicyExecutorIntegrationSpec.groovy
index 56d4bfaee4..f897393a53 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/PolicyExecutorIntegrationSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/PolicyExecutorIntegrationSpec.groovy
@@ -43,9 +43,7 @@ class PolicyExecutorIntegrationSpec extends CpsIntegrationSpecBase {
     }
 
     def cleanup() {
-        deregisterCmHandle(DMI1_URL, 'ch-1')
-        deregisterCmHandle(DMI1_URL, 'ch-2')
-        deregisterCmHandle(DMI1_URL, 'ch-3')
+        deregisterSequenceOfCmHandles(DMI1_URL, 3, 1)
     }
 
     def 'Policy Executor create request with #scenario.'() {
diff --git a/integration-test/src/test/java/org/onap/cps/integration/MicroMeterTestConfig.java b/integration-test/src/test/java/org/onap/cps/integration/MicroMeterTestConfig.java
new file mode 100644
index 0000000000..3b26f42c8a
--- /dev/null
+++ b/integration-test/src/test/java/org/onap/cps/integration/MicroMeterTestConfig.java
@@ -0,0 +1,41 @@
+/*
+ *  ============LICENSE_START=======================================================
+ *  Copyright (C) 2024 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the 'License');
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an 'AS IS' BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.integration;
+
+import io.micrometer.core.aop.TimedAspect;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class MicroMeterTestConfig {
+    @Bean
+    public MeterRegistry meterRegistry() {
+        return new SimpleMeterRegistry(); // Use a simple in-memory registry for testing
+    }
+
+    @Bean
+    public TimedAspect timedAspect(final MeterRegistry meterRegistry) {
+        return new TimedAspect(meterRegistry);
+    }
+}
+
diff --git a/integration-test/src/test/resources/application.yml b/integration-test/src/test/resources/application.yml
index b786a3d4c5..30598dfb90 100644
--- a/integration-test/src/test/resources/application.yml
+++ b/integration-test/src/test/resources/application.yml
@@ -191,7 +191,7 @@ ncmp:
 
   modules-sync-watchdog:
     async-executor:
-      parallelism-level: 1
+      parallelism-level: 2
 
   model-loader:
     maximum-attempt-count: 20