Fix for intermittent failures of module sync tests

- Removed PollingConditions as ModuleSyncWatchdog is now synchronous
- Use Hazelcast IMap::delete instead of IMap::removeAsync to fix root
  cause of test failures: removeAsync may not be finished when module
  sync is re-run during tests.

Issue-ID: CPS-2615
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: I1786a1303606132218dc8e4e91fb58e6f6cc0bcb

5 files changed, 27 insertions, 56 deletions

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy
index 27d85de034..453dbca2bd 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy
@@ -22,6 +22,8 @@
 package org.onap.cps.integration.base
 
 import com.hazelcast.map.IMap
+import java.time.OffsetDateTime
+import java.util.concurrent.BlockingQueue
 import okhttp3.mockwebserver.MockWebServer
 import org.onap.cps.api.CpsAnchorService
 import org.onap.cps.api.CpsDataService
@@ -60,10 +62,6 @@ import org.springframework.test.web.servlet.MockMvc
 import org.testcontainers.spock.Testcontainers
 import spock.lang.Shared
 import spock.lang.Specification
-import spock.util.concurrent.PollingConditions
-
-import java.time.OffsetDateTime
-import java.util.concurrent.BlockingQueue
 
 @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, classes = [CpsDataspaceService])
 @Testcontainers
@@ -160,7 +158,6 @@ abstract class CpsIntegrationSpecBase extends Specification {
     static NO_ALTERNATE_ID = ''
     static GENERAL_TEST_DATASPACE = 'generalTestDataspace'
     static BOOKSTORE_SCHEMA_SET = 'bookstoreSchemaSet'
-    static MODULE_SYNC_WAIT_TIME_IN_SECONDS = 2
 
     static initialized = false
     def now = OffsetDateTime.now()
@@ -262,9 +259,7 @@ abstract class CpsIntegrationSpecBase extends Specification {
     def registerCmHandle(dmiPlugin, cmHandleId, moduleSetTag, alternateId) {
         registerCmHandleWithoutWaitForReady(dmiPlugin, cmHandleId, moduleSetTag, alternateId)
         moduleSyncWatchdog.moduleSyncAdvisedCmHandles()
-        new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-            CmHandleState.READY == networkCmProxyInventoryFacade.getCmHandleCompositeState(cmHandleId).cmHandleState
-        })
+        CmHandleState.READY == networkCmProxyInventoryFacade.getCmHandleCompositeState(cmHandleId).cmHandleState
     }
 
     def registerCmHandleWithoutWaitForReady(dmiPlugin, cmHandleId, moduleSetTag, alternateId) {
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy
index c4946b48b9..41798cb1f1 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy
@@ -20,6 +20,7 @@
 
 package org.onap.cps.integration.functional.ncmp
 
+import java.time.Duration
 import org.apache.kafka.clients.consumer.KafkaConsumer
 import org.apache.kafka.common.serialization.StringDeserializer
 import org.onap.cps.integration.KafkaTestContainer
@@ -32,9 +33,6 @@ import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle
 import org.onap.cps.ncmp.events.lcm.v1.LcmEvent
 import org.onap.cps.ncmp.api.inventory.models.CmHandleState
 import org.onap.cps.ncmp.api.inventory.models.LockReasonCategory
-import spock.util.concurrent.PollingConditions
-
-import java.time.Duration
 
 class CmHandleCreateSpec extends CpsIntegrationSpecBase {
 
@@ -73,9 +71,7 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase {
             moduleSyncWatchdog.moduleSyncAdvisedCmHandles()
 
         then: 'CM-handle goes to READY state after module sync'
-            new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(uniqueId).cmHandleState
-            })
+            assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(uniqueId).cmHandleState
 
         and: 'the CM-handle has expected modules'
             assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(uniqueId).moduleName.sort()
@@ -113,11 +109,9 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase {
             moduleSyncWatchdog.moduleSyncAdvisedCmHandles()
 
         then: 'CM-handle goes to LOCKED state with reason MODULE_SYNC_FAILED'
-            new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState('ch-1')
-                assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED
-                assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_SYNC_FAILED
-            })
+            def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState('ch-1')
+            assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED
+            assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_SYNC_FAILED
 
         and: 'CM-handle has no modules'
             assert objectUnderTest.getYangResourcesModuleReferences('ch-1').empty
@@ -141,9 +135,7 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase {
             moduleSyncWatchdog.moduleSyncAdvisedCmHandles()
 
         then: 'the CM-handle goes to READY state'
-            new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState('ch-3').cmHandleState
-            })
+            assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState('ch-3').cmHandleState
 
         and: 'the CM-handle has expected moduleSetTag'
             assert objectUnderTest.getNcmpServiceCmHandle('ch-3').moduleSetTag == 'B'
@@ -200,9 +192,7 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase {
             moduleSyncWatchdog.moduleSyncAdvisedCmHandles()
 
         then: 'CM-handles go to LOCKED state'
-            new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.LOCKED
-            })
+            assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.LOCKED
 
         when: 'DMI is available for retry'
             dmiDispatcher1.moduleNamesPerCmHandleId = ['ch-1': ['M1', 'M2'], 'ch-2': ['M1', 'M2']]
@@ -212,11 +202,9 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase {
             2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() }
 
         then: 'Both CM-handles go to READY state'
-            new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                ['ch-1', 'ch-2'].each { cmHandleId ->
-                    assert objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState == CmHandleState.READY
-                }
-            })
+            ['ch-1', 'ch-2'].each { cmHandleId ->
+                assert objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState == CmHandleState.READY
+            }
 
         and: 'Both CM-handles have expected modules'
             ['ch-1', 'ch-2'].each { cmHandleId ->
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy
index 097a043556..43540a9675 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy
@@ -27,7 +27,6 @@ import org.onap.cps.ncmp.api.inventory.models.DmiPluginRegistration
 import org.onap.cps.ncmp.api.inventory.models.LockReasonCategory
 import org.onap.cps.ncmp.api.inventory.models.UpgradedCmHandles
 import org.onap.cps.ncmp.impl.NetworkCmProxyInventoryFacadeImpl
-import spock.util.concurrent.PollingConditions
 
 class CmHandleUpgradeSpec extends CpsIntegrationSpecBase {
 
@@ -67,9 +66,7 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase {
             2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() }
 
         then: 'CM-handle goes to READY state'
-            new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState
-            })
+            assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState
 
         and: 'the CM-handle has expected moduleSetTag'
             assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag
@@ -111,9 +108,7 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase {
             2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() }
 
         and: 'CM-handle goes to READY state'
-            new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState
-            })
+            assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState
 
         and: 'the CM-handle has expected moduleSetTag'
             assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag
@@ -170,11 +165,9 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase {
             2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() }
 
         then: 'CM-handle goes to LOCKED state with reason MODULE_UPGRADE_FAILED'
-            new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId)
-                assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED
-                assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE_FAILED
-            })
+            def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId)
+            assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED
+            assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE_FAILED
 
         and: 'the CM-handle has same moduleSetTag as before'
             assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == 'oldTag'
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
index 81fbc3c4f7..8bbe0ca194 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
@@ -101,7 +101,7 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
             })
         and: 'log the relevant instrumentation'
             def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer()
-            def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.store').timer()
+            def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.createFromNewAndExistingModules').timer()
             def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer()
             logInstrumentation(dmiModuleRetrievalTimer, 'get modules from DMI   ')
             logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets      ')
@@ -145,10 +145,8 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
         when: 'advised cm handles are processed on 2 threads (exactly one batch for each)'
             objectUnderTest.moduleSyncAdvisedCmHandles()
             executorService.execute(moduleSyncAdvisedCmHandles)
-        then: 'wait till all cm handles have been processed'
-            new PollingConditions().within(10, () -> {
-                assert getNumberOfProcessedCmHandles() == 200
-            })
+        then: 'all cm handles have been processed'
+            assert getNumberOfProcessedCmHandles() == 200
         then: 'at least 1 cm handle is in state LOCKED'
             assert cmHandlesByState.get('lockedCmHandlesCount') >= 1
         cleanup: 'remove all test cm handles'
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy
index 7ce3cf5e17..77349fe0a5 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy
@@ -1,6 +1,6 @@
 /*
  *  ============LICENSE_START=======================================================
- *  Copyright (C) 2024 Nordix Foundation
+ *  Copyright (C) 2024-2025 Nordix Foundation
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the 'License');
  *  you may not use this file except in compliance with the License.
@@ -29,7 +29,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 
 import org.onap.cps.integration.base.CpsIntegrationSpecBase
 import org.springframework.http.MediaType
-import spock.util.concurrent.PollingConditions
 
 class RestApiSpec extends CpsIntegrationSpecBase {
 
@@ -47,13 +46,11 @@ class RestApiSpec extends CpsIntegrationSpecBase {
         and: 'the module sync watchdog is triggered'
             moduleSyncWatchdog.moduleSyncAdvisedCmHandles()
         then: 'CM-handles go to READY state'
-            new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> {
-                (1..3).each {
-                    mvc.perform(get('/ncmp/v1/ch/ch-'+it))
-                            .andExpect(status().isOk())
-                            .andExpect(jsonPath('$.state.cmHandleState').value('READY'))
-                }
-            })
+            (1..3).each {
+                mvc.perform(get('/ncmp/v1/ch/ch-'+it))
+                        .andExpect(status().isOk())
+                        .andExpect(jsonPath('$.state.cmHandleState').value('READY'))
+            }
     }
 
     def 'Search for CM Handles by module using REST API.'() {