Fix for cm handles stuck in LOCKED during registration

- Additional Error logging when cm handles fail module sync
- Swallow already defined exception upon schema and/or anchor creation
- Updated integration test to try to reproduce the problem (but couldn't)
- Ignored integration tests that depend/affected by race conditions
  (they are useful for troubleshooting but not for pipeline checks)
- Removed last remnants of springboot retry annotation option (incl dependencies)

Issue-ID: CPS-2576
Change-Id: I910e802268332f955134c043bd1b46a7ec57233b
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

1 files changed, 17 insertions, 16 deletions

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
index 9cb8c29e21..81fbc3c4f7 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy
@@ -69,7 +69,7 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
             deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1)
     }
 
-    /** this test has intermittent failures, due to timeouts.
+    /** this test has intermittent failures, due to race conditions
      *  Ignored but left here as it might be valuable to further optimization investigations.
      **/
     @Ignore
@@ -79,7 +79,6 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
             def cmHandlesPerTag = 250
             def totalCmHandles = numberOfTags * cmHandlesPerTag
             def offset = 1
-            def minimumBatches = totalCmHandles / 100
             registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-A', cmHandlesPerTag, offset)
         and: 'register anther 250 cm handles with module set tag cps-2478-B'
             offset += cmHandlesPerTag
@@ -89,23 +88,21 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
         when: 'sync all advised cm handles'
             objectUnderTest.moduleSyncAdvisedCmHandles()
             Thread.sleep(100)
-        then: 'retry until both schema sets are stored in db (1 schema set for each module set tag)'
-            def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.store').timer()
-            new PollingConditions().within(10, () -> {
-                objectUnderTest.moduleSyncAdvisedCmHandles()
-                Thread.sleep(100)
-                assert dbSchemaSetStorageTimer.count() == 2
-            })
-        then: 'wait till at least 5 batches of state updates are done (often more because of retries of locked cm handles)'
-            def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer()
+        then: 'Keep processing until there are no more LOCKED or ADVISED cm handles'
             new PollingConditions().within(10, () -> {
-                assert dbStateUpdateTimer.count() >= minimumBatches
+                def advised = cmHandlesByState.get('advisedCmHandlesCount')
+                def locked = cmHandlesByState.get('lockedCmHandlesCount')
+                if ( locked > 0 | advised > 0 ) {
+                    println "CPS-2576 Need to retry ${locked} LOCKED / ${advised} ADVISED cm Handles"
+                    objectUnderTest.moduleSyncAdvisedCmHandles()
+                    Thread.sleep(100)
+                }
+                assert cmHandlesByState.get('lockedCmHandlesCount') + cmHandlesByState.get('advisedCmHandlesCount') == 0
             })
-        and: 'one call to DMI per module set tag to get module references (may be more due to parallel processing of batches)'
-            def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer()
-            assert dmiModuleRetrievalTimer.count() >= numberOfTags && dmiModuleRetrievalTimer.count() <= minimumBatches
-
         and: 'log the relevant instrumentation'
+            def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer()
+            def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.store').timer()
+            def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer()
             logInstrumentation(dmiModuleRetrievalTimer, 'get modules from DMI   ')
             logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets      ')
             logInstrumentation(dbStateUpdateTimer,      'batch state updates    ')
@@ -134,6 +131,10 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase {
             deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1)
     }
 
+    /** this test has intermittent failures, due to race conditions
+     *  Ignored but left here as it might be valuable to further optimization investigations.
+     **/
+    @Ignore
     def 'Schema sets with overlapping modules processed at the same time (DB constraint violation).'() {
         given: 'register one batch (100) cm handles of tag A (with overlapping module names)'
             registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'tagA', 100, 1, ModuleNameStrategy.OVERLAPPING)