Add multi-threaded Integration Test for Module Sync

- Add tests for multi threaded scenarios around module sync
- Disabled ModuleSyncWatchdog timer using long delay and interval
- Call Module Sync method as needed for more control
  (sometimes it needs to be triggered twice like retry use cases as designed)
- Improve NCMP performance test setup (consistent naming etc.)
- Rename some production code method names to better reflect functionality
- Disabled intermittent failing test for create cm handle as it is not asserting the correct message
- Improved Code Coverage ModuleSyncWatchdog

Issue-ID: CPS-2462
Change-Id: Ia907af77d2037309f1bbb73ea671679b788bab9e
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

3 files changed, 66 insertions, 31 deletions

diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy
index dcff2e9b89..70e26d993c 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy
@@ -87,7 +87,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'cm handle is in READY state'
             mockCmHandleQueries.cmHandleHasState('cmhandle-3', CmHandleState.READY) >> true
         when: 'registration is processed'
-            objectUnderTest.updateDmiRegistrationAndSyncModule(dmiRegistration)
+            objectUnderTest.updateDmiRegistration(dmiRegistration)
         then: 'cm-handles are removed first'
             1 * objectUnderTest.processRemovedCmHandles(*_)
         and: 'de-registered cm handle entry is removed from in progress map'
@@ -108,7 +108,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'exception while checking cm handle state'
             mockInventoryPersistence.getYangModelCmHandle('cmhandle-3') >> new YangModelCmHandle(id: 'cmhandle-3', moduleSetTag: '', compositeState: new CompositeState(cmHandleState: cmHandleState))
         when: 'registration is processed'
-            def result = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiRegistration)
+            def result = objectUnderTest.updateDmiRegistration(dmiRegistration)
         then: 'upgrade operation contains expected error code'
             assert result.upgradedCmHandles[0].status == expectedResponseStatus
         where: 'the following parameters are used'
@@ -124,7 +124,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'exception while checking cm handle state'
             mockInventoryPersistence.getYangModelCmHandle('cmhandle-3') >> { throw exception }
         when: 'registration is processed'
-            def result = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiRegistration)
+            def result = objectUnderTest.updateDmiRegistration(dmiRegistration)
         then: 'upgrade operation contains expected error code'
             assert result.upgradedCmHandles.ncmpResponseStatus.code[0] == expectedErrorCode
         where: 'the following parameters are used'
@@ -139,7 +139,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
                 dmiDataPlugin: dmiDataPlugin)
             dmiPluginRegistration.createdCmHandles = [ncmpServiceCmHandle]
         when: 'update registration and sync module is called with correct DMI plugin information'
-            objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'create cm handles registration and sync modules is called with the correct plugin information'
             1 * objectUnderTest.processCreatedCmHandles(dmiPluginRegistration, _)
         where:
@@ -155,7 +155,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
                 dmiDataPlugin: dmiDataPlugin)
             dmiPluginRegistration.createdCmHandles = [ncmpServiceCmHandle]
         when: 'registration is called with incorrect DMI plugin information'
-            objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a DMI Request Exception is thrown with correct message details'
             def exceptionThrown = thrown(DmiRequestException.class)
             assert exceptionThrown.getMessage().contains(expectedMessageDetails)
@@ -178,7 +178,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
             def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server')
             dmiPluginRegistration.createdCmHandles = [new NcmpServiceCmHandle(cmHandleId: 'cmhandle', dmiProperties: dmiProperties, publicProperties: publicProperties)]
         when: 'registration is updated'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a successful response is received'
             response.createdCmHandles.size() == 1
             with(response.createdCmHandles[0]) {
@@ -206,7 +206,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
             def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server',
                 createdCmHandles:[new NcmpServiceCmHandle(cmHandleId: 'ch-1', registrationTrustLevel: registrationTrustLevel)])
         when: 'registration is updated'
-            objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'trustLevel is set for the created cm-handle'
             1 * mockTrustLevelManager.registerCmHandles(expectedMapping)
         where:
@@ -225,7 +225,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
             def xpath = "somePathWithId[@id='cmhandle2']"
             mockLcmEventsCmHandleStateHandler.initiateStateAdvised(*_) >> { throw AlreadyDefinedException.forDataNodes([xpath], 'some-context') }
         when: 'registration is updated to create cm-handles'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a response is received for all cm-handles'
             response.createdCmHandles.size() == 1
         and: 'all cm-handles creation fails'
@@ -244,7 +244,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'cm-handler registration fails: #scenario'
             mockLcmEventsCmHandleStateHandler.initiateStateAdvised(*_) >> { throw exception }
         when: 'registration is updated'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a failure response is received'
             response.createdCmHandles.size() == 1
             with(response.createdCmHandles[0]) {
@@ -269,7 +269,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
                                            CmHandleRegistrationResponse.createFailureResponse('cm handle 4', CM_HANDLE_INVALID_ID)]
             mockNetworkCmProxyDataServicePropertyHandler.updateCmHandleProperties(_) >> updateOperationResponse
         when: 'registration is updated'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'the response contains updateOperationResponse'
             assert response.updatedCmHandles.size() == 4
             assert response.updatedCmHandles.containsAll(updateOperationResponse)
@@ -281,7 +281,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: '#scenario'
             mockCpsModuleService.deleteSchemaSetsWithCascade(_, ['cmhandle']) >>  { if (!schemaSetExist) { throw new SchemaSetNotFoundException('', '') } }
         when: 'registration is updated to delete cmhandle'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'the cmHandle state is updated to "DELETING"'
             1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >>
                 { args -> args[0].values()[0] == CmHandleState.DELETING }
@@ -315,7 +315,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'cm-handle deletion is successful for 1st and 3rd; failed for 2nd'
             mockInventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='cmhandle2']") >> { throw new RuntimeException("Failed") }
         when: 'registration is updated to delete cmhandles'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'the cmHandle states are all updated to "DELETING"'
             1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch({ assert it.every { entry -> entry.value == CmHandleState.DELETING } })
         and: 'a response is received for all cm-handles'
@@ -361,7 +361,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'schema set single deletion failed with unknown error'
             mockInventoryPersistence.deleteSchemaSetWithCascade(_) >> { throw new RuntimeException('Failed') }
         when: 'registration is updated to delete cmhandle'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'no exception is thrown'
             noExceptionThrown()
         and: 'cm-handle is not deleted'
@@ -387,7 +387,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'cm-handle deletion fails on individual delete'
             mockInventoryPersistence.deleteDataNode(_) >> { throw deleteListElementException }
         when: 'registration is updated to delete cmhandle'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a failure response is received'
             assert response.removedCmHandles.size() == 1
             with(response.removedCmHandles[0]) {
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/NetworkCmProxyInventoryFacadeSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/NetworkCmProxyInventoryFacadeSpec.groovy
index fec07556eb..a7dd38cc15 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/NetworkCmProxyInventoryFacadeSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/NetworkCmProxyInventoryFacadeSpec.groovy
@@ -56,9 +56,9 @@ class NetworkCmProxyInventoryFacadeSpec extends Specification {
         given: 'an (updated) dmi plugin registration'
             def dmiPluginRegistration = Mock(DmiPluginRegistration)
         when: 'the registration is submitted '
-           objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+           objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'the call is delegated to the cm handle registration service'
-            1 * mockCmHandleRegistrationService.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            1 * mockCmHandleRegistrationService.updateDmiRegistration(dmiPluginRegistration)
     }
 
     def 'Execute cm handle id search for inventory'() {
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy
index 3064a78ff9..f2c88a511e 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy
@@ -23,6 +23,7 @@ package org.onap.cps.ncmp.impl.inventory.sync
 
 import com.hazelcast.map.IMap
 import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle
+import org.onap.cps.ncmp.impl.utils.Sleeper
 import org.onap.cps.spi.model.DataNode
 import spock.lang.Specification
 
@@ -31,7 +32,7 @@ import java.util.concurrent.locks.Lock
 
 class ModuleSyncWatchdogSpec extends Specification {
 
-    def mockSyncUtils = Mock(ModuleOperationsUtils)
+    def mockModuleOperationsUtils = Mock(ModuleOperationsUtils)
 
     def static testQueueCapacity = 50 + 2 * ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE
 
@@ -45,16 +46,21 @@ class ModuleSyncWatchdogSpec extends Specification {
 
     def mockWorkQueueLock = Mock(Lock)
 
-    def objectUnderTest = new ModuleSyncWatchdog(mockSyncUtils, moduleSyncWorkQueue , mockModuleSyncStartedOnCmHandles, mockModuleSyncTasks, spiedAsyncTaskExecutor, mockWorkQueueLock)
+    def spiedSleeper = Spy(Sleeper)
+
+    def objectUnderTest = new ModuleSyncWatchdog(mockModuleOperationsUtils, moduleSyncWorkQueue , mockModuleSyncStartedOnCmHandles, mockModuleSyncTasks, spiedAsyncTaskExecutor, mockWorkQueueLock, spiedSleeper)
 
     void setup() {
         spiedAsyncTaskExecutor.setupThreadPool()
-        mockWorkQueueLock.tryLock() >> true
     }
 
     def 'Module sync advised cm handles with #scenario.'() {
-        given: 'sync utilities returns #numberOfAdvisedCmHandles advised cm handles'
-            mockSyncUtils.getAdvisedCmHandles() >> createDataNodes(numberOfAdvisedCmHandles)
+        given: 'module sync utilities returns #numberOfAdvisedCmHandles advised cm handles'
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(numberOfAdvisedCmHandles)
+        and: 'module sync utilities returns no failed (locked) cm handles'
+            mockModuleOperationsUtils.getCmHandlesThatFailedModelSyncOrUpgrade() >> []
+        and: 'the work queue is not locked'
+            mockWorkQueueLock.tryLock() >> true
         and: 'the executor has enough available threads'
             spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >> 3
         when: ' module sync is started'
@@ -63,6 +69,7 @@ class ModuleSyncWatchdogSpec extends Specification {
             expectedNumberOfTaskExecutions * spiedAsyncTaskExecutor.executeTask(*_)
         where: 'the following parameter are used'
             scenario              | numberOfAdvisedCmHandles                                          || expectedNumberOfTaskExecutions
+            'none at all'         | 0                                                                 || 0
             'less then 1 batch'   | 1                                                                 || 1
             'exactly 1 batch'     | ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE                         || 1
             '2 batches'           | 2 * ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE                     || 2
@@ -70,9 +77,11 @@ class ModuleSyncWatchdogSpec extends Specification {
             'over queue capacity' | testQueueCapacity + 2 * ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE || 3
     }
 
-    def 'Module sync advised cm handles starts with no available threads.'() {
-        given: 'sync utilities returns a advise cm handles'
-            mockSyncUtils.getAdvisedCmHandles() >> createDataNodes(1)
+    def 'Module sync cm handles starts with no available threads.'() {
+        given: 'module sync utilities returns a advise cm handles'
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(1)
+        and: 'the work queue is not locked'
+            mockWorkQueueLock.tryLock() >> true
         and: 'the executor first has no threads but has one thread on the second attempt'
             spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >>> [ 0, 1 ]
         when: ' module sync is started'
@@ -81,9 +90,11 @@ class ModuleSyncWatchdogSpec extends Specification {
             1 * spiedAsyncTaskExecutor.executeTask(*_)
     }
 
-    def 'Module sync advised cm handles already handled.'() {
-        given: 'sync utilities returns a advise cm handles'
-            mockSyncUtils.getAdvisedCmHandles() >> createDataNodes(1)
+    def 'Module sync advised cm handle already handled by other thread.'() {
+        given: 'module sync utilities returns an advised cm handle'
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(1)
+        and: 'the work queue is not locked'
+            mockWorkQueueLock.tryLock() >> true
         and: 'the executor has a thread available'
             spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >> 1
         and: 'the semaphore cache indicates the cm handle is already being processed'
@@ -98,7 +109,7 @@ class ModuleSyncWatchdogSpec extends Specification {
         given: 'there is still a cm handle in the queue'
             moduleSyncWorkQueue.offer(new DataNode())
         and: 'sync utilities returns many advise cm handles'
-            mockSyncUtils.getAdvisedCmHandles() >> createDataNodes(500)
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(500)
         and: 'the executor has plenty threads available'
             spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >> 10
         when: ' module sync is started'
@@ -108,18 +119,42 @@ class ModuleSyncWatchdogSpec extends Specification {
     }
 
     def 'Reset failed cm handles.'() {
-        given: 'sync utilities returns failed cm handles'
+        given: 'module sync utilities returns failed cm handles'
             def failedCmHandles = [new YangModelCmHandle()]
-            mockSyncUtils.getCmHandlesThatFailedModelSyncOrUpgrade() >> failedCmHandles
+            mockModuleOperationsUtils.getCmHandlesThatFailedModelSyncOrUpgrade() >> failedCmHandles
         when: 'reset failed cm handles is started'
             objectUnderTest.setPreviouslyLockedCmHandlesToAdvised()
         then: 'it is delegated to the module sync task (service)'
             1 * mockModuleSyncTasks.setCmHandlesToAdvised(failedCmHandles)
     }
 
+    def 'Module Sync Locking.'() {
+        given: 'module sync utilities returns an advised cm handle'
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(1)
+        and: 'can lock is : #canLock'
+            mockWorkQueueLock.tryLock() >> canLock
+        when: 'attempt to populate the work queue'
+            objectUnderTest.populateWorkQueueIfNeeded()
+        then: 'the queue remains empty is #expectQueueRemainsEmpty'
+            assert moduleSyncWorkQueue.isEmpty() == expectQueueRemainsEmpty
+        where: 'the following lock states are applied'
+            canLock | expectQueueRemainsEmpty
+            false   | true
+            true    | false
+    }
+
+    def 'Sleeper gets interrupted.'() {
+        given: 'sleeper gets interrupted'
+            spiedSleeper.haveALittleRest(_) >> { throw new InterruptedException() }
+        when: 'the watchdog attempts to sleep to save cpu cycles'
+            objectUnderTest.preventBusyWait()
+        then: 'no exception is thrown'
+            noExceptionThrown()
+    }
+
     def createDataNodes(numberOfDataNodes) {
         def dataNodes = []
-        (1..numberOfDataNodes).each {dataNodes.add(new DataNode())}
+        numberOfDataNodes.times { dataNodes.add(new DataNode()) }
         return dataNodes
     }
 }