Add multi-threaded Integration Test for Module Sync

- Add tests for multi threaded scenarios around module sync
- Disabled ModuleSyncWatchdog timer using long delay and interval
- Call Module Sync method as needed for more control
  (sometimes it needs to be triggered twice like retry use cases as designed)
- Improve NCMP performance test setup (consistent naming etc.)
- Rename some production code method names to better reflect functionality
- Disabled intermittent failing test for create cm handle as it is not asserting the correct message
- Improved Code Coverage ModuleSyncWatchdog

Issue-ID: CPS-2462
Change-Id: Ia907af77d2037309f1bbb73ea671679b788bab9e
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

7 files changed, 126 insertions, 48 deletions

diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/NetworkCmProxyInventoryFacade.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/NetworkCmProxyInventoryFacade.java
index 1fa801c3c5..a4ea7b454a 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/NetworkCmProxyInventoryFacade.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/NetworkCmProxyInventoryFacade.java
@@ -64,15 +64,16 @@ public class NetworkCmProxyInventoryFacade {
     private final TrustLevelManager trustLevelManager;
     private final AlternateIdMatcher alternateIdMatcher;
 
+
+
     /**
      * Registration of Created, Removed, Updated or Upgraded CM Handles.
      *
      * @param dmiPluginRegistration Dmi Plugin Registration details
      * @return dmiPluginRegistrationResponse
      */
-    public DmiPluginRegistrationResponse updateDmiRegistrationAndSyncModule(
-        final DmiPluginRegistration dmiPluginRegistration) {
-        return cmHandleRegistrationService.updateDmiRegistrationAndSyncModule(dmiPluginRegistration);
+    public DmiPluginRegistrationResponse updateDmiRegistration(final DmiPluginRegistration dmiPluginRegistration) {
+        return cmHandleRegistrationService.updateDmiRegistration(dmiPluginRegistration);
     }
 
     /**
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java
index d9f7e38993..cb55b09d41 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java
@@ -87,8 +87,7 @@ public class CmHandleRegistrationService {
      * @param dmiPluginRegistration Dmi Plugin Registration details
      * @return dmiPluginRegistrationResponse
      */
-    public DmiPluginRegistrationResponse updateDmiRegistrationAndSyncModule(
-        final DmiPluginRegistration dmiPluginRegistration) {
+    public DmiPluginRegistrationResponse updateDmiRegistration(final DmiPluginRegistration dmiPluginRegistration) {
 
         dmiPluginRegistration.validateDmiPluginRegistration();
         final DmiPluginRegistrationResponse dmiPluginRegistrationResponse = new DmiPluginRegistrationResponse();
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java
index 6b3452734e..898b8d5bf4 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java
@@ -32,6 +32,7 @@ import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle;
+import org.onap.cps.ncmp.impl.utils.Sleeper;
 import org.onap.cps.spi.model.DataNode;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
@@ -47,6 +48,7 @@ public class ModuleSyncWatchdog {
     private final ModuleSyncTasks moduleSyncTasks;
     private final AsyncTaskExecutor asyncTaskExecutor;
     private final Lock workQueueLock;
+    private final Sleeper sleeper;
 
     private static final int MODULE_SYNC_BATCH_SIZE = 100;
     private static final long PREVENT_CPU_BURN_WAIT_TIME_MILLIS = 10;
@@ -59,9 +61,10 @@ public class ModuleSyncWatchdog {
      * Check DB for any cm handles in 'ADVISED' state.
      * Queue and create batches to process them asynchronously.
      * This method will only finish when there are no more 'ADVISED' cm handles in the DB.
-     * This method wil be triggered on a configurable interval
+     * This method is triggered on a configurable interval (ncmp.timers.advised-modules-sync.sleep-time-ms)
      */
-    @Scheduled(fixedDelayString = "${ncmp.timers.advised-modules-sync.sleep-time-ms:5000}")
+    @Scheduled(initialDelayString = "${test.ncmp.timers.advised-modules-sync.initial-delay-ms:0}",
+               fixedDelayString = "${ncmp.timers.advised-modules-sync.sleep-time-ms:5000}")
     public void moduleSyncAdvisedCmHandles() {
         log.debug("Processing module sync watchdog waking up.");
         populateWorkQueueIfNeeded();
@@ -82,16 +85,12 @@ public class ModuleSyncWatchdog {
         }
     }
 
-    private void preventBusyWait() {
-        try {
-            log.debug("Busy waiting now");
-            TimeUnit.MILLISECONDS.sleep(PREVENT_CPU_BURN_WAIT_TIME_MILLIS);
-        } catch (final InterruptedException e) {
-            Thread.currentThread().interrupt();
-        }
-    }
-
-    private void populateWorkQueueIfNeeded() {
+    /**
+     * Populate work queue with advised cm handles from db.
+     * This method is made public for (integration) testing purposes.
+     * So it can be tested without the queue being emptied immediately as the main public method does.
+     */
+    public void populateWorkQueueIfNeeded() {
         if (moduleSyncWorkQueue.isEmpty() && workQueueLock.tryLock()) {
             try {
                 populateWorkQueue();
@@ -154,4 +153,13 @@ public class ModuleSyncWatchdog {
         log.info("nextBatch size : {}", nextBatch.size());
         return nextBatch;
     }
+
+    private void preventBusyWait() {
+        try {
+            log.debug("Busy waiting now");
+            sleeper.haveALittleRest(PREVENT_CPU_BURN_WAIT_TIME_MILLIS);
+        } catch (final InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+    }
 }
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/Sleeper.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/Sleeper.java
new file mode 100644
index 0000000000..7a02fa06e0
--- /dev/null
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/Sleeper.java
@@ -0,0 +1,35 @@
+/*
+ *  ============LICENSE_START=======================================================
+ *  Copyright (C) 2024 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.ncmp.impl.utils;
+
+import java.util.concurrent.TimeUnit;
+import org.springframework.stereotype.Service;
+
+/**
+ * This class is to extract out sleep functionality so the interrupted exception handling can
+ * be covered with a test (e.g. using spy on Sleeper) and help to get to 100% code coverage.
+ */
+@Service
+public class Sleeper {
+    public void haveALittleRest(final long timeInMillis) throws InterruptedException {
+        TimeUnit.MILLISECONDS.sleep(timeInMillis);
+    }
+}
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy
index dcff2e9b89..70e26d993c 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy
@@ -87,7 +87,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'cm handle is in READY state'
             mockCmHandleQueries.cmHandleHasState('cmhandle-3', CmHandleState.READY) >> true
         when: 'registration is processed'
-            objectUnderTest.updateDmiRegistrationAndSyncModule(dmiRegistration)
+            objectUnderTest.updateDmiRegistration(dmiRegistration)
         then: 'cm-handles are removed first'
             1 * objectUnderTest.processRemovedCmHandles(*_)
         and: 'de-registered cm handle entry is removed from in progress map'
@@ -108,7 +108,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'exception while checking cm handle state'
             mockInventoryPersistence.getYangModelCmHandle('cmhandle-3') >> new YangModelCmHandle(id: 'cmhandle-3', moduleSetTag: '', compositeState: new CompositeState(cmHandleState: cmHandleState))
         when: 'registration is processed'
-            def result = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiRegistration)
+            def result = objectUnderTest.updateDmiRegistration(dmiRegistration)
         then: 'upgrade operation contains expected error code'
             assert result.upgradedCmHandles[0].status == expectedResponseStatus
         where: 'the following parameters are used'
@@ -124,7 +124,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'exception while checking cm handle state'
             mockInventoryPersistence.getYangModelCmHandle('cmhandle-3') >> { throw exception }
         when: 'registration is processed'
-            def result = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiRegistration)
+            def result = objectUnderTest.updateDmiRegistration(dmiRegistration)
         then: 'upgrade operation contains expected error code'
             assert result.upgradedCmHandles.ncmpResponseStatus.code[0] == expectedErrorCode
         where: 'the following parameters are used'
@@ -139,7 +139,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
                 dmiDataPlugin: dmiDataPlugin)
             dmiPluginRegistration.createdCmHandles = [ncmpServiceCmHandle]
         when: 'update registration and sync module is called with correct DMI plugin information'
-            objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'create cm handles registration and sync modules is called with the correct plugin information'
             1 * objectUnderTest.processCreatedCmHandles(dmiPluginRegistration, _)
         where:
@@ -155,7 +155,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
                 dmiDataPlugin: dmiDataPlugin)
             dmiPluginRegistration.createdCmHandles = [ncmpServiceCmHandle]
         when: 'registration is called with incorrect DMI plugin information'
-            objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a DMI Request Exception is thrown with correct message details'
             def exceptionThrown = thrown(DmiRequestException.class)
             assert exceptionThrown.getMessage().contains(expectedMessageDetails)
@@ -178,7 +178,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
             def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server')
             dmiPluginRegistration.createdCmHandles = [new NcmpServiceCmHandle(cmHandleId: 'cmhandle', dmiProperties: dmiProperties, publicProperties: publicProperties)]
         when: 'registration is updated'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a successful response is received'
             response.createdCmHandles.size() == 1
             with(response.createdCmHandles[0]) {
@@ -206,7 +206,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
             def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server',
                 createdCmHandles:[new NcmpServiceCmHandle(cmHandleId: 'ch-1', registrationTrustLevel: registrationTrustLevel)])
         when: 'registration is updated'
-            objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'trustLevel is set for the created cm-handle'
             1 * mockTrustLevelManager.registerCmHandles(expectedMapping)
         where:
@@ -225,7 +225,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
             def xpath = "somePathWithId[@id='cmhandle2']"
             mockLcmEventsCmHandleStateHandler.initiateStateAdvised(*_) >> { throw AlreadyDefinedException.forDataNodes([xpath], 'some-context') }
         when: 'registration is updated to create cm-handles'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a response is received for all cm-handles'
             response.createdCmHandles.size() == 1
         and: 'all cm-handles creation fails'
@@ -244,7 +244,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'cm-handler registration fails: #scenario'
             mockLcmEventsCmHandleStateHandler.initiateStateAdvised(*_) >> { throw exception }
         when: 'registration is updated'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a failure response is received'
             response.createdCmHandles.size() == 1
             with(response.createdCmHandles[0]) {
@@ -269,7 +269,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
                                            CmHandleRegistrationResponse.createFailureResponse('cm handle 4', CM_HANDLE_INVALID_ID)]
             mockNetworkCmProxyDataServicePropertyHandler.updateCmHandleProperties(_) >> updateOperationResponse
         when: 'registration is updated'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'the response contains updateOperationResponse'
             assert response.updatedCmHandles.size() == 4
             assert response.updatedCmHandles.containsAll(updateOperationResponse)
@@ -281,7 +281,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: '#scenario'
             mockCpsModuleService.deleteSchemaSetsWithCascade(_, ['cmhandle']) >>  { if (!schemaSetExist) { throw new SchemaSetNotFoundException('', '') } }
         when: 'registration is updated to delete cmhandle'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'the cmHandle state is updated to "DELETING"'
             1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >>
                 { args -> args[0].values()[0] == CmHandleState.DELETING }
@@ -315,7 +315,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'cm-handle deletion is successful for 1st and 3rd; failed for 2nd'
             mockInventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='cmhandle2']") >> { throw new RuntimeException("Failed") }
         when: 'registration is updated to delete cmhandles'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'the cmHandle states are all updated to "DELETING"'
             1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch({ assert it.every { entry -> entry.value == CmHandleState.DELETING } })
         and: 'a response is received for all cm-handles'
@@ -361,7 +361,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'schema set single deletion failed with unknown error'
             mockInventoryPersistence.deleteSchemaSetWithCascade(_) >> { throw new RuntimeException('Failed') }
         when: 'registration is updated to delete cmhandle'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'no exception is thrown'
             noExceptionThrown()
         and: 'cm-handle is not deleted'
@@ -387,7 +387,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'cm-handle deletion fails on individual delete'
             mockInventoryPersistence.deleteDataNode(_) >> { throw deleteListElementException }
         when: 'registration is updated to delete cmhandle'
-            def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'a failure response is received'
             assert response.removedCmHandles.size() == 1
             with(response.removedCmHandles[0]) {
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/NetworkCmProxyInventoryFacadeSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/NetworkCmProxyInventoryFacadeSpec.groovy
index fec07556eb..a7dd38cc15 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/NetworkCmProxyInventoryFacadeSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/NetworkCmProxyInventoryFacadeSpec.groovy
@@ -56,9 +56,9 @@ class NetworkCmProxyInventoryFacadeSpec extends Specification {
         given: 'an (updated) dmi plugin registration'
             def dmiPluginRegistration = Mock(DmiPluginRegistration)
         when: 'the registration is submitted '
-           objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+           objectUnderTest.updateDmiRegistration(dmiPluginRegistration)
         then: 'the call is delegated to the cm handle registration service'
-            1 * mockCmHandleRegistrationService.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+            1 * mockCmHandleRegistrationService.updateDmiRegistration(dmiPluginRegistration)
     }
 
     def 'Execute cm handle id search for inventory'() {
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy
index 3064a78ff9..f2c88a511e 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy
@@ -23,6 +23,7 @@ package org.onap.cps.ncmp.impl.inventory.sync
 
 import com.hazelcast.map.IMap
 import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle
+import org.onap.cps.ncmp.impl.utils.Sleeper
 import org.onap.cps.spi.model.DataNode
 import spock.lang.Specification
 
@@ -31,7 +32,7 @@ import java.util.concurrent.locks.Lock
 
 class ModuleSyncWatchdogSpec extends Specification {
 
-    def mockSyncUtils = Mock(ModuleOperationsUtils)
+    def mockModuleOperationsUtils = Mock(ModuleOperationsUtils)
 
     def static testQueueCapacity = 50 + 2 * ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE
 
@@ -45,16 +46,21 @@ class ModuleSyncWatchdogSpec extends Specification {
 
     def mockWorkQueueLock = Mock(Lock)
 
-    def objectUnderTest = new ModuleSyncWatchdog(mockSyncUtils, moduleSyncWorkQueue , mockModuleSyncStartedOnCmHandles, mockModuleSyncTasks, spiedAsyncTaskExecutor, mockWorkQueueLock)
+    def spiedSleeper = Spy(Sleeper)
+
+    def objectUnderTest = new ModuleSyncWatchdog(mockModuleOperationsUtils, moduleSyncWorkQueue , mockModuleSyncStartedOnCmHandles, mockModuleSyncTasks, spiedAsyncTaskExecutor, mockWorkQueueLock, spiedSleeper)
 
     void setup() {
         spiedAsyncTaskExecutor.setupThreadPool()
-        mockWorkQueueLock.tryLock() >> true
     }
 
     def 'Module sync advised cm handles with #scenario.'() {
-        given: 'sync utilities returns #numberOfAdvisedCmHandles advised cm handles'
-            mockSyncUtils.getAdvisedCmHandles() >> createDataNodes(numberOfAdvisedCmHandles)
+        given: 'module sync utilities returns #numberOfAdvisedCmHandles advised cm handles'
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(numberOfAdvisedCmHandles)
+        and: 'module sync utilities returns no failed (locked) cm handles'
+            mockModuleOperationsUtils.getCmHandlesThatFailedModelSyncOrUpgrade() >> []
+        and: 'the work queue is not locked'
+            mockWorkQueueLock.tryLock() >> true
         and: 'the executor has enough available threads'
             spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >> 3
         when: ' module sync is started'
@@ -63,6 +69,7 @@ class ModuleSyncWatchdogSpec extends Specification {
             expectedNumberOfTaskExecutions * spiedAsyncTaskExecutor.executeTask(*_)
         where: 'the following parameter are used'
             scenario              | numberOfAdvisedCmHandles                                          || expectedNumberOfTaskExecutions
+            'none at all'         | 0                                                                 || 0
             'less then 1 batch'   | 1                                                                 || 1
             'exactly 1 batch'     | ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE                         || 1
             '2 batches'           | 2 * ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE                     || 2
@@ -70,9 +77,11 @@ class ModuleSyncWatchdogSpec extends Specification {
             'over queue capacity' | testQueueCapacity + 2 * ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE || 3
     }
 
-    def 'Module sync advised cm handles starts with no available threads.'() {
-        given: 'sync utilities returns a advise cm handles'
-            mockSyncUtils.getAdvisedCmHandles() >> createDataNodes(1)
+    def 'Module sync cm handles starts with no available threads.'() {
+        given: 'module sync utilities returns a advise cm handles'
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(1)
+        and: 'the work queue is not locked'
+            mockWorkQueueLock.tryLock() >> true
         and: 'the executor first has no threads but has one thread on the second attempt'
             spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >>> [ 0, 1 ]
         when: ' module sync is started'
@@ -81,9 +90,11 @@ class ModuleSyncWatchdogSpec extends Specification {
             1 * spiedAsyncTaskExecutor.executeTask(*_)
     }
 
-    def 'Module sync advised cm handles already handled.'() {
-        given: 'sync utilities returns a advise cm handles'
-            mockSyncUtils.getAdvisedCmHandles() >> createDataNodes(1)
+    def 'Module sync advised cm handle already handled by other thread.'() {
+        given: 'module sync utilities returns an advised cm handle'
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(1)
+        and: 'the work queue is not locked'
+            mockWorkQueueLock.tryLock() >> true
         and: 'the executor has a thread available'
             spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >> 1
         and: 'the semaphore cache indicates the cm handle is already being processed'
@@ -98,7 +109,7 @@ class ModuleSyncWatchdogSpec extends Specification {
         given: 'there is still a cm handle in the queue'
             moduleSyncWorkQueue.offer(new DataNode())
         and: 'sync utilities returns many advise cm handles'
-            mockSyncUtils.getAdvisedCmHandles() >> createDataNodes(500)
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(500)
         and: 'the executor has plenty threads available'
             spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >> 10
         when: ' module sync is started'
@@ -108,18 +119,42 @@ class ModuleSyncWatchdogSpec extends Specification {
     }
 
     def 'Reset failed cm handles.'() {
-        given: 'sync utilities returns failed cm handles'
+        given: 'module sync utilities returns failed cm handles'
             def failedCmHandles = [new YangModelCmHandle()]
-            mockSyncUtils.getCmHandlesThatFailedModelSyncOrUpgrade() >> failedCmHandles
+            mockModuleOperationsUtils.getCmHandlesThatFailedModelSyncOrUpgrade() >> failedCmHandles
         when: 'reset failed cm handles is started'
             objectUnderTest.setPreviouslyLockedCmHandlesToAdvised()
         then: 'it is delegated to the module sync task (service)'
             1 * mockModuleSyncTasks.setCmHandlesToAdvised(failedCmHandles)
     }
 
+    def 'Module Sync Locking.'() {
+        given: 'module sync utilities returns an advised cm handle'
+            mockModuleOperationsUtils.getAdvisedCmHandles() >> createDataNodes(1)
+        and: 'can lock is : #canLock'
+            mockWorkQueueLock.tryLock() >> canLock
+        when: 'attempt to populate the work queue'
+            objectUnderTest.populateWorkQueueIfNeeded()
+        then: 'the queue remains empty is #expectQueueRemainsEmpty'
+            assert moduleSyncWorkQueue.isEmpty() == expectQueueRemainsEmpty
+        where: 'the following lock states are applied'
+            canLock | expectQueueRemainsEmpty
+            false   | true
+            true    | false
+    }
+
+    def 'Sleeper gets interrupted.'() {
+        given: 'sleeper gets interrupted'
+            spiedSleeper.haveALittleRest(_) >> { throw new InterruptedException() }
+        when: 'the watchdog attempts to sleep to save cpu cycles'
+            objectUnderTest.preventBusyWait()
+        then: 'no exception is thrown'
+            noExceptionThrown()
+    }
+
     def createDataNodes(numberOfDataNodes) {
         def dataNodes = []
-        (1..numberOfDataNodes).each {dataNodes.add(new DataNode())}
+        numberOfDataNodes.times { dataNodes.add(new DataNode()) }
         return dataNodes
     }
 }