Retry for yang module upgrade operation

- Added retry code for yang module upgrade failed

Issue-ID: CPS-1802

Signed-off-by: sourabh_sourabh <sourabh.sourabh@est.tech>
Change-Id: Ida68d45f926ed48b92f4b5a3b82e641d778303ce
Signed-off-by: sourabh_sourabh <sourabh.sourabh@est.tech>

5 files changed, 70 insertions, 49 deletions

diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdogSpec.groovy
index 65dfc0579..0ffb56791 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdogSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/DataSyncWatchdogSpec.groovy
@@ -24,8 +24,6 @@ import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NFP_OPE
 
 import com.hazelcast.map.IMap
 import org.onap.cps.api.CpsDataService
-import org.onap.cps.ncmp.api.impl.inventory.sync.DataSyncWatchdog
-import org.onap.cps.ncmp.api.impl.inventory.sync.SyncUtils
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
 import org.onap.cps.ncmp.api.impl.inventory.CmHandleState
 import org.onap.cps.ncmp.api.impl.inventory.CompositeState
@@ -39,7 +37,7 @@ class DataSyncWatchdogSpec extends Specification {
 
     def mockCpsDataService = Mock(CpsDataService)
 
-    def mockSyncUtils = Mock(SyncUtils)
+    def mockSyncUtils = Mock(ModuleOperationsUtils)
 
     def mockDataSyncSemaphores = Mock(IMap<String,Boolean>)
 
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtilsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy
index 025d9482d..099fc5ac1 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/SyncUtilsSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleOperationsUtilsSpec.groovy
@@ -22,15 +22,14 @@
 package org.onap.cps.ncmp.api.impl.inventory.sync
 
 import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.LOCKED_MISBEHAVING
+import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE
 import static org.onap.cps.ncmp.api.impl.operations.DatastoreType.PASSTHROUGH_OPERATIONAL
 import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_SYNC_FAILED
-import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE
 import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE_FAILED
 
 import ch.qos.logback.classic.Level
 import ch.qos.logback.classic.Logger
 import ch.qos.logback.core.read.ListAppender
-import org.onap.cps.ncmp.api.impl.inventory.sync.SyncUtils
 import org.slf4j.LoggerFactory
 import org.springframework.context.annotation.AnnotationConfigApplicationContext
 import com.fasterxml.jackson.databind.JsonNode
@@ -51,7 +50,7 @@ import java.time.OffsetDateTime
 import java.time.format.DateTimeFormatter
 import java.util.stream.Collectors
 
-class SyncUtilsSpec extends Specification{
+class ModuleOperationsUtilsSpec extends Specification{
 
     def mockCmHandleQueries = Mock(CmHandleQueries)
 
@@ -59,7 +58,7 @@ class SyncUtilsSpec extends Specification{
 
     def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper())
 
-    def objectUnderTest = new SyncUtils(mockCmHandleQueries, mockDmiDataOperations, jsonObjectMapper)
+    def objectUnderTest = new ModuleOperationsUtils(mockCmHandleQueries, mockDmiDataOperations, jsonObjectMapper)
 
     def static neverUpdatedBefore = '1900-01-01T00:00:00.000+0100'
 
@@ -71,7 +70,7 @@ class SyncUtilsSpec extends Specification{
 
     def applicationContext = new AnnotationConfigApplicationContext()
 
-    def logger = (Logger) LoggerFactory.getLogger(SyncUtils)
+    def logger = (Logger) LoggerFactory.getLogger(ModuleOperationsUtils)
     def loggingListAppender
 
     void setup() {
@@ -83,7 +82,7 @@ class SyncUtilsSpec extends Specification{
     }
 
     void cleanup() {
-        ((Logger) LoggerFactory.getLogger(SyncUtils.class)).detachAndStopAllAppenders()
+        ((Logger) LoggerFactory.getLogger(ModuleOperationsUtils.class)).detachAndStopAllAppenders()
         applicationContext.close()
     }
 
@@ -107,7 +106,7 @@ class SyncUtilsSpec extends Specification{
             objectUnderTest.updateLockReasonDetailsAndAttempts(compositeState, MODULE_SYNC_FAILED, 'new error message')
         then: 'the composite state lock reason and details are updated'
             assert compositeState.lockReason.lockReasonCategory == MODULE_SYNC_FAILED
-            assert compositeState.lockReason.details == expectedDetails
+            assert compositeState.lockReason.details.contains(expectedDetails)
         where:
             scenario         | lockReason                                                                                   || expectedDetails
             'does not exist' | null                                                                                         || 'Attempt #1 failed: new error message'
@@ -143,28 +142,28 @@ class SyncUtilsSpec extends Specification{
             def logs = loggingListAppender.list.toString()
             assert logs.contains(logReason)
         where: 'the following parameters are used'
-            scenario                                    | lastUpdateMinutesAgo | lockDetails          | logReason                               || retryExpected
-            'never attempted before'                    | -1                   | 'fist attempt:'      | 'First Attempt:'                        || true
-            '1st attempt, last attempt > 2 minute ago'  | 3                    | 'Attempt #1 failed:' | 'Retry due now'                         || true
-            '2nd attempt, last attempt < 4 minutes ago' | 1                    | 'Attempt #2 failed:' | 'Time until next attempt is 3 minutes:' || false
-            '2nd attempt, last attempt > 4 minutes ago' | 5                    | 'Attempt #2 failed:' | 'Retry due now'                         || true
+            scenario                                    | lastUpdateMinutesAgo | lockDetails                     | logReason                               || retryExpected
+            'never attempted before'                    | -1                   | 'Fist attempt:'                 | 'First Attempt:'                        || true
+            '1st attempt, last attempt > 2 minute ago'  | 3                    | 'Attempt #1 failed: some error' | 'Retry due now'                         || true
+            '2nd attempt, last attempt < 4 minutes ago' | 1                    | 'Attempt #2 failed: some error' | 'Time until next attempt is 3 minutes:' || false
+            '2nd attempt, last attempt > 4 minutes ago' | 5                    | 'Attempt #2 failed: some error' | 'Retry due now'                         || true
     }
 
-    def 'Retry Locked Cm-Handle with other lock reasons (category) #lockReasonCategory'() {
+    def 'Retry Locked Cm-Handle with lock reasons (category) #lockReasonCategory'() {
         when: 'checking to see if cm handle is ready for retry'
-        def result = objectUnderTest.needsModuleSyncRetryOrUpgrade(new CompositeStateBuilder()
+            def result = objectUnderTest.needsModuleSyncRetryOrUpgrade(new CompositeStateBuilder()
                 .withLockReason(lockReasonCategory, 'some details')
                 .withLastUpdatedTime(nowAsString).build())
         then: 'verify retry attempts'
-        assert result == retryAttempt
+            assert !result
         and: 'logs contain related information'
-        def logs = loggingListAppender.list.toString()
-        assert logs.contains(logReason)
+            def logs = loggingListAppender.list.toString()
+            assert logs.contains(logReason)
         where: 'the following lock reasons occurred'
-        scenario             | lockReasonCategory || logReason                    | retryAttempt
-        'module upgrade'     | MODULE_UPGRADE     || 'Locked for module upgrade.' | true
-        'module sync failed' | MODULE_SYNC_FAILED || 'First Attempt:'             | false
-        'lock misbehaving'   | LOCKED_MISBEHAVING || 'Locked for other reason'    | false
+            scenario             | lockReasonCategory    || logReason
+            'module upgrade'     | MODULE_UPGRADE_FAILED || 'First Attempt:'
+            'module sync failed' | MODULE_SYNC_FAILED    || 'First Attempt:'
+            'lock misbehaving'   | LOCKED_MISBEHAVING    || 'Locked for other reason'
     }
 
     def 'Get a Cm-Handle where #scenario'() {
@@ -195,4 +194,18 @@ class SyncUtilsSpec extends Specification{
         then: 'the returned data is correct'
             result == jsonString
     }
+
+    def 'Extract module set tag and number of attempt when lock reason contains #scenario'() {
+        expect: 'lock reason details are extracted correctly'
+            def result = objectUnderTest.getLockedCompositeStateDetails(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, lockReasonDetails).build().lockReason)
+        and: 'the result contains the correct moduleSetTag'
+            assert result['moduleSetTag'] == expectedModuleSetTag
+        and: 'the result contains the correct number of attempts'
+            assert result['attempt'] == expectedNumberOfAttempts
+        where: 'the following scenarios are used'
+            scenario                                     | lockReasonDetails                                                           || expectedModuleSetTag | expectedNumberOfAttempts
+            'module set tag only'                        | 'Upgrade to ModuleSetTag: targetModuleSetTag'                               || 'targetModuleSetTag' | null
+            'number of attempts only'                    | 'Attempt #1 failed: some error'                                             || null                 | '1'
+            'number of attempts and module set tag both' | 'Upgrade to ModuleSetTag: targetModuleSetTag Attempt #1 failed: some error' || 'targetModuleSetTag' | '1'
+    }
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncServiceSpec.groovy
index 18b87af9a..5384f31cc 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncServiceSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncServiceSpec.groovy
@@ -20,7 +20,6 @@
 
 package org.onap.cps.ncmp.api.impl.inventory.sync
 
-import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.LOCKED_MISBEHAVING
 import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME
 import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE
 
@@ -31,7 +30,6 @@ import org.onap.cps.api.CpsAdminService
 import org.onap.cps.api.CpsDataService
 import org.onap.cps.api.CpsModuleService
 import org.onap.cps.spi.model.DataNodeBuilder
-import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncService
 import org.onap.cps.ncmp.api.impl.operations.DmiModelOperations
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
 import org.onap.cps.ncmp.api.impl.inventory.CmHandleQueries
@@ -90,7 +88,7 @@ class ModuleSyncServiceSpec extends Specification {
     def 'Upgrade model for an existing cm handle with Module Set Tag where the modules are #scenario'() {
         given: 'a cm handle being upgraded to module set tag: tag-1'
             def ncmpServiceCmHandle = new NcmpServiceCmHandle()
-            ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, 'new moduleSetTag: tag-1').build())
+            ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, 'Upgrade to ModuleSetTag: tag-1').build())
             def dmiServiceName = 'some service name'
             ncmpServiceCmHandle.cmHandleId = 'upgraded-ch'
             def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(dmiServiceName, '', '', ncmpServiceCmHandle,'tag-1')
@@ -98,7 +96,7 @@ class ModuleSyncServiceSpec extends Specification {
             def moduleReferences =  [ new ModuleReference('module1','1') ]
         and: 'cache or DMI operations returns some module references for upgraded cm handle'
             if (populateCache) {
-                mockModuleSetTagCache.put('tag-1',moduleReferences)
+                mockModuleSetTagCache.put('tag-1', moduleReferences)
             } else {
                 mockDmiModelOperations.getModuleReferences(yangModelCmHandle) >> moduleReferences
             }
@@ -127,7 +125,7 @@ class ModuleSyncServiceSpec extends Specification {
         given: 'a cm handle that is ready but locked for upgrade'
             def ncmpServiceCmHandle = new NcmpServiceCmHandle()
             ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder()
-                .withLockReason(MODULE_UPGRADE, 'new moduleSetTag: targetModuleSetTag').build())
+                .withLockReason(MODULE_UPGRADE, 'Upgrade to ModuleSetTag: targetModuleSetTag').build())
             ncmpServiceCmHandle.setCmHandleId('cmHandleId-1')
             def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle('some service name', '', '', ncmpServiceCmHandle, 'targetModuleSetTag')
             mockCmHandleQueries.cmHandleHasState('cmHandleId-1', CmHandleState.READY) >> true
@@ -170,11 +168,6 @@ class ModuleSyncServiceSpec extends Specification {
             result == unsupportedOperationException
     }
 
-    def 'Extract module set tag'() {
-        expect: 'the module set tag is extracted correctly'
-            assert 'targetModuleSetTag' == objectUnderTest.extractModuleSetTag(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, 'new moduleSetTag: targetModuleSetTag').build())
-    }
-
     def toModuleReferences(moduleReferenceAsMap) {
         def moduleReferences = [].withDefault { [:] }
         moduleReferenceAsMap.forEach(property ->
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasksSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasksSpec.groovy
index 0d927551d..3bdac1855 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasksSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncTasksSpec.groovy
@@ -21,6 +21,10 @@
 
 package org.onap.cps.ncmp.api.impl.inventory.sync
 
+
+import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_SYNC_FAILED
+import static org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory.MODULE_UPGRADE_FAILED
+
 import ch.qos.logback.classic.Level
 import ch.qos.logback.classic.Logger
 import ch.qos.logback.classic.spi.ILoggingEvent
@@ -31,9 +35,6 @@ import com.hazelcast.map.IMap
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.BeforeEach
 import org.onap.cps.ncmp.api.impl.events.lcm.LcmEventsCmHandleStateHandler
-import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncService
-import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncTasks
-import org.onap.cps.ncmp.api.impl.inventory.sync.SyncUtils
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
 import org.onap.cps.ncmp.api.impl.inventory.CmHandleState
 import org.onap.cps.ncmp.api.impl.inventory.CompositeState
@@ -62,7 +63,7 @@ class ModuleSyncTasksSpec extends Specification {
 
     def mockInventoryPersistence = Mock(InventoryPersistence)
 
-    def mockSyncUtils = Mock(SyncUtils)
+    def mockSyncUtils = Mock(ModuleOperationsUtils)
 
     def mockModuleSyncService = Mock(ModuleSyncService)
 
@@ -79,8 +80,8 @@ class ModuleSyncTasksSpec extends Specification {
 
     def 'Module Sync ADVISED cm handles.'() {
         given: 'cm handles in an ADVISED state'
-            def cmHandle1 = advisedCmHandleAsDataNode('cm-handle-1')
-            def cmHandle2 = advisedCmHandleAsDataNode('cm-handle-2')
+            def cmHandle1 = cmHandleAsDataNodeByIdAndState('cm-handle-1', CmHandleState.ADVISED)
+            def cmHandle2 = cmHandleAsDataNodeByIdAndState('cm-handle-2', CmHandleState.ADVISED)
         and: 'the inventory persistence cm handle returns a ADVISED state for the any handle'
             mockInventoryPersistence.getCmHandleState(_) >> new CompositeState(cmHandleState: CmHandleState.ADVISED)
         when: 'module sync poll is executed'
@@ -101,7 +102,7 @@ class ModuleSyncTasksSpec extends Specification {
 
     def 'Module Sync ADVISED cm handle with failure during sync.'() {
         given: 'a cm handle in an ADVISED state'
-            def cmHandle = advisedCmHandleAsDataNode('cm-handle')
+            def cmHandle = cmHandleAsDataNodeByIdAndState('cm-handle', CmHandleState.ADVISED)
         and: 'the inventory persistence cm handle returns a ADVISED state for the cm handle'
             def cmHandleState = new CompositeState(cmHandleState: CmHandleState.ADVISED)
             1 * mockInventoryPersistence.getCmHandleState('cm-handle') >> cmHandleState
@@ -110,7 +111,7 @@ class ModuleSyncTasksSpec extends Specification {
         when: 'module sync is executed'
             objectUnderTest.performModuleSync([cmHandle], batchCount)
         then: 'update lock reason, details and attempts is invoked'
-            1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(cmHandleState, LockReasonCategory.MODULE_SYNC_FAILED, 'some exception')
+            1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(cmHandleState, MODULE_SYNC_FAILED, 'some exception')
         and: 'the state handler is called to update the state to LOCKED'
             1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> { args ->
                 assertBatch(args, ['cm-handle'], CmHandleState.LOCKED)
@@ -119,6 +120,25 @@ class ModuleSyncTasksSpec extends Specification {
             assert batchCount.get() == 4
     }
 
+    def 'Failed cm handle during #scenario.'() {
+        given: 'a cm handle in LOCKED state'
+            def cmHandle = cmHandleAsDataNodeByIdAndState('cm-handle', CmHandleState.LOCKED)
+        and: 'the inventory persistence cm handle returns a LOCKED state with reason for the cm handle'
+            def expectedCmHandleState = new CompositeState(cmHandleState: cmHandleState, lockReason: CompositeState
+                .LockReason.builder().lockReasonCategory(lockReasonCategory).details(lockReasonDetails).build())
+            1 * mockInventoryPersistence.getCmHandleState('cm-handle') >> expectedCmHandleState
+        and: 'module sync service attempts to sync/upgrade the cm handle and throws an exception'
+            1 * mockModuleSyncService.syncAndCreateOrUpgradeSchemaSetAndAnchor(*_) >> { throw new Exception('some exception') }
+        when: 'module sync is executed'
+            objectUnderTest.performModuleSync([cmHandle], batchCount)
+        then: 'update lock reason, details and attempts is invoked'
+            1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(expectedCmHandleState, expectedLockReasonCategory, 'some exception')
+        where:
+            scenario         | cmHandleState        | lockReasonCategory    | lockReasonDetails                              || expectedLockReasonCategory
+            'module upgrade' | CmHandleState.LOCKED | MODULE_UPGRADE_FAILED | 'Upgrade to ModuleSetTag: some-module-set-tag' || MODULE_UPGRADE_FAILED
+            'module sync'    | CmHandleState.LOCKED | MODULE_SYNC_FAILED    | 'some lock details'                            || MODULE_SYNC_FAILED
+    }
+
     def 'Reset failed CM Handles #scenario.'() {
         given: 'cm handles in an locked state'
             def lockedState = new CompositeStateBuilder().withCmHandleState(CmHandleState.LOCKED)
@@ -147,7 +167,7 @@ class ModuleSyncTasksSpec extends Specification {
 
     def 'Module Sync ADVISED cm handle without entry in progress map.'() {
         given: 'cm handles in an ADVISED state'
-            def cmHandle1 = advisedCmHandleAsDataNode('cm-handle-1')
+            def cmHandle1 = cmHandleAsDataNodeByIdAndState('cm-handle-1', CmHandleState.ADVISED)
         and: 'the inventory persistence cm handle returns a ADVISED state for the any handle'
             mockInventoryPersistence.getCmHandleState(_) >> new CompositeState(cmHandleState: CmHandleState.ADVISED)
         and: 'entry in progress map for other cm handle'
@@ -183,8 +203,8 @@ class ModuleSyncTasksSpec extends Specification {
             assert loggingEvent == null
     }
 
-    def advisedCmHandleAsDataNode(cmHandleId) {
-        return new DataNode(anchorName: cmHandleId, leaves: ['id': cmHandleId, 'cm-handle-state': 'ADVISED'])
+    def cmHandleAsDataNodeByIdAndState(cmHandleId, cmHandleState) {
+        return new DataNode(anchorName: cmHandleId, leaves: ['id': cmHandleId, 'cm-handle-state': cmHandleState])
     }
 
     def assertYamgModelCmHandleArgument(args, expectedCmHandleId) {
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy
index 7425a52ff..1752a17a1 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy
@@ -22,9 +22,6 @@
 package org.onap.cps.ncmp.api.impl.inventory.sync
 
 import com.hazelcast.map.IMap
-import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncTasks
-import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncWatchdog
-import org.onap.cps.ncmp.api.impl.inventory.sync.SyncUtils
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
 import org.onap.cps.ncmp.api.impl.inventory.sync.executor.AsyncTaskExecutor
 import java.util.concurrent.ArrayBlockingQueue
@@ -33,7 +30,7 @@ import spock.lang.Specification
 
 class ModuleSyncWatchdogSpec extends Specification {
 
-    def mockSyncUtils = Mock(SyncUtils)
+    def mockSyncUtils = Mock(ModuleOperationsUtils)
 
     def static testQueueCapacity = 50 + 2 * ModuleSyncWatchdog.MODULE_SYNC_BATCH_SIZE