Hazelcast Optimisation

Issue-ID: CPS-2420
Change-Id: I707e0fbbddeb4ddc7c573a2b3ebd8fbca08126a6
Signed-off-by: emaclee <lee.anjella.macabuhay@est.tech>

13 files changed, 89 insertions, 54 deletions

diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java
index 770dde1c08..75007e2e35 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java
@@ -23,6 +23,7 @@ package org.onap.cps.ncmp.impl.cache;
 import com.hazelcast.config.Config;
 import com.hazelcast.config.MapConfig;
 import com.hazelcast.config.NamedConfig;
+import com.hazelcast.config.NearCacheConfig;
 import com.hazelcast.config.QueueConfig;
 import com.hazelcast.config.RestEndpointGroup;
 import com.hazelcast.config.SetConfig;
@@ -94,6 +95,21 @@ public class HazelcastCacheConfig {
         return mapConfig;
     }
 
+    protected static MapConfig createMapConfigWithTimeToLiveInSeconds(final String configName,
+                                                                      final int timeToLiveInSeconds) {
+        final MapConfig mapConfig = new MapConfig(configName);
+        mapConfig.setBackupCount(1);
+        mapConfig.setTimeToLiveSeconds(timeToLiveInSeconds);
+        return mapConfig;
+    }
+
+    protected static MapConfig createNearCacheMapConfig(final String configName) {
+        final MapConfig mapConfig = new MapConfig(configName);
+        mapConfig.setBackupCount(1);
+        mapConfig.setNearCacheConfig(new NearCacheConfig(configName));
+        return mapConfig;
+    }
+
     protected static QueueConfig createQueueConfig(final String configName) {
         final QueueConfig commonQueueConfig = new QueueConfig(configName);
         commonQueueConfig.setBackupCount(1);
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java
index 5610013863..c1291632de 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java
@@ -26,6 +26,7 @@ import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DATASPACE_NA
 import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_ANCHOR;
 import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_PARENT;
 
+import com.hazelcast.map.IMap;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -59,10 +60,10 @@ public class CmHandleQueryServiceImpl implements CmHandleQueryService {
     private final CpsQueryService cpsQueryService;
 
     @Qualifier(TrustLevelCacheConfig.TRUST_LEVEL_PER_DMI_PLUGIN)
-    private final Map<String, TrustLevel> trustLevelPerDmiPlugin;
+    private final IMap<String, TrustLevel> trustLevelPerDmiPlugin;
 
     @Qualifier(TrustLevelCacheConfig.TRUST_LEVEL_PER_CM_HANDLE)
-    private final Map<String, TrustLevel> trustLevelPerCmHandleId;
+    private final IMap<String, TrustLevel> trustLevelPerCmHandleId;
 
     private final CpsValidator cpsValidator;
 
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java
index daac63fce4..281d64edb4 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java
@@ -319,7 +319,8 @@ public class CmHandleRegistrationService {
 
     // CPS-1239 Robustness cleaning of in progress cache
     private void removeDeletedCmHandleFromModuleSyncMap(final String cmHandleId) {
-        if (moduleSyncStartedOnCmHandles.remove(cmHandleId) != null) {
+        if (moduleSyncStartedOnCmHandles.containsKey(cmHandleId)) {
+            moduleSyncStartedOnCmHandles.removeAsync(cmHandleId);
             log.debug("{} removed from in progress map", cmHandleId);
         }
     }
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java
index c97b284bf1..0d618cfa40 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java
@@ -125,7 +125,8 @@ public class ModuleSyncTasks {
     }
 
     private void removeResetCmHandleFromModuleSyncMap(final String resetCmHandleId) {
-        if (moduleSyncStartedOnCmHandles.remove(resetCmHandleId) != null) {
+        if (moduleSyncStartedOnCmHandles.containsKey(resetCmHandleId)) {
+            moduleSyncStartedOnCmHandles.removeAsync(resetCmHandleId);
             log.info("{} removed from in progress map", resetCmHandleId);
         }
     }
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java
index 74bef43d0b..3f2bb4f4ef 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java
@@ -139,8 +139,7 @@ public class ModuleSyncWatchdog {
         log.info("nextBatchCandidates size : {}", nextBatchCandidates.size());
         for (final String cmHandleId : nextBatchCandidates) {
             final boolean alreadyAddedToInProgressMap = VALUE_FOR_HAZELCAST_IN_PROGRESS_MAP.equals(
-                    moduleSyncStartedOnCmHandles.putIfAbsent(cmHandleId, VALUE_FOR_HAZELCAST_IN_PROGRESS_MAP,
-                            SynchronizationCacheConfig.MODULE_SYNC_STARTED_TTL_SECS, TimeUnit.SECONDS));
+                    moduleSyncStartedOnCmHandles.putIfAbsent(cmHandleId, VALUE_FOR_HAZELCAST_IN_PROGRESS_MAP));
             if (alreadyAddedToInProgressMap) {
                 log.info("module sync for {} already in progress by other instance", cmHandleId);
             } else {
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java
index 671e791ac2..def8f37eb7 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java
@@ -43,7 +43,8 @@ public class SynchronizationCacheConfig extends HazelcastCacheConfig {
     public static final int DATA_SYNC_SEMAPHORE_TTL_SECS = 1800;
 
     private static final QueueConfig commonQueueConfig = createQueueConfig("defaultQueueConfig");
-    private static final MapConfig moduleSyncStartedConfig = createMapConfig("moduleSyncStartedConfig");
+    private static final MapConfig moduleSyncStartedConfig =
+            createMapConfigWithTimeToLiveInSeconds("moduleSyncStartedConfig", MODULE_SYNC_STARTED_TTL_SECS);
     private static final MapConfig dataSyncSemaphoresConfig = createMapConfig("dataSyncSemaphoresConfig");
     private static final SetConfig moduleSetTagsBeingProcessedConfig
         = createSetConfig("moduleSetTagsBeingProcessedConfig");
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/DmiPluginTrustLevelWatchDog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/DmiPluginTrustLevelWatchDog.java
index aca485f53d..044e2b5553 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/DmiPluginTrustLevelWatchDog.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/DmiPluginTrustLevelWatchDog.java
@@ -20,8 +20,8 @@
 
 package org.onap.cps.ncmp.impl.inventory.trustlevel;
 
+import com.hazelcast.map.IMap;
 import java.util.Collection;
-import java.util.Map;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.onap.cps.ncmp.api.inventory.models.TrustLevel;
@@ -43,7 +43,7 @@ public class DmiPluginTrustLevelWatchDog {
     private final TrustLevelManager trustLevelManager;
 
     @Qualifier(TrustLevelCacheConfig.TRUST_LEVEL_PER_DMI_PLUGIN)
-    private final Map<String, TrustLevel> trustLevelPerDmiPlugin;
+    private final IMap<String, TrustLevel> trustLevelPerDmiPlugin;
 
     /**
      * This class monitors the trust level of all DMI plugin by checking the health status
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelCacheConfig.java
index f9ad3ff937..a11dec7ba6 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelCacheConfig.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelCacheConfig.java
@@ -34,7 +34,7 @@ public class TrustLevelCacheConfig extends HazelcastCacheConfig {
 
     public static final String TRUST_LEVEL_PER_CM_HANDLE = "trustLevelPerCmHandle";
     private static final MapConfig trustLevelPerCmHandleIdCacheConfig =
-            createMapConfig("trustLevelPerCmHandleCacheConfig");
+            createNearCacheMapConfig("trustLevelPerCmHandleCacheConfig");
 
     private static final MapConfig trustLevelPerDmiPluginCacheConfig =
             createMapConfig("trustLevelPerDmiPluginCacheConfig");
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManager.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManager.java
index b61e53854e..6e896620c6 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManager.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManager.java
@@ -64,7 +64,7 @@ public class TrustLevelManager {
     public void registerDmiPlugin(final DmiPluginRegistration dmiPluginRegistration) {
         final String dmiServiceName = DmiServiceNameResolver.resolveDmiServiceName(RequiredDmiService.DATA,
                 dmiPluginRegistration);
-        trustLevelPerDmiPlugin.put(dmiServiceName, TrustLevel.COMPLETE);
+        trustLevelPerDmiPlugin.putAsync(dmiServiceName, TrustLevel.COMPLETE);
     }
 
     /**
@@ -73,24 +73,22 @@ public class TrustLevelManager {
      * @param cmHandlesToBeCreated a list of cmHandles being created
      */
     public void registerCmHandles(final Map<String, TrustLevel> cmHandlesToBeCreated) {
+        final Map<String, TrustLevel> trustLevelPerCmHandleIdForCache = new HashMap<>();
         for (final Map.Entry<String, TrustLevel> entry : cmHandlesToBeCreated.entrySet()) {
             final String cmHandleId = entry.getKey();
-            if (trustLevelPerCmHandleId.containsKey(cmHandleId)) {
-                log.warn("Cm handle: {} already registered", cmHandleId);
-            } else {
-                TrustLevel initialTrustLevel = entry.getValue();
-                if (initialTrustLevel == null) {
-                    initialTrustLevel = TrustLevel.COMPLETE;
-                }
-                trustLevelPerCmHandleId.put(cmHandleId, initialTrustLevel);
-                if (TrustLevel.NONE.equals(initialTrustLevel)) {
-                    cmAvcEventPublisher.publishAvcEvent(cmHandleId,
+            TrustLevel initialTrustLevel = entry.getValue();
+            if (initialTrustLevel == null) {
+                initialTrustLevel = TrustLevel.COMPLETE;
+            }
+            trustLevelPerCmHandleIdForCache.put(cmHandleId, initialTrustLevel);
+            if (TrustLevel.NONE.equals(initialTrustLevel)) {
+                cmAvcEventPublisher.publishAvcEvent(cmHandleId,
                         AVC_CHANGED_ATTRIBUTE_NAME,
                         AVC_NO_OLD_VALUE,
                         initialTrustLevel.name());
-                }
             }
         }
+        trustLevelPerCmHandleId.putAllAsync(trustLevelPerCmHandleIdForCache);
     }
 
     /**
@@ -105,7 +103,7 @@ public class TrustLevelManager {
                           final Collection<String> affectedCmHandleIds,
                           final TrustLevel newDmiTrustLevel) {
         final TrustLevel oldDmiTrustLevel  = trustLevelPerDmiPlugin.get(dmiServiceName);
-        trustLevelPerDmiPlugin.put(dmiServiceName, newDmiTrustLevel);
+        trustLevelPerDmiPlugin.putAsync(dmiServiceName, newDmiTrustLevel);
         for (final String affectedCmHandleId : affectedCmHandleIds) {
             final TrustLevel cmHandleTrustLevel = trustLevelPerCmHandleId.get(affectedCmHandleId);
             final TrustLevel oldEffectiveTrustLevel = cmHandleTrustLevel.getEffectiveTrustLevel(oldDmiTrustLevel);
@@ -131,7 +129,7 @@ public class TrustLevelManager {
         final TrustLevel oldEffectiveTrustLevel = oldCmHandleTrustLevel.getEffectiveTrustLevel(dmiTrustLevel);
         final TrustLevel newEffectiveTrustLevel = newCmHandleTrustLevel.getEffectiveTrustLevel(dmiTrustLevel);
 
-        trustLevelPerCmHandleId.put(cmHandleId, newCmHandleTrustLevel);
+        trustLevelPerCmHandleId.putAsync(cmHandleId, newCmHandleTrustLevel);
         sendAvcNotificationIfRequired(cmHandleId, oldEffectiveTrustLevel, newEffectiveTrustLevel);
     }
 
@@ -174,10 +172,11 @@ public class TrustLevelManager {
      * @param cmHandleIds       cm handle ids to be removed from the cache
      */
     public void removeCmHandles(final Collection<String> cmHandleIds) {
-        for (final String cmHandleId : cmHandleIds) {
-            if (trustLevelPerCmHandleId.remove(cmHandleId) == null) {
-                log.debug("Removed Cm handle: {} is not in trust level cache", cmHandleId);
-            }
+        final Set<String> cmHandlesToRemove = trustLevelPerCmHandleId.keySet().stream()
+                .filter(cmHandleIds::contains)
+                .collect(Collectors.toSet());
+        for (final String cmHandleId : cmHandlesToRemove) {
+            trustLevelPerCmHandleId.removeAsync(cmHandleId);
         }
     }
 
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy
index d19081cee5..7ac2b2cded 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy
@@ -21,6 +21,9 @@
 
 package org.onap.cps.ncmp.impl.inventory
 
+import com.hazelcast.config.Config
+import com.hazelcast.core.Hazelcast
+import com.hazelcast.instance.impl.HazelcastInstanceFactory
 import org.onap.cps.api.CpsDataService
 import org.onap.cps.api.CpsQueryService
 import org.onap.cps.impl.utils.CpsValidator
@@ -39,11 +42,17 @@ class CmHandleQueryServiceImplSpec extends Specification {
 
     def mockCpsQueryService = Mock(CpsQueryService)
     def mockCpsDataService = Mock(CpsDataService)
-    def trustLevelPerDmiPlugin = [:]
-    def trustLevelPerCmHandleId = [ 'PNFDemo': TrustLevel.COMPLETE, 'PNFDemo2': TrustLevel.NONE, 'PNFDemo4': TrustLevel.NONE ]
+    def trustLevelPerDmiPlugin = HazelcastInstanceFactory
+        .getOrCreateHazelcastInstance(new Config('hazelcastInstanceName'))
+        .getMap('trustLevelPerDmiPlugin')
+    def trustLevelPerCmHandleId = HazelcastInstanceFactory
+        .getOrCreateHazelcastInstance(new Config('hazelcastInstanceName'))
+        .getMap('trustLevelPerCmHandleId')
     def mockCpsValidator = Mock(CpsValidator)
 
-    def objectUnderTest = new CmHandleQueryServiceImpl(mockCpsDataService, mockCpsQueryService, trustLevelPerDmiPlugin, trustLevelPerCmHandleId, mockCpsValidator)
+
+    def objectUnderTest = new CmHandleQueryServiceImpl(mockCpsDataService, mockCpsQueryService,
+        trustLevelPerDmiPlugin, trustLevelPerCmHandleId, mockCpsValidator)
 
     def static sampleDataNodes = [new DataNode(xpath: "/dmi-registry/cm-handles[@id='ch-1']"),
                                   new DataNode(xpath: "/dmi-registry/cm-handles[@id='ch-2']")]
@@ -56,6 +65,16 @@ class CmHandleQueryServiceImplSpec extends Specification {
     def static pnfDemo4 = createDataNode('PNFDemo4')
     def static pnfDemo5 = createDataNode('PNFDemo5')
 
+    def setup() {
+        trustLevelPerCmHandleId.put("PNFDemo", TrustLevel.COMPLETE)
+        trustLevelPerCmHandleId.put("PNFDemo2", TrustLevel.NONE)
+        trustLevelPerCmHandleId.put("PNFDemo4", TrustLevel.NONE)
+    }
+
+    def cleanupSpec() {
+        Hazelcast.getHazelcastInstanceByName('hazelcastInstanceName').shutdown()
+    }
+
     def 'Query CmHandles with public properties query pair.'() {
         given: 'the DataNodes queried for a given cpsPath are returned from the persistence service.'
             mockResponses()
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy
index 67778fc0a3..faa193ea33 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy
@@ -64,7 +64,7 @@ class CmHandleRegistrationServiceSpec extends Specification {
 
     def objectUnderTest = Spy(new CmHandleRegistrationService(
         mockNetworkCmProxyDataServicePropertyHandler, mockInventoryPersistence, mockCpsDataService, mockLcmEventsCmHandleStateHandler,
-        mockModuleSyncStartedOnCmHandles, mockTrustLevelManager, mockAlternateIdChecker))
+        mockModuleSyncStartedOnCmHandles as IMap<String, Object>, mockTrustLevelManager, mockAlternateIdChecker))
 
     def setup() {
         // always accept all cm handles
@@ -86,14 +86,14 @@ class CmHandleRegistrationServiceSpec extends Specification {
             mockInventoryPersistence.getYangModelCmHandle('cmhandle-3') >> new YangModelCmHandle(id: 'cmhandle-3', moduleSetTag: '', compositeState: new CompositeState(cmHandleState: CmHandleState.READY))
         and: 'cm handle is in READY state'
             mockCmHandleQueries.cmHandleHasState('cmhandle-3', CmHandleState.READY) >> true
+        and: 'cm handle to be removed is in progress map'
+            mockModuleSyncStartedOnCmHandles.containsKey('cmhandle-2') >> true
         when: 'registration is processed'
             objectUnderTest.updateDmiRegistration(dmiRegistration)
         then: 'cm-handles are removed first'
             1 * objectUnderTest.processRemovedCmHandles(*_)
         and: 'de-registered cm handle entry is removed from in progress map'
-            1 * mockModuleSyncStartedOnCmHandles.remove('cmhandle-2')
-        then: 'cm-handles are created'
-            1 * objectUnderTest.processCreatedCmHandles(*_)
+            1 * mockModuleSyncStartedOnCmHandles.removeAsync('cmhandle-2')
         then: 'cm-handles are updated'
             1 * objectUnderTest.processUpdatedCmHandles(*_)
             1 * mockNetworkCmProxyDataServicePropertyHandler.updateCmHandleProperties(*_) >> []
@@ -310,6 +310,9 @@ class CmHandleRegistrationServiceSpec extends Specification {
         given: 'a registration with three cm-handles to be deleted'
             def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server',
                 removedCmHandles: ['cmhandle1', 'cmhandle2', 'cmhandle3'])
+        and: 'cm handles to be deleted in the progress map'
+            mockModuleSyncStartedOnCmHandles.containsKey("cmhandle1") >> true
+            mockModuleSyncStartedOnCmHandles.containsKey("cmhandle3") >> true
         and: 'cm-handle deletion fails on batch'
             mockInventoryPersistence.deleteDataNodes(_) >> { throw new RuntimeException("Failed") }
         and: 'cm-handle deletion is successful for 1st and 3rd; failed for 2nd'
@@ -321,11 +324,12 @@ class CmHandleRegistrationServiceSpec extends Specification {
         and: 'a response is received for all cm-handles'
             response.removedCmHandles.size() == 3
         and: 'successfully de-registered cm handle 1 is removed from in progress map'
-            1 * mockModuleSyncStartedOnCmHandles.remove('cmhandle1')
+            1 * mockModuleSyncStartedOnCmHandles.removeAsync('cmhandle1')
         and: 'successfully de-registered cm handle 3 is removed from in progress map even though it was already being removed'
-            1 * mockModuleSyncStartedOnCmHandles.remove('cmhandle3') >> 'already in progress'
+            1 * mockModuleSyncStartedOnCmHandles.removeAsync('cmhandle3')
         and: 'failed de-registered cm handle entries should NOT be removed from in progress map'
-            0 * mockModuleSyncStartedOnCmHandles.remove('cmhandle2')
+            0 * mockModuleSyncStartedOnCmHandles.containsKey('cmhandle2')
+            0 * mockModuleSyncStartedOnCmHandles.removeAsync('cmhandle2')
         and: '1st and 3rd cm-handle deletes successfully'
             with(response.removedCmHandles[0]) {
                 assert it.status == Status.SUCCESS
@@ -349,7 +353,6 @@ class CmHandleRegistrationServiceSpec extends Specification {
             })
         and: 'No cm handles state updates for "upgraded cm handles"'
             1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch([:])
-
     }
 
     def 'Remove CmHandle Error Handling: Schema Set Deletion failed'() {
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/trustlevel/DmiPluginTrustLevelWatchDogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/trustlevel/DmiPluginTrustLevelWatchDogSpec.groovy
index 32f4503005..097387c038 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/trustlevel/DmiPluginTrustLevelWatchDogSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/trustlevel/DmiPluginTrustLevelWatchDogSpec.groovy
@@ -20,6 +20,9 @@
 
 package org.onap.cps.ncmp.impl.inventory.trustlevel
 
+import com.hazelcast.config.Config
+import com.hazelcast.core.Hazelcast
+import com.hazelcast.instance.impl.HazelcastInstanceFactory
 import org.onap.cps.ncmp.api.inventory.models.TrustLevel
 import org.onap.cps.ncmp.impl.dmi.DmiRestClient
 import org.onap.cps.ncmp.impl.inventory.CmHandleQueryService
@@ -32,11 +35,16 @@ class DmiPluginTrustLevelWatchDogSpec extends Specification {
     def mockDmiRestClient = Mock(DmiRestClient)
     def mockCmHandleQueryService = Mock(CmHandleQueryService)
     def mockTrustLevelManager = Mock(TrustLevelManager)
-    def trustLevelPerDmiPlugin = [:]
-
+    def trustLevelPerDmiPlugin = HazelcastInstanceFactory
+        .getOrCreateHazelcastInstance(new Config('hazelcastInstanceName'))
+        .getMap('trustLevelPerDmiPlugin')
 
     def objectUnderTest = new DmiPluginTrustLevelWatchDog(mockDmiRestClient, mockCmHandleQueryService, mockTrustLevelManager, trustLevelPerDmiPlugin)
 
+    def cleanupSpec() {
+        Hazelcast.getHazelcastInstanceByName('hazelcastInstanceName').shutdown()
+    }
+
     def 'watch dmi plugin health status for #dmiHealhStatus'() {
         given: 'the cache has been initialised and "knows" about dmi-1'
             trustLevelPerDmiPlugin.put('dmi-1', dmiOldTrustLevel)
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManagerSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManagerSpec.groovy
index 1088ca8e06..1ab517cdcf 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManagerSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManagerSpec.groovy
@@ -77,19 +77,6 @@ class TrustLevelManagerSpec extends Specification {
             assert trustLevelPerCmHandleId.get('ch-2') == TrustLevel.COMPLETE
     }
 
-    def 'Initial cm handle registration where a cm handle is already in the cache'() {
-        given: 'a trusted cm handle'
-            def cmHandleModelsToBeCreated = ['ch-1': TrustLevel.NONE]
-        and: 'the cm handle id already in the cache'
-            trustLevelPerCmHandleId.put('ch-1', TrustLevel.COMPLETE)
-        when: 'method to register to the cache is called'
-            objectUnderTest.registerCmHandles(cmHandleModelsToBeCreated)
-        then: 'no notification sent'
-            0 * mockAttributeValueChangeEventPublisher.publishAvcEvent(*_)
-        and: 'cm handle cache is not updated'
-            assert trustLevelPerCmHandleId.get('ch-1') == TrustLevel.COMPLETE
-    }
-
     def 'Initial cm handle registration with a cm handle that is not trusted'() {
         given: 'a not trusted cm handle'
             def cmHandleModelsToBeCreated = ['ch-2': TrustLevel.NONE]