Module Sync Lock State implementation

Implementation of Lock state for module sync watchdog
Cm Handle state is locked if any exception is found
during sync process
Make changes around READY state method in line with the
new schema set
Add last updated time to composite state
Remove running datastore references as this is being done
at a later time

Issue-ID: CPS-875
Change-Id: I6bd159faefef2fa84dbf536c292ff0a132793381
Signed-off-by: DylanB95EST <dylan.byrne@est.tech>

10 files changed, 158 insertions, 133 deletions

diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy
index 5683d57e54..f56aea7f33 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy
@@ -28,8 +28,7 @@ import org.onap.cps.api.CpsDataService
 import org.onap.cps.api.CpsModuleService
 import org.onap.cps.ncmp.api.impl.exception.DmiRequestException
 import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations
-import org.onap.cps.ncmp.api.impl.operations.DmiModelOperations
-import org.onap.cps.ncmp.api.impl.operations.YangModelCmHandleRetriever
+import org.onap.cps.ncmp.api.inventory.InventoryPersistence
 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse
 import org.onap.cps.ncmp.api.models.DmiPluginRegistration
 import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle
@@ -61,10 +60,9 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification {
     def mockCpsModuleService = Mock(CpsModuleService)
     def spiedJsonObjectMapper = Spy(new JsonObjectMapper(new ObjectMapper()))
     def mockCpsAdminService = Mock(CpsAdminService)
-    def mockDmiModelOperations = Mock(DmiModelOperations)
     def mockDmiDataOperations = Mock(DmiDataOperations)
     def mockNetworkCmProxyDataServicePropertyHandler = Mock(NetworkCmProxyDataServicePropertyHandler)
-    def mockYangModelCmHandleRetriever = Mock(YangModelCmHandleRetriever)
+    def mockInventoryPersistence = Mock(InventoryPersistence)
     def mockModuleSyncService = Mock(ModuleSyncService)
 
     def noTimestamp = null
@@ -389,6 +387,6 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification {
 
     def getObjectUnderTest() {
         return Spy(new NetworkCmProxyDataServiceImpl(mockCpsDataService, spiedJsonObjectMapper, mockDmiDataOperations,
-            mockCpsModuleService, mockCpsAdminService, mockNetworkCmProxyDataServicePropertyHandler, mockYangModelCmHandleRetriever, mockModuleSyncService))
+            mockCpsModuleService, mockCpsAdminService, mockNetworkCmProxyDataServicePropertyHandler, mockInventoryPersistence, mockModuleSyncService))
     }
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplSpec.groovy
index be13344cf2..55a1a8d1fa 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplSpec.groovy
@@ -22,10 +22,11 @@
 
 package org.onap.cps.ncmp.api.impl
 
-import org.onap.cps.ncmp.api.impl.operations.YangModelCmHandleRetriever
+import org.onap.cps.ncmp.api.impl.exception.HttpClientRequestException
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
 import org.onap.cps.ncmp.api.inventory.CmHandleState
 import org.onap.cps.ncmp.api.inventory.CompositeState
+import org.onap.cps.ncmp.api.inventory.InventoryPersistence
 import org.onap.cps.ncmp.api.models.DmiPluginRegistration
 import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle
 import org.onap.cps.spi.exceptions.DataValidationException
@@ -57,7 +58,7 @@ class NetworkCmProxyDataServiceImplSpec extends Specification {
     def spiedJsonObjectMapper = Spy(new JsonObjectMapper(new ObjectMapper()))
     def mockDmiDataOperations = Mock(DmiDataOperations)
     def nullNetworkCmProxyDataServicePropertyHandler = null
-    def mockYangModelCmHandleRetriever = Mock(YangModelCmHandleRetriever)
+    def mockInventoryPersistence = Mock(InventoryPersistence)
     def mockModuleSyncService = Mock(ModuleSyncService)
     def mockDmiPluginRegistration = Mock(DmiPluginRegistration)
 
@@ -69,7 +70,7 @@ class NetworkCmProxyDataServiceImplSpec extends Specification {
     def ncmpServiceCmHandle = new NcmpServiceCmHandle(cmHandleId: 'some-cm-handle-id')
 
     def objectUnderTest = new NetworkCmProxyDataServiceImpl(mockCpsDataService, spiedJsonObjectMapper, mockDmiDataOperations,
-        mockCpsModuleService, mockCpsAdminService, nullNetworkCmProxyDataServicePropertyHandler, mockYangModelCmHandleRetriever, mockModuleSyncService)
+        mockCpsModuleService, mockCpsAdminService, nullNetworkCmProxyDataServicePropertyHandler, mockInventoryPersistence, mockModuleSyncService)
 
     def cmHandleXPath = "/dmi-registry/cm-handles[@id='testCmHandle']"
 
@@ -171,17 +172,17 @@ class NetworkCmProxyDataServiceImplSpec extends Specification {
             def dmiServiceName = 'some service name'
             def dmiProperties = [new YangModelCmHandle.Property('Book', 'Romance Novel')]
             def publicProperties = [new YangModelCmHandle.Property('Public Book', 'Public Romance Novel')]
-            def compositeState = new CompositeState(cmhandleState: 'ADVISED')
+            def compositeState = new CompositeState(cmHandleState: 'ADVISED')
             def yangModelCmHandle = new YangModelCmHandle(id: 'some-cm-handle', dmiServiceName: dmiServiceName,
                 dmiProperties: dmiProperties, publicProperties: publicProperties, compositeState: compositeState)
-            1 * mockYangModelCmHandleRetriever.getYangModelCmHandle('some-cm-handle') >> yangModelCmHandle
+            1 * mockInventoryPersistence.getYangModelCmHandle('some-cm-handle') >> yangModelCmHandle
         when: 'getting cm handle details for a given cm handle id from ncmp service'
             def result = objectUnderTest.getNcmpServiceCmHandle('some-cm-handle')
         then: 'the result returns the correct data'
             result.cmHandleId == 'some-cm-handle'
             result.dmiProperties ==[ Book:'Romance Novel' ]
             result.publicProperties == [ "Public Book":'Public Romance Novel' ]
-            result.compositeState.cmhandleState == CmHandleState.ADVISED
+            result.compositeState.cmHandleState == CmHandleState.ADVISED
     }
 
     def 'Get a cm handle with an invalid id.'() {
@@ -190,7 +191,7 @@ class NetworkCmProxyDataServiceImplSpec extends Specification {
         then: 'an exception is thrown'
             thrown(DataValidationException)
         and: 'the yang model cm handle retriever is not invoked'
-            0 * mockYangModelCmHandleRetriever.getYangModelCmHandle(*_)
+            0 * mockInventoryPersistence.getYangModelCmHandle(*_)
     }
 
     def 'Get cm handle public properties'() {
@@ -199,7 +200,7 @@ class NetworkCmProxyDataServiceImplSpec extends Specification {
             def publicProperties = [new YangModelCmHandle.Property('public prop', 'some public prop')]
             def yangModelCmHandle = new YangModelCmHandle(id:'some-cm-handle', dmiServiceName: 'some service name', dmiProperties: dmiProperties, publicProperties: publicProperties)
         and: 'the system returns this yang modelled cm handle'
-            1 * mockYangModelCmHandleRetriever.getYangModelCmHandle('some-cm-handle') >> yangModelCmHandle
+            1 * mockInventoryPersistence.getYangModelCmHandle('some-cm-handle') >> yangModelCmHandle
         when: 'getting cm handle public properties for a given cm handle id from ncmp service'
             def result = objectUnderTest.getCmHandlePublicProperties('some-cm-handle')
         then: 'the result returns the correct data'
@@ -212,7 +213,7 @@ class NetworkCmProxyDataServiceImplSpec extends Specification {
         then: 'an exception is thrown'
             thrown(DataValidationException)
         and: 'the yang model cm handle retriever is not invoked'
-            0 * mockYangModelCmHandleRetriever.getYangModelCmHandle(*_)
+            0 * mockInventoryPersistence.getYangModelCmHandle(*_)
     }
 
     def 'Update resource data for pass-through running from dmi using POST #scenario DMI properties.'() {
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiOperationsBaseSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiOperationsBaseSpec.groovy
index dae2bcc0ae..193b94d264 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiOperationsBaseSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiOperationsBaseSpec.groovy
@@ -25,6 +25,7 @@ import org.onap.cps.ncmp.api.impl.client.DmiRestClient
 import org.onap.cps.ncmp.api.impl.config.NcmpConfiguration
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
 import org.onap.cps.ncmp.api.impl.utils.DmiServiceUrlBuilder
+import org.onap.cps.ncmp.api.inventory.InventoryPersistence
 import org.spockframework.spring.SpringBean
 import spock.lang.Shared
 import spock.lang.Specification
@@ -38,7 +39,7 @@ abstract class DmiOperationsBaseSpec extends Specification {
     DmiRestClient mockDmiRestClient = Mock()
 
     @SpringBean
-    YangModelCmHandleRetriever mockCmHandlePropertiesRetriever = Mock()
+    InventoryPersistence mockInventoryPersistence = Mock()
 
     @SpringBean
     ObjectMapper spyObjectMapper = Spy()
@@ -56,6 +57,6 @@ abstract class DmiOperationsBaseSpec extends Specification {
         yangModelCmHandle.dmiServiceName = dmiServiceName
         yangModelCmHandle.dmiProperties = dmiProperties
         yangModelCmHandle.id = cmHandleId
-        mockCmHandlePropertiesRetriever.getYangModelCmHandle(cmHandleId) >> yangModelCmHandle
+        mockInventoryPersistence.getYangModelCmHandle(cmHandleId) >> yangModelCmHandle
     }
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateBuilderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateBuilderSpec.groovy
index 2be523915e..d6f4ba6ede 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateBuilderSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateBuilderSpec.groovy
@@ -1,6 +1,7 @@
 /*
  * ============LICENSE_START=======================================================
  * Copyright (C) 2022 Bell Canada
+ * Copyright (C) 2022 Nordix Foundation.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -36,7 +37,7 @@ class CompositeStateBuilderSpec extends Specification {
     def static cmHandleId = 'myHandle1'
     def static cmHandleXpath = "/dmi-registry/cm-handles[@id='${cmHandleId}/state']"
     def static stateDataNodes = [new DataNodeBuilder().withXpath("/dmi-registry/cm-handles[@id='${cmHandleId}']/state/lock-reason")
-                                         .withLeaves(['reason': 'lock reason', 'details': 'lock details']).build(),
+                                         .withLeaves(['reason': 'LOCKED_MISBEHAVING', 'details': 'lock details']).build(),
                                  new DataNodeBuilder().withXpath("/dmi-registry/cm-handles[@id='${cmHandleId}']/state/datastores")
                                             .withChildDataNodes(Arrays.asList(new DataNodeBuilder()
                                                     .withXpath("/dmi-registry/cm-handles[@id='${cmHandleId}']/state/datastores/operational")
@@ -46,20 +47,20 @@ class CompositeStateBuilderSpec extends Specification {
     def "Composite State Specification"() {
         when: 'using composite state builder '
             def compositeState = new CompositeStateBuilder().withCmHandleState(CmHandleState.ADVISED)
-                    .withLockReason("lock-reason","").withOperationalDataStores("UNSYNCHRONIZED",
-                    formattedDateAndTime.toString()).withLastUpdatedTime(formattedDateAndTime).build();
+                    .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING,"").withOperationalDataStores("UNSYNCHRONIZED",
+                    formattedDateAndTime.toString()).withLastUpdatedTime(formattedDateAndTime).build()
         then: 'it matches expected cm handle state and data store sync state'
-            assert compositeState.getCmhandleState() == CmHandleState.ADVISED
+            assert compositeState.cmHandleState == CmHandleState.ADVISED
             assert compositeState.dataStores.operationalDataStore.syncState == 'UNSYNCHRONIZED'
     }
 
     def "Build composite state from DataNode "() {
         given: "a Data Node "
-            def dataNode = new DataNode(leaves: ['cm-handle-state': 'ADVISED'])
+            new DataNode(leaves: ['cm-handle-state': 'ADVISED'])
         when: 'build from data node function is invoked'
             def compositeState = new CompositeStateBuilder().fromDataNode(cmHandleDataNode).build()
         then: 'it matches expected state model as JSON'
-            assert compositeState.cmhandleState == CmHandleState.ADVISED
+            assert compositeState.cmHandleState == CmHandleState.ADVISED
     }
 
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateSpec.groovy
index 59c9951d47..5387fc675f 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateSpec.groovy
@@ -28,9 +28,7 @@ import java.time.ZoneOffset
 import java.time.format.DateTimeFormatter
 
 import static CompositeState.DataStores
-import static CompositeState.LockReason
 import static CompositeState.Operational
-import static CompositeState.Running
 import static org.onap.cps.ncmp.utils.TestUtils.getResourceFileContent
 import static org.springframework.util.StringUtils.trimAllWhitespace
 
@@ -42,8 +40,8 @@ class CompositeStateSpec extends Specification {
 
     def "Composite State Specification"() {
         given: "a Composite State"
-            def compositeState = new CompositeState(cmhandleState: CmHandleState.ADVISED,
-                lockReason: LockReason.builder().reason('lock-reason').details("lock-misbehaving-details").build(),
+            def compositeState = new CompositeState(cmHandleState: CmHandleState.ADVISED,
+                lockReason: CompositeState.LockReason.builder().lockReasonCategory(LockReasonCategory.LOCKED_MISBEHAVING).details("lock misbehaving details").build(),
                 lastUpdateTime: formattedDateAndTime.toString(),
                 dataSyncEnabled: false,
                 dataStores: dataStores())
@@ -57,8 +55,6 @@ class CompositeStateSpec extends Specification {
     def dataStores() {
         DataStores.builder().operationalDataStore(Operational.builder()
             .syncState('NONE_REQUESTED')
-            .lastSyncTime(formattedDateAndTime.toString()).build()).runningDataStore(Running.builder()
-            .syncState('NONE_REQUESTED')
             .lastSyncTime(formattedDateAndTime.toString()).build())
             .build()
     }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/YangModelCmHandleRetrieverSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy
index 7fbfa779dc..b638eecd41 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/YangModelCmHandleRetrieverSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy
@@ -1,6 +1,6 @@
 /*
- * ============LICENSE_START=======================================================
- *  Copyright (C) 2021-2022 Nordix Foundation
+ *  ============LICENSE_START=======================================================
+ *  Copyright (C) 2022 Nordix Foundation
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -18,33 +18,45 @@
  *  ============LICENSE_END=========================================================
  */
 
-package org.onap.cps.ncmp.api.impl.operations
+package org.onap.cps.ncmp.api.inventory
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import org.onap.cps.api.CpsDataService
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
-import org.onap.cps.ncmp.api.inventory.CmHandleState
-import org.onap.cps.ncmp.api.inventory.CompositeStateBuilder
+import org.onap.cps.spi.CpsDataPersistenceService
+import org.onap.cps.spi.FetchDescendantsOption
 import org.onap.cps.spi.exceptions.DataValidationException
+import org.onap.cps.spi.model.DataNode
+import org.onap.cps.utils.JsonObjectMapper
 import spock.lang.Shared
+import spock.lang.Specification
+
+import java.time.OffsetDateTime
+import java.time.ZoneOffset
+import java.time.format.DateTimeFormatter
 
 import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS
-import org.onap.cps.spi.model.DataNode
-import spock.lang.Specification
+import static org.onap.cps.spi.FetchDescendantsOption.OMIT_DESCENDANTS
 
-class YangModelCmHandleRetrieverSpec extends Specification {
+class InventoryPersistenceSpec extends Specification {
+
+    def spiedJsonObjectMapper = Spy(new JsonObjectMapper(new ObjectMapper()))
 
     def mockCpsDataService = Mock(CpsDataService)
 
-    def objectUnderTest = new YangModelCmHandleRetriever(mockCpsDataService)
+    def mockCpsDataPersistenceService = Mock(CpsDataPersistenceService)
+
+
+    def objectUnderTest = new InventoryPersistence(spiedJsonObjectMapper, mockCpsDataService, mockCpsDataPersistenceService)
+
+    def formattedDateAndTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
+        .format(OffsetDateTime.of(2022, 12, 31, 20, 30, 40, 1, ZoneOffset.UTC))
 
     def cmHandleId = 'some-cm-handle'
     def leaves = ["dmi-service-name":"common service name","dmi-data-service-name":"data service name","dmi-model-service-name":"model service name"]
     def xpath = "/dmi-registry/cm-handles[@id='some-cm-handle']"
 
     @Shared
-    def compositeState = new CompositeStateBuilder().withCmHandleState(CmHandleState.ADVISED).build()
-
-    @Shared
     def childDataNodesForCmHandleWithAllProperties = [new DataNode(xpath: "/dmi-registry/cm-handles[@id='some cm handle']/additional-properties[@name='name1']", leaves: ["name":"name1", "value":"value1"]),
                                                       new DataNode(xpath: "/dmi-registry/cm-handles[@id='some cm handle']/public-properties[@name='name2']", leaves: ["name":"name2","value":"value2"])]
 
@@ -72,11 +84,11 @@ class YangModelCmHandleRetrieverSpec extends Specification {
             result.dmiProperties == expectedDmiProperties
             result.publicProperties == expectedPublicProperties
         and: 'the state details are returned'
-            result.compositeState.cmhandleState == expectedCompositeState
+            result.compositeState.cmHandleState == expectedCompositeState
         where: 'the following parameters are used'
             scenario                    | childDataNodes                                || expectedDmiProperties                               || expectedPublicProperties                              || expectedCompositeState
             'no properties'             | []                                            || []                                                  || []                                                    || null
-            'DMI and public properties' | childDataNodesForCmHandleWithAllProperties    || [new YangModelCmHandle.Property("name1", "value1")] || [new YangModelCmHandle.Property("name2", "value2")]   || null
+            'DMI and public properties' | childDataNodesForCmHandleWithAllProperties    || [new YangModelCmHandle.Property("name1", "value1")] || [new YangModelCmHandle.Property("name2", "value2")] || null
             'just DMI properties'       | childDataNodesForCmHandleWithDMIProperties    || [new YangModelCmHandle.Property("name1", "value1")] || []                                                    || null
             'just public properties'    | childDataNodesForCmHandleWithPublicProperties || []                                                  || [new YangModelCmHandle.Property("name2", "value2")]   || null
             'with state details'        | childDataNodesForCmHandleWithState            || []                                                  || []                                                    || CmHandleState.ADVISED
@@ -102,4 +114,45 @@ class YangModelCmHandleRetrieverSpec extends Specification {
             result.dmiDataServiceName == null
             result.dmiModelServiceName == null
     }
+
+    def 'Get a Cm Handle Composite State'() {
+        given: 'a valid cm handle id'
+            def cmHandleId = 'Some-Cm-Handle'
+            def dataNode = new DataNode(leaves: ['cm-handle-state': 'ADVISED'])
+        and: 'cps data service returns a valid data node'
+            mockCpsDataService.getDataNode('NCMP-Admin', 'ncmp-dmi-registry',
+                '/dmi-registry/cm-handles[@id=\'Some-Cm-Handle\']/state', FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) >> dataNode
+        when: 'get cm handle state is invoked'
+            def result = objectUnderTest.getCmHandleState(cmHandleId)
+        then: 'result has returned the correct cm handle state'
+            result.cmHandleState == CmHandleState.ADVISED
+    }
+
+    def 'Update Cm Handle with #scenario State'() {
+        given: 'a cm handle and a composite state'
+            def cmHandleId = 'Some-Cm-Handle'
+            def compositeState = new CompositeState(cmHandleState: cmHandleState, lastUpdateTime: formattedDateAndTime)
+        when: 'update cm handle state is invoked with the #scenario state'
+            objectUnderTest.saveCmHandleState(cmHandleId, compositeState)
+        then: 'update node leaves is invoked with the correct params'
+            1 * mockCpsDataService.replaceNodeTree('NCMP-Admin', 'ncmp-dmi-registry', '/dmi-registry/cm-handles[@id=\'Some-Cm-Handle\']', expectedJsonData, _ as OffsetDateTime)
+        where: 'the following states are used'
+             scenario | cmHandleState        || expectedJsonData
+            'READY'   | CmHandleState.READY  || '{"state":{"cm-handle-state":"READY","last-update-time":"2022-12-31T20:30:40.000+0000"}}'
+            'LOCKED'  | CmHandleState.LOCKED || '{"state":{"cm-handle-state":"LOCKED","last-update-time":"2022-12-31T20:30:40.000+0000"}}'
+    }
+
+    def 'Get Cm Handles By State'() {
+        given: 'a cm handle state to query'
+            def cmHandleState = CmHandleState.ADVISED
+        and: 'cps data service returns a list of data nodes'
+            def dataNodes = [new DataNode()]
+            mockCpsDataPersistenceService.queryDataNodes('NCMP-Admin', 'ncmp-dmi-registry',
+                '//state[@cm-handle-state="ADVISED"]/ancestor::cm-handles', OMIT_DESCENDANTS) >> dataNodes
+        when: 'get cm handles by state is invoked'
+            def result = objectUnderTest.getCmHandlesByState(cmHandleState)
+        then: 'the returned result is a list of data nodes returned by cps data service'
+            assert result == dataNodes
+    }
+
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/CmHandleStateSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/CmHandleStateSpec.groovy
deleted file mode 100644
index bfc5c6fa39..0000000000
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/CmHandleStateSpec.groovy
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * ============LICENSE_START=======================================================
- *  Copyright (C) 2022 Nordix Foundation
- *  ================================================================================
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- *  SPDX-License-Identifier: Apache-2.0
- *  ============LICENSE_END=========================================================
- */
-
-package org.onap.cps.ncmp.api.inventory.sync
-
-import org.onap.cps.ncmp.api.inventory.CmHandleState
-import spock.lang.Specification
-
-class CmHandleStateSpec extends Specification{
-
-    def 'Transition to READY state from ADVISED state'() {
-        given: 'a cm handle with an ADVISED state'
-            def cmHandleState = CmHandleState.ADVISED
-        when: 'the state transitions to the READY state'
-            cmHandleState = CmHandleState.READY
-        then: 'the cm handle state changes to READY'
-            assert CmHandleState.READY == cmHandleState
-    }
-
-    def 'Transition to READY state from READY state'() {
-        given: 'a cm handle with a READY state'
-            def cmHandleState = CmHandleState.READY
-        when: 'the state transitions to READY state'
-            cmHandleState = CmHandleState.READY
-        then: 'the cm handle state remains as READY'
-            assert CmHandleState.READY == cmHandleState
-    }
-
-}
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
index 35de99fefb..bcfe47fd5d 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
@@ -1,5 +1,5 @@
 /*
- * ============LICENSE_START=======================================================
+ *  ============LICENSE_START=======================================================
  *  Copyright (C) 2022 Nordix Foundation
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,40 +20,72 @@
 
 package org.onap.cps.ncmp.api.inventory.sync
 
-
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
 import org.onap.cps.ncmp.api.inventory.CmHandleState
 import org.onap.cps.ncmp.api.inventory.CompositeState
+import org.onap.cps.ncmp.api.inventory.InventoryPersistence
+import org.onap.cps.ncmp.api.inventory.LockReasonCategory
 import spock.lang.Specification
 
 class ModuleSyncSpec extends Specification {
 
+    def mockInventoryPersistence = Mock(InventoryPersistence)
+
     def mockSyncUtils = Mock(SyncUtils)
 
     def mockModuleSyncService = Mock(ModuleSyncService)
 
     def cmHandleState = CmHandleState.ADVISED
 
-    def objectUnderTest = new ModuleSyncWatchdog(mockSyncUtils, mockModuleSyncService)
+    def objectUnderTest = new ModuleSyncWatchdog(mockInventoryPersistence, mockSyncUtils, mockModuleSyncService)
 
     def 'Schedule a Cm-Handle Sync for ADVISED Cm-Handles'() {
         given: 'cm handles in an advised state'
-            def compositeState = new CompositeState()
-            compositeState.cmhandleState = cmHandleState
-            def yangModelCmHandle1 = new YangModelCmHandle(compositeState: compositeState)
-            def yangModelCmHandle2 = new YangModelCmHandle(compositeState: compositeState)
+            def compositeState1 = new CompositeState(cmHandleState: cmHandleState)
+            def compositeState2 = new CompositeState(cmHandleState: cmHandleState)
+            def yangModelCmHandle1 = new YangModelCmHandle(id: 'some-cm-handle', compositeState: compositeState1)
+            def yangModelCmHandle2 = new YangModelCmHandle(id: 'some-cm-handle-2', compositeState: compositeState2)
         and: 'sync utilities return a cm handle twice'
             mockSyncUtils.getAnAdvisedCmHandle() >>> [yangModelCmHandle1, yangModelCmHandle2, null]
         when: 'module sync poll is executed'
             objectUnderTest.executeAdvisedCmHandlePoll()
-        then: 'module sync service syncs the first cm handle and creates a schema set'
+        then: 'the inventory persistence cm handle returns a composite state for the first cm handle'
+            1 * mockInventoryPersistence.getCmHandleState('some-cm-handle') >> compositeState1
+        and: 'module sync service syncs the first cm handle and creates a schema set'
             1 * mockModuleSyncService.syncAndCreateSchemaSet(yangModelCmHandle1)
-        and: 'the first cm handle is updated to state "READY" from "ADVISED"'
-            1 * mockSyncUtils.updateCmHandleState(yangModelCmHandle1, CmHandleState.READY)
-        then: 'module sync service syncs the second cm handle and creates a schema set'
+        and: 'the composite state cm handle state is now READY'
+            assert compositeState1.getCmHandleState() == CmHandleState.READY
+        and: 'the first cm handle state is updated'
+            1 * mockInventoryPersistence.saveCmHandleState('some-cm-handle', compositeState1)
+        then: 'the inventory persistence cm handle returns a composite state for the second cm handle'
+            mockInventoryPersistence.getCmHandleState('some-cm-handle-2') >> compositeState2
+        and: 'module sync service syncs the second cm handle and creates a schema set'
             1 * mockModuleSyncService.syncAndCreateSchemaSet(yangModelCmHandle2)
-        then: 'the second cm handle is updated to state "READY" from "ADVISED"'
-            1 * mockSyncUtils.updateCmHandleState(yangModelCmHandle2, CmHandleState.READY)
+        and: 'the composite state cm handle state is now READY'
+            assert compositeState2.getCmHandleState() == CmHandleState.READY
+        and: 'the second cm handle state is updated'
+            1 * mockInventoryPersistence.saveCmHandleState('some-cm-handle-2', compositeState2)
+    }
+
+    def 'Schedule a Cm-Handle Sync for ADVISED Cm-Handle with failure'() {
+        given: 'cm handles in an advised state'
+            def compositeState = new CompositeState(cmHandleState: cmHandleState)
+            def yangModelCmHandle = new YangModelCmHandle(id: 'some-cm-handle', compositeState: compositeState)
+        and: 'sync utilities return a cm handle'
+            mockSyncUtils.getAnAdvisedCmHandle() >>> [yangModelCmHandle, null]
+        when: 'module sync poll is executed'
+            objectUnderTest.executeAdvisedCmHandlePoll()
+        then: 'the inventory persistence cm handle returns a composite state for the cm handle'
+            1 * mockInventoryPersistence.getCmHandleState('some-cm-handle') >> compositeState
+        and: 'module sync service attempts to sync the cm handle and throws an exception'
+            1 * mockModuleSyncService.syncAndCreateSchemaSet(*_) >> { throw new Exception('some exception') }
+        and: 'the composite state cm handle state is now LOCKED'
+            assert compositeState.getCmHandleState() == CmHandleState.LOCKED
+        and: 'update lock reason, details and attempts is invoked'
+            1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(compositeState, LockReasonCategory.LOCKED_MISBEHAVING ,'some exception')
+        and: 'the cm handle state is updated'
+            1 * mockInventoryPersistence.saveCmHandleState('some-cm-handle', compositeState)
+
     }
 
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
index c80263ef05..7d67acccc3 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
@@ -20,29 +20,22 @@
 
 package org.onap.cps.ncmp.api.inventory.sync
 
-import com.fasterxml.jackson.databind.ObjectMapper
-import org.onap.cps.api.CpsDataService
-import org.onap.cps.ncmp.api.impl.operations.YangModelCmHandleRetriever
-import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
 import org.onap.cps.ncmp.api.inventory.CmHandleState
 import org.onap.cps.ncmp.api.inventory.CompositeState
+import org.onap.cps.ncmp.api.inventory.InventoryPersistence
+import org.onap.cps.ncmp.api.inventory.LockReasonCategory
 import org.onap.cps.spi.CpsDataPersistenceService
 import org.onap.cps.spi.FetchDescendantsOption
 import org.onap.cps.spi.model.DataNode
-import org.onap.cps.utils.JsonObjectMapper
 import spock.lang.Shared
 import spock.lang.Specification
 
-import java.time.OffsetDateTime
-
 class SyncUtilsSpec extends Specification{
 
-    def mockCpsDataService = Mock(CpsDataService)
     def mockCpsDataPersistenceService = Mock(CpsDataPersistenceService)
-    def spiedJsonObjectMapper = Spy(new JsonObjectMapper(new ObjectMapper()))
-    def mockYangModelCmHandleRetriever = Mock(YangModelCmHandleRetriever)
+    def mockInventoryPersistence = Mock(InventoryPersistence)
 
-    def objectUnderTest = new SyncUtils(mockCpsDataService, mockCpsDataPersistenceService, spiedJsonObjectMapper, mockYangModelCmHandleRetriever)
+    def objectUnderTest = new SyncUtils(mockInventoryPersistence)
 
     @Shared
     def dataNode = new DataNode(leaves: ['id': 'cm-handle-123'])
@@ -50,16 +43,14 @@ class SyncUtilsSpec extends Specification{
 
 
     def 'Get an advised Cm-Handle where ADVISED cm handle #scenario'() {
-        given: 'the cps (persistence service) returns a collection of data nodes'
-            mockCpsDataPersistenceService.queryDataNodes('NCMP-Admin',
-                'ncmp-dmi-registry', '//cm-handles[@state=\"ADVISED\"]',
-                FetchDescendantsOption.OMIT_DESCENDANTS) >> dataNodeCollection
+        given: 'the inventory persistence service returns a collection of data nodes'
+            mockInventoryPersistence.getCmHandlesByState(CmHandleState.ADVISED) >> dataNodeCollection
         when: 'get advised cm handle is called'
             objectUnderTest.getAnAdvisedCmHandle()
         then: 'the returned data node collection is the correct size'
             dataNodeCollection.size() == expectedDataNodeSize
         and: 'get yang model cm handles is invoked the correct number of times'
-           expectedCallsToGetYangModelCmHandle * mockYangModelCmHandleRetriever.getYangModelCmHandle('cm-handle-123')
+           expectedCallsToGetYangModelCmHandle * mockInventoryPersistence.getYangModelCmHandle('cm-handle-123')
         where: 'the following scenarios are used'
             scenario         | dataNodeCollection || expectedCallsToGetYangModelCmHandle | expectedDataNodeSize
             'exists'         | [ dataNode ]       || 1                                   | 1
@@ -67,16 +58,18 @@ class SyncUtilsSpec extends Specification{
 
     }
 
-    def 'Update cm handle state from Advised to Ready'() {
-        given: 'a yang model cm handle and the expected json data'
-            def compositeState = new CompositeState()
-            compositeState.cmhandleState = CmHandleState.ADVISED
-            def yangModelCmHandle = new YangModelCmHandle(id: 'Some-Cm-Handle', compositeState: compositeState )
-            def expectedJsonData = '{"cm-handles":[{"id":"Some-Cm-Handle","state":{"cm-handle-state":"READY"}}]}'
-        when: 'update cm handle state is called'
-            objectUnderTest.updateCmHandleState(yangModelCmHandle, CmHandleState.READY)
-        then: 'update data note leaves is invoked with the correct params'
-            1 * mockCpsDataService.updateNodeLeaves('NCMP-Admin', 'ncmp-dmi-registry', '/dmi-registry', expectedJsonData, _ as OffsetDateTime)
+    def 'Update Lock Reason, Details and Attempts where lock reason #scenario'() {
+        given: 'A locked state'
+           def compositeState = new CompositeState(lockReason: lockReason)
+        when: 'update cm handle details and attempts is called'
+            objectUnderTest.updateLockReasonDetailsAndAttempts(compositeState, LockReasonCategory.LOCKED_MISBEHAVING, 'new error message')
+        then: 'the composite state lock reason and details are updated'
+            assert compositeState.lockReason.lockReasonCategory == LockReasonCategory.LOCKED_MISBEHAVING
+            assert compositeState.lockReason.details == expectedDetails
+        where:
+            scenario         | lockReason                                                                                   || expectedDetails
+            'does not exist' | null                                                                                         || 'Attempt #1 failed: new error message'
+            'exists'         | CompositeState.LockReason.builder().details("Attempt #2 failed: some error message").build() || 'Attempt #3 failed: new error message'
     }
 
 }
diff --git a/cps-ncmp-service/src/test/resources/expectedStateModel.json b/cps-ncmp-service/src/test/resources/expectedStateModel.json
index f68d725edb..5d246d5dd1 100644
--- a/cps-ncmp-service/src/test/resources/expectedStateModel.json
+++ b/cps-ncmp-service/src/test/resources/expectedStateModel.json
@@ -1,8 +1,8 @@
 {
   "cm-handle-state" : "ADVISED",
   "lock-reason" : {
-    "reason" : "lock-reason",
-    "details" : "lock-misbehaving-details"
+    "reason" : "LOCKED_MISBEHAVING",
+    "details" : "lock misbehaving details"
   },
   "last-update-time" : "2022-12-31T20:30:40.000+0000",
   "data-sync-enabled" : false,
@@ -10,10 +10,6 @@
     "operational" : {
       "sync-state" : "NONE_REQUESTED",
       "last-sync-time" : "2022-12-31T20:30:40.000+0000"
-    },
-    "running" : {
-      "sync-state" : "NONE_REQUESTED",
-      "last-sync-time" : "2022-12-31T20:30:40.000+0000"
     }
   }
 }
\ No newline at end of file