Data Sync Watchdog Process

- Get all the Cm Handles state in READY and Operational datastores sync state in UNSYNCHRONIZED
- Get a random Cm Handle
- Get the first resource data from the node
- Save the data in Cps Db
- Update the Operational datastores sync state to SYNCHRONIZED

Issue-ID: CPS-1052
Issue-ID: CPS-1053
Issue-ID: CPS-1054
Change-Id: I9a20391ef30e6d56c4d789a92b8bf42cd3756c62
Signed-off-by: Lathish <lathishbabu.ganesan@est.tech>
Signed-off-by: sourabh_sourabh <sourabh.sourabh@est.tech>

8 files changed, 194 insertions, 13 deletions

diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiDataOperationsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiDataOperationsSpec.groovy
index b7ebf2965a..03825c2bbf 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiDataOperationsSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiDataOperationsSpec.groovy
@@ -80,6 +80,20 @@ class DmiDataOperationsSpec extends DmiOperationsBaseSpec {
             'datastore running with properties'    | [yangModelCmHandleProperty] | PASSTHROUGH_RUNNING     | OPTIONS_PARAM || '{"operation":"read","cmHandleProperties":{"prop1":"val1"}}' | 'passthrough-running'     | '&options=(a=1,b=2)'
     }
 
+    def 'call get all resource data.'() {
+        given: 'the system returns a cm handle with a sample property'
+            mockYangModelCmHandleRetrieval([yangModelCmHandleProperty])
+        and: 'a positive response from DMI service when it is called with the expected parameters'
+            def responseFromDmi = new ResponseEntity<Object>(HttpStatus.OK)
+            def expectedUrl = dmiServiceBaseUrl + "passthrough-operational?resourceIdentifier=/"
+            mockDmiRestClient.postOperationWithJsonData(expectedUrl, '{"operation":"read","cmHandleProperties":{"prop1":"val1"}}', READ) >> responseFromDmi
+            dmiServiceUrlBuilder.getDmiDatastoreUrl(_, _) >> expectedUrl
+        when: 'get resource data is invoked'
+            def result = objectUnderTest.getResourceDataFromDmi(cmHandleId, PASSTHROUGH_OPERATIONAL, NO_REQUEST_ID)
+        then: 'the result is the response from the DMI service'
+            assert result == responseFromDmi
+    }
+
     def 'Write data for pass-through:running datastore in DMI.'() {
         given: 'a cm handle for #cmHandleId'
             mockYangModelCmHandleRetrieval([yangModelCmHandleProperty])
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateBuilderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateBuilderSpec.groovy
index d6f4ba6ede..60fec6f867 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateBuilderSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateBuilderSpec.groovy
@@ -47,11 +47,11 @@ class CompositeStateBuilderSpec extends Specification {
     def "Composite State Specification"() {
         when: 'using composite state builder '
             def compositeState = new CompositeStateBuilder().withCmHandleState(CmHandleState.ADVISED)
-                    .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING,"").withOperationalDataStores("UNSYNCHRONIZED",
+                    .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING,"").withOperationalDataStores(SyncState.UNSYNCHRONIZED,
                     formattedDateAndTime.toString()).withLastUpdatedTime(formattedDateAndTime).build()
         then: 'it matches expected cm handle state and data store sync state'
             assert compositeState.cmHandleState == CmHandleState.ADVISED
-            assert compositeState.dataStores.operationalDataStore.syncState == 'UNSYNCHRONIZED'
+            assert compositeState.dataStores.operationalDataStore.syncState == SyncState.UNSYNCHRONIZED
     }
 
     def "Build composite state from DataNode "() {
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateSpec.groovy
index 5387fc675f..0a6f8c350a 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/CompositeStateSpec.groovy
@@ -54,7 +54,7 @@ class CompositeStateSpec extends Specification {
 
     def dataStores() {
         DataStores.builder().operationalDataStore(Operational.builder()
-            .syncState('NONE_REQUESTED')
+            .syncState(SyncState.NONE_REQUESTED)
             .lastSyncTime(formattedDateAndTime.toString()).build())
             .build()
     }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy
index e6346cb024..578225e87d 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceSpec.groovy
@@ -28,7 +28,6 @@ import org.onap.cps.spi.CpsDataPersistenceService
 import org.onap.cps.spi.FetchDescendantsOption
 import org.onap.cps.spi.exceptions.DataValidationException
 import org.onap.cps.spi.model.DataNode
-import org.onap.cps.spi.model.DataNodeBuilder
 import org.onap.cps.utils.JsonObjectMapper
 import spock.lang.Shared
 import spock.lang.Specification
@@ -71,6 +70,9 @@ class InventoryPersistenceSpec extends Specification {
     @Shared
     def childDataNodesForCmHandleWithState = [new DataNode(xpath: "/dmi-registry/cm-handles[@id='some-cm-handle']/state", leaves: ['cm-handle-state': 'ADVISED'])]
 
+    @Shared
+    def static sampleDataNodes = [new DataNode()]
+
     def "Retrieve CmHandle using datanode with #scenario."() {
         given: 'the cps data service returns a data node from the DMI registry'
             def dataNode = new DataNode(childDataNodes:childDataNodes, leaves: leaves)
@@ -148,13 +150,36 @@ class InventoryPersistenceSpec extends Specification {
         given: 'a cm handle state to query'
             def cmHandleState = CmHandleState.ADVISED
         and: 'cps data service returns a list of data nodes'
-            def dataNodes = [new DataNode()]
             mockCpsDataPersistenceService.queryDataNodes('NCMP-Admin', 'ncmp-dmi-registry',
-                    '//state[@cm-handle-state="ADVISED"]/ancestor::cm-handles', OMIT_DESCENDANTS) >> dataNodes
+                '//state[@cm-handle-state="ADVISED"]/ancestor::cm-handles', OMIT_DESCENDANTS) >> sampleDataNodes
         when: 'get cm handles by state is invoked'
             def result = objectUnderTest.getCmHandlesByState(cmHandleState)
         then: 'the returned result is a list of data nodes returned by cps data service'
-            assert result == dataNodes
+            assert result == sampleDataNodes
+    }
+
+    def 'Get Cm Handles By State and Cm-Handle Id'() {
+        given: 'a cm handle state to query'
+            def cmHandleState = CmHandleState.READY
+        and: 'cps data service returns a list of data nodes'
+            mockCpsDataPersistenceService.queryDataNodes('NCMP-Admin', 'ncmp-dmi-registry',
+                '//cm-handles[@id=\'some-cm-handle\']/state[@cm-handle-state="'+ 'READY'+'"]/ancestor::cm-handles', OMIT_DESCENDANTS) >> sampleDataNodes
+        when: 'get cm handles by state and id is invoked'
+            def result = objectUnderTest.getCmHandlesByIdAndState(cmHandleId, cmHandleState)
+        then: 'the returned result is a list of data nodes returned by cps data service'
+            assert result == sampleDataNodes
+    }
+
+    def 'Get Cm Handles By Operational Sync State : UNSYNCHRONIZED'() {
+        given: 'a cm handle state to query'
+            def cmHandleState = CmHandleState.READY
+        and: 'cps data service returns a list of data nodes'
+            mockCpsDataPersistenceService.queryDataNodes('NCMP-Admin', 'ncmp-dmi-registry',
+                '//state/datastores/operational[@sync-state="'+'UNSYNCHRONIZED'+'"]/ancestor::cm-handles', OMIT_DESCENDANTS) >> sampleDataNodes
+        when: 'get cm handles by operational sync state as UNSYNCHRONIZED is invoked'
+            def result = objectUnderTest.getCmHandlesByOperationalSyncState(SyncState.UNSYNCHRONIZED)
+        then: 'the returned result is a list of data nodes returned by cps data service'
+            assert result == sampleDataNodes
     }
 
     def 'Retrieve cm handle by cps path '() {
@@ -166,7 +191,7 @@ class InventoryPersistenceSpec extends Specification {
                     cpsPath, OMIT_DESCENDANTS)
                     >> Arrays.asList(cmHandleDataNode)
         when: 'get cm handles by cps path is invoked'
-            def result = objectUnderTest.getCmHandlesByCpsPath(cpsPath)
+            def result = objectUnderTest.getCmHandleDataNodesByCpsPath(cpsPath)
         then: 'the returned result is a list of data nodes returned by cps data service'
             assert result.contains(cmHandleDataNode)
     }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/DataSyncSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/DataSyncSpec.groovy
new file mode 100644
index 0000000000..b062635db0
--- /dev/null
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/DataSyncSpec.groovy
@@ -0,0 +1,101 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2022 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.ncmp.api.inventory.sync
+
+import org.onap.cps.api.CpsDataService
+import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
+import org.onap.cps.ncmp.api.inventory.CmHandleState
+import org.onap.cps.ncmp.api.inventory.CompositeState
+import org.onap.cps.ncmp.api.inventory.InventoryPersistence
+import org.onap.cps.ncmp.api.inventory.SyncState
+import spock.lang.Specification
+
+class DataSyncSpec extends Specification {
+
+    def mockInventoryPersistence = Mock(InventoryPersistence)
+
+    def mockCpsDataService = Mock(CpsDataService)
+
+    def mockSyncUtils = Mock(SyncUtils)
+
+    def jsonString = '{"stores:bookstore":{"categories":[{"code":"01"}]}}'
+
+    def objectUnderTest = new DataSyncWatchdog(mockInventoryPersistence, mockCpsDataService, mockSyncUtils)
+
+    def compositeState = getCompositeState()
+
+    def yangModelCmHandle1 = createSampleYangModelCmHandle('some-cm-handle-1')
+
+    def yangModelCmHandle2 = createSampleYangModelCmHandle('some-cm-handle-2')
+
+    def 'Schedule Data Sync for Cm Handle State in READY and Operational Sync State in UNSYNCHRONIZED'() {
+        given: 'sample resource data'
+            def resourceData = jsonString;
+        and: 'sync utilities return a cm handle twice'
+            mockSyncUtils.getAnUnSynchronizedReadyCmHandle() >>> [yangModelCmHandle1, yangModelCmHandle2, null]
+        when: 'data sync poll is executed'
+            objectUnderTest.executeUnSynchronizedReadyCmHandlePoll()
+        then: 'the inventory persistence cm handle returns a composite state for the first cm handle'
+            1 * mockInventoryPersistence.getCmHandleState('some-cm-handle-1') >> compositeState
+        and: 'the sync util returns first resource data'
+            1 * mockSyncUtils.getResourceData('some-cm-handle-1') >> resourceData
+        and: 'the cm-handle data is saved'
+            1 * mockCpsDataService.saveData('NFP-Operational', 'some-cm-handle-1', jsonString, _)
+        and: 'the first cm handle operational sync state is updated'
+            1 * mockInventoryPersistence.saveCmHandleState('some-cm-handle-1', compositeState)
+        then: 'the inventory persistence cm handle returns a composite state for the second cm handle'
+            1 * mockInventoryPersistence.getCmHandleState('some-cm-handle-2') >> compositeState
+        and: 'the sync util returns first resource data'
+            1 * mockSyncUtils.getResourceData('some-cm-handle-2') >> resourceData
+        and: 'the cm-handle data is saved'
+            1 * mockCpsDataService.saveData('NFP-Operational', 'some-cm-handle-2', jsonString, _)
+        and: 'the second cm handle operational sync state is updated from "UNSYNCHRONIZED" to "SYNCHRONIZED"'
+            1 * mockInventoryPersistence.saveCmHandleState('some-cm-handle-2', compositeState)
+    }
+
+    def 'Schedule Data Sync for Cm Handle State in READY and Operational Sync State in UNSYNCHRONIZED which return empty data from Node'() {
+        given: 'cm handles in an ready state and operational sync state in unsynchronized'
+        and: 'sync utilities return a cm handle twice'
+            mockSyncUtils.getAnUnSynchronizedReadyCmHandle() >>> [yangModelCmHandle1, null]
+        when: 'data sync poll is executed'
+            objectUnderTest.executeUnSynchronizedReadyCmHandlePoll()
+        then: 'the inventory persistence cm handle returns a composite state for the first cm handle'
+            1 * mockInventoryPersistence.getCmHandleState('some-cm-handle-1') >> compositeState
+        and: 'the sync util returns first resource data'
+            1 * mockSyncUtils.getResourceData('some-cm-handle-1') >> null
+        and: 'the cm-handle data is not saved'
+            0 * mockCpsDataService.saveData('NFP-Operational', 'some-cm-handle-1', jsonString, _)
+    }
+
+    def createSampleYangModelCmHandle(cmHandleId) {
+        def compositeState = getCompositeState()
+        return new YangModelCmHandle(id: cmHandleId, compositeState: compositeState)
+    }
+
+    def getCompositeState() {
+        def cmHandleState = CmHandleState.READY
+        def compositeState = new CompositeState(cmHandleState: cmHandleState)
+        compositeState.setDataStores(CompositeState.DataStores.builder()
+            .operationalDataStore(CompositeState.Operational.builder().syncState(SyncState.SYNCHRONIZED)
+                .build()).build())
+        return compositeState
+    }
+}
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
index 802035c726..8752bff582 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy
@@ -106,7 +106,7 @@ class ModuleSyncSpec extends Specification {
                     .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING, '').build()
             def yangModelCmHandle = new YangModelCmHandle(id: 'some-cm-handle', compositeState: compositeState)
         and: 'sync utilities return a cm handle twice'
-            mockSyncUtils.getLockedMisbehavingCmHandles() >> [yangModelCmHandle, yangModelCmHandle]
+            mockSyncUtils.getLockedMisbehavingYangModelCmHandles() >> [yangModelCmHandle, yangModelCmHandle]
         when: 'module sync poll is executed'
             objectUnderTest.executeLockedMisbehavingCmHandlePoll()
         then: 'the first cm handle is updated to state "ADVISED" from "READY"'
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
index 15d1efe068..14f2015755 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy
@@ -21,11 +21,19 @@
 
 package org.onap.cps.ncmp.api.inventory.sync
 
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.ObjectMapper
+import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations
+import org.onap.cps.ncmp.api.impl.operations.DmiOperations
 import org.onap.cps.ncmp.api.inventory.CmHandleState
 import org.onap.cps.ncmp.api.inventory.CompositeState
 import org.onap.cps.ncmp.api.inventory.InventoryPersistence
 import org.onap.cps.ncmp.api.inventory.LockReasonCategory
+import org.onap.cps.ncmp.api.inventory.SyncState
 import org.onap.cps.spi.model.DataNode
+import org.onap.cps.utils.JsonObjectMapper
+import org.springframework.http.HttpStatus
+import org.springframework.http.ResponseEntity
 import spock.lang.Shared
 import spock.lang.Specification
 
@@ -33,7 +41,11 @@ class SyncUtilsSpec extends Specification{
 
     def mockInventoryPersistence = Mock(InventoryPersistence)
 
-    def objectUnderTest = new SyncUtils(mockInventoryPersistence)
+    def mockDmiDataOperations = Mock(DmiDataOperations)
+
+    def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper())
+
+    def objectUnderTest = new SyncUtils(mockInventoryPersistence, mockDmiDataOperations, jsonObjectMapper)
 
     @Shared
     def dataNode = new DataNode(leaves: ['id': 'cm-handle-123'])
@@ -67,15 +79,45 @@ class SyncUtilsSpec extends Specification{
             'does not exist' | null                                                                                         || 'Attempt #1 failed: new error message'
             'exists'         | CompositeState.LockReason.builder().details("Attempt #2 failed: some error message").build() || 'Attempt #3 failed: new error message'
     }
+
     def 'Get all locked Cm-Handle where Lock Reason is LOCKED_MISBEHAVING cm handle #scenario'() {
         given: 'the cps (persistence service) returns a collection of data nodes'
-            mockInventoryPersistence.getCmHandlesByCpsPath(
+            mockInventoryPersistence.getCmHandleDataNodesByCpsPath(
                     '//lock-reason[@reason="LOCKED_MISBEHAVING"]/ancestor::cm-handles') >> [dataNode ]
         when: 'get locked Misbehaving cm handle is called'
-            def result = objectUnderTest.getLockedMisbehavingCmHandles()
+            def result = objectUnderTest.getLockedMisbehavingYangModelCmHandles()
         then: 'the returned cm handle collection is the correct size'
             result.size() == 1
         and: 'the correct cm handle is returned'
             result[0].id == 'cm-handle-123'
     }
+
+    def 'Get a Cm-Handle where Operational Sync state is UnSynchronized and Cm-handle state is READY and #scenario'() {
+        given: 'the inventory persistence service returns a collection of data nodes'
+            mockInventoryPersistence.getCmHandlesByOperationalSyncState(SyncState.UNSYNCHRONIZED) >> unSynchronizedDataNodes
+            mockInventoryPersistence.getCmHandlesByIdAndState("cm-handle-123", CmHandleState.READY) >> readyDataNodes
+        when: 'get advised cm handle is called'
+            objectUnderTest.getAnUnSynchronizedReadyCmHandle()
+        then: 'the returned data node collection is the correct size'
+            readyDataNodes.size() == expectedDataNodeSize
+        and: 'get yang model cm handles is invoked the correct number of times'
+            expectedCallsToGetYangModelCmHandle * mockInventoryPersistence.getYangModelCmHandle('cm-handle-123')
+        where: 'the following scenarios are used'
+            scenario                             | unSynchronizedDataNodes | readyDataNodes || expectedCallsToGetYangModelCmHandle | expectedDataNodeSize
+            'exists'                             | [dataNode]              | [dataNode]     || 1                                   | 1
+            'unsynchronized exist but not ready' | [dataNode]              | []             || 0                                   | 0
+            'does not exist'                     | []                      | []             || 0                                   | 0
+    }
+
+    def 'Get resource data through DMI Operations #scenario'() {
+        given: 'the inventory persistence service returns a collection of data nodes'
+            def jsonString = '{"stores:bookstore":{"categories":[{"code":"01"}]}}'
+            JsonNode jsonNode = jsonObjectMapper.convertToJsonNode(jsonString);
+            def responseEntity = new ResponseEntity<>(jsonNode, HttpStatus.OK)
+            mockDmiDataOperations.getResourceDataFromDmi('cm-handle-123', DmiOperations.DataStoreEnum.PASSTHROUGH_OPERATIONAL, _) >> responseEntity
+        when: 'get resource data is called'
+            def result = objectUnderTest.getResourceData('cm-handle-123')
+        then: 'the returned data is correct'
+            result == jsonString
+    }
 }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/models/YangModelCmHandleSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/models/YangModelCmHandleSpec.groovy
index cdfcf59ef4..3376691ee7 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/models/YangModelCmHandleSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/models/YangModelCmHandleSpec.groovy
@@ -22,7 +22,6 @@ package org.onap.cps.ncmp.api.models
 
 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle
 import org.onap.cps.ncmp.api.inventory.CmHandleState
-import org.onap.ncmp.cmhandle.lcm.event.Event
 import spock.lang.Specification
 
 import static org.onap.cps.ncmp.api.impl.operations.RequiredDmiService.DATA