Test of retry of failed module sync

A new integration test verifies that LOCKED handles that failed
module sync will be retried after a few minutes.

Issue-ID: CPS-2033
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: If194daeb315090f2710a017270875b6301f7140a

3 files changed, 78 insertions, 18 deletions

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy
index 23504e49cd..21e225e0b8 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy
@@ -53,8 +53,13 @@ import spock.lang.Shared
 import spock.lang.Specification
 import spock.util.concurrent.PollingConditions
 
+import java.time.format.DateTimeFormatter
+
 import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo
 import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus
+import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NCMP_DATASPACE_NAME;
+import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NCMP_DMI_REGISTRY_ANCHOR;
+import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NCMP_DMI_REGISTRY_PARENT;
 
 @SpringBootTest(classes = [CpsDataspaceService])
 @Testcontainers
@@ -116,7 +121,7 @@ abstract class CpsIntegrationSpecBase extends Specification {
             createStandardBookStoreSchemaSet(GENERAL_TEST_DATASPACE)
             initialized = true
         }
-        mockDmiServer = MockRestServiceServer.createServer(restTemplate)
+        mockDmiServer = MockRestServiceServer.bindTo(restTemplate).ignoreExpectOrder(true).build()
     }
 
     def cleanup() {
@@ -188,7 +193,7 @@ abstract class CpsIntegrationSpecBase extends Specification {
     def registerCmHandle(dmiPlugin, cmHandleId, moduleSetTag, dmiModuleReferencesResponse, dmiModuleResourcesResponse) {
         def cmHandleToCreate = new NcmpServiceCmHandle(cmHandleId: cmHandleId, moduleSetTag: moduleSetTag)
         networkCmProxyDataService.updateDmiRegistrationAndSyncModule(new DmiPluginRegistration(dmiPlugin: dmiPlugin, createdCmHandles: [cmHandleToCreate]))
-        mockDmiResponsesForRegistration(dmiPlugin, cmHandleId, dmiModuleReferencesResponse, dmiModuleResourcesResponse)
+        mockDmiResponsesForModuleSync(dmiPlugin, cmHandleId, dmiModuleReferencesResponse, dmiModuleResourcesResponse)
         moduleSyncWatchdog.moduleSyncAdvisedCmHandles()
         new PollingConditions().within(3, () -> {
             CmHandleState.READY == networkCmProxyDataService.getCmHandleCompositeState(cmHandleId).cmHandleState
@@ -204,14 +209,23 @@ abstract class CpsIntegrationSpecBase extends Specification {
         networkCmProxyDataService.updateDmiRegistrationAndSyncModule(new DmiPluginRegistration(dmiPlugin: dmiPlugin, removedCmHandles: cmHandleIds))
     }
 
-    def mockDmiResponsesForRegistration(dmiPlugin, cmHandleId, dmiModuleReferencesResponse, dmiModuleResourcesResponse) {
-        if (dmiModuleReferencesResponse != null) {
-            mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/modules"))
-                    .andRespond(withStatus(HttpStatus.OK).contentType(MediaType.APPLICATION_JSON).body(dmiModuleReferencesResponse))
-        }
-        if (dmiModuleResourcesResponse != null) {
-            mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/moduleResources"))
-                    .andRespond(withStatus(HttpStatus.OK).contentType(MediaType.APPLICATION_JSON).body(dmiModuleResourcesResponse))
-        }
+    def mockDmiResponsesForModuleSync(dmiPlugin, cmHandleId, dmiModuleReferencesResponse, dmiModuleResourcesResponse) {
+        mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/modules"))
+                .andRespond(withStatus(HttpStatus.OK).contentType(MediaType.APPLICATION_JSON).body(dmiModuleReferencesResponse))
+        mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/moduleResources"))
+                .andRespond(withStatus(HttpStatus.OK).contentType(MediaType.APPLICATION_JSON).body(dmiModuleResourcesResponse))
+    }
+
+    def mockDmiIsNotAvailableForModuleSync(dmiPlugin, cmHandleId) {
+        mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/modules"))
+                .andRespond(withStatus(HttpStatus.SERVICE_UNAVAILABLE))
+    }
+
+    def overrideCmHandleLastUpdateTime(cmHandleId, newUpdateTime) {
+        String ISO_TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
+        DateTimeFormatter ISO_TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern(ISO_TIMESTAMP_PATTERN);
+        def jsonForUpdate = '{ "state": { "last-update-time": "%s" } }'.formatted(ISO_TIMESTAMP_FORMATTER.format(newUpdateTime))
+        cpsDataService.updateNodeLeaves(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR,
+                NCMP_DMI_REGISTRY_PARENT + "/cm-handles[@id='${cmHandleId}']", jsonForUpdate, now)
     }
 }
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleCreateSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleCreateSpec.groovy
index 4369b799bd..6b6f62edf9 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleCreateSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleCreateSpec.groovy
@@ -20,6 +20,7 @@
 
 package org.onap.cps.integration.functional
 
+import java.time.OffsetDateTime
 import org.onap.cps.integration.base.CpsIntegrationSpecBase
 import org.onap.cps.ncmp.api.NetworkCmProxyDataService
 import org.onap.cps.ncmp.api.impl.inventory.CmHandleState
@@ -27,12 +28,8 @@ import org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory
 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse
 import org.onap.cps.ncmp.api.models.DmiPluginRegistration
 import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle
-import org.springframework.http.HttpStatus
 import spock.util.concurrent.PollingConditions
 
-import static org.springframework.test.web.client.match.MockRestRequestMatchers.anything
-import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus
-
 class NcmpCmHandleCreateSpec extends CpsIntegrationSpecBase {
 
     NetworkCmProxyDataService objectUnderTest
@@ -48,7 +45,7 @@ class NcmpCmHandleCreateSpec extends CpsIntegrationSpecBase {
 
     def 'CM Handle registration is successful.'() {
         given: 'DMI will return modules when requested'
-            mockDmiResponsesForRegistration(DMI_URL, 'ch-1', MODULE_REFERENCES_RESPONSE_A, MODULE_RESOURCES_RESPONSE_A)
+            mockDmiResponsesForModuleSync(DMI_URL, 'ch-1', MODULE_REFERENCES_RESPONSE_A, MODULE_RESOURCES_RESPONSE_A)
 
         when: 'a CM-handle is registered for creation'
             def cmHandleToCreate = new NcmpServiceCmHandle(cmHandleId: 'ch-1')
@@ -81,7 +78,7 @@ class NcmpCmHandleCreateSpec extends CpsIntegrationSpecBase {
 
     def 'CM Handle goes to LOCKED state when DMI gives error during module sync.'() {
         given: 'DMI is not available to handle requests'
-            mockDmiServer.expect(anything()).andRespond(withStatus(HttpStatus.SERVICE_UNAVAILABLE))
+            mockDmiIsNotAvailableForModuleSync(DMI_URL, 'ch-1')
 
         when: 'a CM-handle is registered for creation'
             def cmHandleToCreate = new NcmpServiceCmHandle(cmHandleId: 'ch-1')
@@ -131,4 +128,53 @@ class NcmpCmHandleCreateSpec extends CpsIntegrationSpecBase {
         cleanup: 'deregister CM handles'
             deregisterCmHandles(DMI_URL, ['ch-1', 'ch-2', 'ch-3'])
     }
+
+    def 'CM Handle retry after failed module sync.'() {
+        given: 'DMI is not initially available to handle requests'
+            mockDmiIsNotAvailableForModuleSync(DMI_URL, 'ch-1')
+            mockDmiIsNotAvailableForModuleSync(DMI_URL, 'ch-2')
+        and: 'DMI will be available for retry'
+            mockDmiResponsesForModuleSync(DMI_URL, 'ch-1', MODULE_REFERENCES_RESPONSE_A, MODULE_RESOURCES_RESPONSE_A)
+            mockDmiResponsesForModuleSync(DMI_URL, 'ch-2', MODULE_REFERENCES_RESPONSE_B, MODULE_RESOURCES_RESPONSE_B)
+
+        when: 'CM-handles are registered for creation'
+            def cmHandlesToCreate = [new NcmpServiceCmHandle(cmHandleId: 'ch-1'), new NcmpServiceCmHandle(cmHandleId: 'ch-2')]
+            def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: DMI_URL, createdCmHandles: cmHandlesToCreate)
+            networkCmProxyDataService.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
+        and: 'module sync runs'
+            moduleSyncWatchdog.moduleSyncAdvisedCmHandles()
+        then: 'CM-handles go to LOCKED state'
+            new PollingConditions().within(3, () -> {
+                assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.LOCKED
+                assert objectUnderTest.getCmHandleCompositeState('ch-2').cmHandleState == CmHandleState.LOCKED
+            })
+
+        when: 'we wait for LOCKED CM handle retry time (actually just subtract 3 minutes from handles lastUpdateTime)'
+            overrideCmHandleLastUpdateTime('ch-1', OffsetDateTime.now().minusMinutes(3))
+            overrideCmHandleLastUpdateTime('ch-2', OffsetDateTime.now().minusMinutes(3))
+        and: 'failed CM handles are reset'
+            moduleSyncWatchdog.resetPreviouslyFailedCmHandles()
+        then: 'CM-handles are ADVISED state'
+            assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.ADVISED
+            assert objectUnderTest.getCmHandleCompositeState('ch-2').cmHandleState == CmHandleState.ADVISED
+
+        when: 'module sync runs'
+            moduleSyncWatchdog.moduleSyncAdvisedCmHandles()
+        then: 'CM-handles go to READY state'
+            new PollingConditions().within(3, () -> {
+                assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.READY
+                assert objectUnderTest.getCmHandleCompositeState('ch-2').cmHandleState == CmHandleState.READY
+            })
+        and: 'CM-handles have expected modules'
+            assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences('ch-1').moduleName.sort()
+            assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences('ch-2').moduleName.sort()
+        and: 'CM-handles have expected module set tags (blank)'
+            assert objectUnderTest.getNcmpServiceCmHandle('ch-1').moduleSetTag == ''
+            assert objectUnderTest.getNcmpServiceCmHandle('ch-2').moduleSetTag == ''
+        and: 'DMI received expected requests'
+            mockDmiServer.verify()
+
+        cleanup: 'deregister CM handle'
+            deregisterCmHandles(DMI_URL, ['ch-1', 'ch-2'])
+    }
 }
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleUpgradeSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleUpgradeSpec.groovy
index ffa4382d49..c5c59e05ca 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleUpgradeSpec.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleUpgradeSpec.groovy
@@ -55,7 +55,7 @@ class NcmpCmHandleUpgradeSpec extends CpsIntegrationSpecBase {
             assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort()
 
         and: 'DMI returns different modules for upgrade'
-            mockDmiResponsesForRegistration(DMI_URL, CM_HANDLE_ID, UPDATED_MODULE_REFERENCES_RESPONSE, UPDATED_MODULE_RESOURCES_RESPONSE)
+            mockDmiResponsesForModuleSync(DMI_URL, CM_HANDLE_ID, UPDATED_MODULE_REFERENCES_RESPONSE, UPDATED_MODULE_RESOURCES_RESPONSE)
 
         when: "CM-handle is upgraded with given moduleSetTag '${updatedModuleSetTag}'"
             def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: updatedModuleSetTag)