Pause Upon VF Module Failure

Issue-ID: VID-862

Currently in VID, once a failure is received from SO on the
instantiation of a VF Module, it is assumed that other VF Modules may
continue with the execution without stopping. However, in most of the
cases, it is a good idea to stop further requests to SO if one of the VF
Module instantiation returns an error. This user story, makes sure this
functionality is achieved.

Change-Id: I07c51b81111f805b61c8b3714a65171d7cbaa254
Signed-off-by: PATTANAYAK, SAUMYA SWARUP (sp931a) <sp931a@att.com>

7 files changed, 122 insertions, 37 deletions

diff --git a/vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java b/vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java
index f4dd6780a..e5da62adb 100644
--- a/vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java
+++ b/vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java
@@ -135,7 +135,8 @@ public class AsyncInstantiationBase extends BaseMsoApiTest {
     }
 
     protected boolean getExpectedRetryEnabled(JobStatus jobStatus) {
-        return Features.FLAG_1902_RETRY_JOB.isActive() && (jobStatus==JobStatus.FAILED || jobStatus==JobStatus.COMPLETED_WITH_ERRORS);
+        return Features.FLAG_1902_RETRY_JOB.isActive() && (jobStatus==JobStatus.FAILED || jobStatus==JobStatus.COMPLETED_WITH_ERRORS
+                || jobStatus==JobStatus.FAILED_AND_PAUSED);
     }
 
     public List<BasePreset> getPresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
@@ -203,7 +204,7 @@ public class AsyncInstantiationBase extends BaseMsoApiTest {
         return ImmutableList.of(
                 vidAuditStatus(jobId, "PENDING", false),
                 vidAuditStatus(jobId, "IN_PROGRESS", false),
-                vidAuditStatus(jobId, "COMPLETED_WITH_ERRORS", true)
+                vidAuditStatus(jobId, "COMPLETED_WITH_ERROR", true)
         );
     }
 
@@ -214,7 +215,13 @@ public class AsyncInstantiationBase extends BaseMsoApiTest {
                 vidAuditStatus(jobId, "FAILED", true)
         );
     }
-
+    protected ImmutableList<JobAuditStatus> vidAuditStatusesFailedAndPaused(String jobId) {
+        return ImmutableList.of(
+                vidAuditStatus(jobId, "PENDING", false),
+                vidAuditStatus(jobId, "IN_PROGRESS", false),
+                vidAuditStatus(jobId, "FAILED_AND_PAUSED", true)
+        );
+    }
     protected JobAuditStatus vidAuditStatus(String jobId, String jobStatus, boolean isFinal) {
         return new JobAuditStatus(UUID.fromString(jobId), jobStatus, JobAuditStatus.SourceStatus.VID, null, null, isFinal);
     }
@@ -578,4 +585,8 @@ public class AsyncInstantiationBase extends BaseMsoApiTest {
             org.junit.Assert.assertEquals("MSO instanceType  #" + i + " is not as expected", expectedStatus.getInstanceType(), actualStatus.getInstanceType());
         });
     }
+    protected static JobStatus getErrorStatus() {
+            return Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE.isActive() ?
+                            JobStatus.FAILED_AND_PAUSED : JobStatus.COMPLETED_WITH_ERRORS;
+   }
 }
diff --git a/vid-automation/src/main/java/org/onap/vid/api/CreateServiceWithFailedVnf.java b/vid-automation/src/main/java/org/onap/vid/api/CreateServiceWithFailedVnf.java
index 30076a964..a8bcc9f75 100644
--- a/vid-automation/src/main/java/org/onap/vid/api/CreateServiceWithFailedVnf.java
+++ b/vid-automation/src/main/java/org/onap/vid/api/CreateServiceWithFailedVnf.java
@@ -13,6 +13,7 @@ import org.onap.simulator.presetGenerator.presets.sdc.PresetSDCGetServiceToscaMo
 import org.onap.vid.model.asyncInstantiation.ServiceInfo;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
+import vid.automation.test.infra.Features;
 import vid.automation.test.infra.ModelInfo;
 import vid.automation.test.model.JobStatus;
 import vid.automation.test.model.ServiceAction;
@@ -105,9 +106,13 @@ Legit Preset  ||  deploy 1 Service, 1 VNF which will fail
     public void firstTimeAssertion() {
         assertThat(uuids, hasSize(1));
         originalJobId = uuids.get(0);
+        boolean isPauseOnFailureFlagOn = Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE.isActive();
 
-        asyncInstantiationBase.assertServiceInfoSpecific1(originalJobId, JobStatus.COMPLETED_WITH_ERRORS, names.get(SERVICE_NAME), "us16807000", firstIds.serviceId, ServiceAction.INSTANTIATE);
-        asyncInstantiationBase.assertAuditStatuses(originalJobId, asyncInstantiationBase.vidAuditStatusesCompletedWithErrors(originalJobId),null);
+        asyncInstantiationBase.assertServiceInfoSpecific1(originalJobId, isPauseOnFailureFlagOn ?
+                JobStatus.FAILED_AND_PAUSED : JobStatus.COMPLETED_WITH_ERRORS, names.get(SERVICE_NAME), "us16807000", firstIds.serviceId, ServiceAction.INSTANTIATE);
+        asyncInstantiationBase.assertAuditStatuses(originalJobId, isPauseOnFailureFlagOn ?
+                asyncInstantiationBase.vidAuditStatusesFailedAndPaused(originalJobId) :
+                asyncInstantiationBase.vidAuditStatusesCompletedWithErrors(originalJobId),null);
         assertThat(SimulatorApi.retrieveRecordedRequestsPathCounter(), allOf(
                 hasOrLacksOfEntry(createPresets.get(0).getReqPath(), 1L),
                 hasOrLacksOfEntry(createPresets.get(1).getReqPath(), 1L),
diff --git a/vid-automation/src/main/java/vid/automation/test/infra/Features.java b/vid-automation/src/main/java/vid/automation/test/infra/Features.java
index 3e499de2f..ba7ebdf0e 100644
--- a/vid-automation/src/main/java/vid/automation/test/infra/Features.java
+++ b/vid-automation/src/main/java/vid/automation/test/infra/Features.java
@@ -71,6 +71,7 @@ public enum Features implements Feature {
     FLAG_2008_CREATE_VFMODULE_INSTANTIATION_ORDER_NUMBER,
     FLAG_2008_PAUSE_INSTANTIATION_ON_VFMODULE_POPUP,
     FLAG_2008_REMOVE_PAUSE_INSTANTIATION,
+    FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE,
     ;
     public boolean isActive() {
         return FeatureContext.getFeatureManager().isActive(this);
diff --git a/vid-automation/src/main/java/vid/automation/test/model/JobStatus.java b/vid-automation/src/main/java/vid/automation/test/model/JobStatus.java
index bbe1e3e86..cfe32bc9a 100644
--- a/vid-automation/src/main/java/vid/automation/test/model/JobStatus.java
+++ b/vid-automation/src/main/java/vid/automation/test/model/JobStatus.java
@@ -9,5 +9,6 @@ public enum JobStatus {
     PENDING,
     STOPPED,
     COMPLETED_AND_PAUSED,
-    COMPLETED_WITH_ERRORS;
+    COMPLETED_WITH_ERRORS,
+    FAILED_AND_PAUSED;
 }
diff --git a/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationALaCarteApiTest.java b/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationALaCarteApiTest.java
index 20fce4012..c032d5fed 100644
--- a/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationALaCarteApiTest.java
+++ b/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationALaCarteApiTest.java
@@ -208,7 +208,7 @@ public class AsyncInstantiationALaCarteApiTest extends AsyncInstantiationBase {
 
         assertThat(uuids, hasSize(1));
         final String jobId = uuids.get(0);
-        assertServiceInfoSpecificDeletion(jobId, JobStatus.COMPLETED_WITH_ERRORS, "SERVICE_INSTANCE_NAME", "service-instance-type");
+        assertServiceInfoSpecificDeletion(jobId, getErrorStatus(), "SERVICE_INSTANCE_NAME", "service-instance-type");
         assertThat(SimulatorApi.retrieveRecordedRequestsPathCounter(), allOf(
                 TestUtils.hasOrLacksOfEntry(deletePresets.get(0).getReqPath(), 1L),
                 TestUtils.hasOrLacksOfEntry(deletePresets.get(1).getReqPath(), 1L),
@@ -324,29 +324,42 @@ public class AsyncInstantiationALaCarteApiTest extends AsyncInstantiationBase {
         String parentServiceInstanceId = "service-instance-id";
         String vnfGroupToDeleteInstanceId = "VNF_GROUP1_INSTANCE_ID";
 
+        SimulatorApi.clearExpectations();
+
         //failed to create vnf group, failed to remove 1 member (and then also vnf group isn't deleted)
         viewEditVnfGroup_registerPresets(parentServiceInstanceId, vnfGroupToDeleteInstanceId, MSO_FAILED_STATUS);
 
         final List<String> uuids = createBulkOfInstances(false, 1, ImmutableMap.of(), VIEW_EDIT_VNF_GROUPS_REQUEST);
 
+        try {
+            Thread.sleep(20000);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+
         assertThat(uuids, hasSize(1));
         final String jobId = uuids.get(0);
 
-        assertServiceInfoSpecificUpdate(jobId, JobStatus.COMPLETED_WITH_ERRORS, "SERVICE_INSTANCE_NAME");
-        assertExpectedStatus(JobStatus.COMPLETED_WITH_ERRORS, jobId);
+        assertServiceInfoSpecificUpdate(jobId, getErrorStatus(), "SERVICE_INSTANCE_NAME");
+        assertExpectedStatus(getErrorStatus(), jobId);
         Map<String, Long> recordedRequest = SimulatorApi.retrieveRecordedRequestsPathCounter();
         assertThat(recordedRequest, allOf(
                 TestUtils.hasOrLacksOfEntry("/mso/serviceInstantiation/v./instanceGroups/" + vnfGroupToDeleteInstanceId, 0L), //delete vnf group
                 TestUtils.hasOrLacksOfEntry("/mso/serviceInstantiation/v./instanceGroups", 1L), //create vnf group
                 TestUtils.hasOrLacksOfEntry("/mso/serviceInstantiation/v./instanceGroups/" + vnfGroupToDeleteInstanceId + "/removeMembers", 3L) //remove vnf group members
         ));
-
+        SimulatorApi.clearExpectations();
         //retry - vnf group create, 1 member remove, vnf group delete
         viewEditVnfGroup_registerPresets(parentServiceInstanceId, vnfGroupToDeleteInstanceId, MSO_COMPLETE_STATUS);
         final List<String> retryUuids = retryJob(jobId);
         assertThat(retryUuids, hasSize(1));
         final String retryJobId = retryUuids.get(0);
 
+        try {
+            Thread.sleep(20000);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
         assertServiceInfoSpecificUpdate(retryJobId, JobStatus.COMPLETED, "SERVICE_INSTANCE_NAME");
         assertExpectedStatus(JobStatus.COMPLETED, retryJobId);
         assertThat(SimulatorApi.retrieveRecordedRequestsPathCounter(), allOf(
@@ -406,26 +419,67 @@ public class AsyncInstantiationALaCarteApiTest extends AsyncInstantiationBase {
 
     @DataProvider
     Object[][] data_deploy1ServiceFromCypress__verifyStatusAndMsoCalls() {
-        return Features.FLAG_ASYNC_ALACARTE_VFMODULE.isActive() ? new Object[][]{
-                {"none", emptyMap(), emptyMap(), true},
-                {"none", emptyMap(), emptyMap(), false},
-                {"instance",  ImmutableMap.of("vnfs", 0L, "networks", 0L, "vfModules", 0L, "volumeGroups", 0L),
-                        ImmutableMap.of("serviceInstances", 1L, "vnfs", 1L, "networks", 1L, "vfModules", 3L, "volumeGroups", 1L),true},
-                {"network", emptyMap(),
-                        ImmutableMap.of("networks", 1L), true},
-                {"vnf0", ImmutableMap.of("vfModules", 0L, "volumeGroups", 0L),
-                        ImmutableMap.of("vnfs", 1L, "vfModules", 3L, "volumeGroups", 1L), true},
-                {"vfModule0", ImmutableMap.of("vfModules", 1L, "volumeGroups", 0L),
-                        ImmutableMap.of("vfModules", 3L, "volumeGroups", 1L), true},
-                {"volumeGroup", ImmutableMap.of("vfModules", 2L),
-                        ImmutableMap.of("vfModules", 1L, "volumeGroups", 1L), true},
-                {"vfModule1", emptyMap(),
-                        ImmutableMap.of("vfModules", 1L, "volumeGroups", 1L), true},
-                {"vfModule2", emptyMap(),
-                        ImmutableMap.of("vfModules", 1L), true}
-        } : new Object[][]{
-                {"none", ImmutableMap.of("vfModules", 0L, "volumeGroups", 0L), emptyMap(), true}
-        };
+        boolean isAsyncAlacarteVfModuleWithPauseOnFailure = Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE.isActive() &&
+                Features.FLAG_ASYNC_ALACARTE_VFMODULE.isActive() ;
+
+        if(isAsyncAlacarteVfModuleWithPauseOnFailure) {
+            return new Object[][] {
+                    {"none", ImmutableMap.of("vfModules", 3L), emptyMap() , true},
+                    {"none", ImmutableMap.of("vfModules", 3L), emptyMap() , false},
+                    {"instance",
+                            ImmutableMap.of("vnfs", 0L, "networks", 0L, "vfModules", 0L, "volumeGroups", 0L),
+                            ImmutableMap.of("serviceInstances", 1L, "vnfs", 1L, "networks", 1L, "vfModules", 3L, "volumeGroups", 1L),
+                            true},
+                    {"network",
+                            ImmutableMap.of("vnfs", 1L, "networks", 1L, "vfModules", 3L, "volumeGroups", 1L, "serviceInstances", 1L),
+                            ImmutableMap.of("networks", 1L),
+                            true},
+                    {"vnf0",
+                            ImmutableMap.of("vfModules", 0L, "volumeGroups", 0L),
+                            ImmutableMap.of("vnfs", 1L, "vfModules", 3L, "volumeGroups", 1L),
+                            true},
+                    {"vfModule0", ImmutableMap.of("vfModules", 1L, "volumeGroups", 0L),
+                            ImmutableMap.of("vfModules", 3L, "volumeGroups", 1L),
+                            true},
+//                    {"volumeGroup",
+//                            ImmutableMap.of("vnfs", 1L, "networks", 1L, "vfModules", 1L, "volumeGroups", 1L, "serviceInstances", 1L),
+//                            ImmutableMap.of("vfModules", 1L, "volumeGroups", 1L),
+//                            true},
+                    {"vfModule1",
+                            ImmutableMap.of("vfModules", 2L, "volumeGroups", 1L) ,
+                            ImmutableMap.of("vfModules", 2L, "volumeGroups", 1L) ,
+                            true},
+                    {"vfModule2",
+                            ImmutableMap.of("vfModules", 3L, "volumeGroups", 1L),
+                            ImmutableMap.of("vfModules", 1L, "volumeGroups", 0L),
+                            true }
+            };
+        } else if (Features.FLAG_ASYNC_ALACARTE_VFMODULE.isActive()) {
+            return new Object[][]{
+                    {"none", emptyMap(), emptyMap(), true},
+                    {"none", emptyMap(), emptyMap(), false},
+                    {"instance", ImmutableMap.of("vnfs", 0L, "networks", 0L, "vfModules", 0L, "volumeGroups", 0L),
+                            ImmutableMap.of("serviceInstances", 1L, "vnfs", 1L, "networks", 1L, "vfModules", 3L, "volumeGroups",
+                                    1L), true},
+                    {"network", emptyMap(),
+                            ImmutableMap.of("networks", 1L), true},
+                    {"vnf0", ImmutableMap.of("vfModules", 0L, "volumeGroups", 0L),
+                            ImmutableMap.of("vnfs", 1L, "vfModules", 3L, "volumeGroups", 1L), true},
+                    {"vfModule0", ImmutableMap.of("vfModules", 1L, "volumeGroups", 0L),
+                            ImmutableMap.of("vfModules", 3L, "volumeGroups", 1L), true},
+                    {"volumeGroup", ImmutableMap.of("vfModules", 2L),
+                            ImmutableMap.of("vfModules", 1L, "volumeGroups", 1L), true},
+                    {"vfModule1", emptyMap(),
+                            ImmutableMap.of("vfModules", 1L, "volumeGroups", 1L), true},
+                    {"vfModule2", emptyMap(),
+                            ImmutableMap.of("vfModules", 1L), true}
+            };
+        } else {
+            return new Object[][]{
+                    {"none", ImmutableMap.of("vfModules", 0L, "volumeGroups", 0L), emptyMap(), true}
+            };
+        }
+
     }
 
     @Test(dataProvider = "data_deploy1ServiceFromCypress__verifyStatusAndMsoCalls")
@@ -443,7 +497,7 @@ public class AsyncInstantiationALaCarteApiTest extends AsyncInstantiationBase {
         switch (whatToFail) {
             case "none": finalJobStatus = JobStatus.COMPLETED; break;
             case "instance": finalJobStatus = JobStatus.FAILED; break;
-            default: finalJobStatus = JobStatus.COMPLETED_WITH_ERRORS; break;
+            default: finalJobStatus = getErrorStatus(); break;
         }
         assertServiceInfoSpecific2(jobId, finalJobStatus, names.get(SERVICE_NAME));
         assertRecordedRequests(pathCounterOverride, 1L, vnfRequestId);
@@ -486,7 +540,8 @@ public class AsyncInstantiationALaCarteApiTest extends AsyncInstantiationBase {
 
         final String msoURL = "/mso";
 
-        deploy1ServiceFromCypress__verifyStatusAndMsoCalls_andRetry("none", emptyMap(), emptyMap(), true);
+        deploy1ServiceFromCypress__verifyStatusAndMsoCalls_andRetry("none",
+                Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE.isActive() ? ImmutableMap.of("vfModules", 3L):emptyMap(), emptyMap(), true);
         List<String> logLines =  LoggerFormatTest.getLogLinesAsList(LogName.metrics2019, 200, 1, restTemplate, uri);
         List<RecordedRequests> underTestRequests = retrieveRecordedRequests();
 
@@ -603,7 +658,8 @@ public class AsyncInstantiationALaCarteApiTest extends AsyncInstantiationBase {
                         pathCounterOverride.getOrDefault("volumeGroups", defaultValue)),
 
                 TestUtils.hasOrLacksOfEntry("/mso/serviceInstantiation/v./serviceInstances/" + DEFAULT_INSTANCE_ID + "/vnfs/" + vnfRequestId + "/vfModules",
-                        pathCounterOverride.getOrDefault("vfModules", vfModulesDefaultValue))
+                        pathCounterOverride.getOrDefault("vfModules",
+                                Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE.isActive() ? defaultValue : vfModulesDefaultValue))
         ));
     }
 
diff --git a/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationALaCarteApiTest3.java b/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationALaCarteApiTest3.java
index 07b81a46c..2e21653d0 100644
--- a/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationALaCarteApiTest3.java
+++ b/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationALaCarteApiTest3.java
@@ -135,7 +135,9 @@ public class AsyncInstantiationALaCarteApiTest3 extends AsyncInstantiationBase {
         } else if (expectedStatus.equals("SERVICE_FAILED")){
             vidAuditStatuses = vidAuditStatusesFailed(jobId);
         } else {
-            vidAuditStatuses = vidAuditStatusesCompletedWithErrors(jobId);
+            vidAuditStatuses = Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE.isActive() ?
+                                vidAuditStatusesFailedAndPaused(jobId) :
+                                vidAuditStatusesCompletedWithErrors(jobId);
         }
 
         assertServiceInfoSpecific3(jobId, expectedJobStatus , names.get(SERVICE_NAME));
@@ -145,7 +147,12 @@ public class AsyncInstantiationALaCarteApiTest3 extends AsyncInstantiationBase {
 
     @DataProvider
     Object[][] multipleVnfDataProvider() {
-        return new Object[][]{{MSO_FAILED_STATUS, JobStatus.COMPLETED_WITH_ERRORS},{MSO_COMPLETE_STATUS, JobStatus.COMPLETED}, {"SERVICE_FAILED", JobStatus.FAILED}};
+        return new Object[][]{
+                {
+                    MSO_FAILED_STATUS, Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE.isActive() ? JobStatus.FAILED_AND_PAUSED :
+                                JobStatus.COMPLETED_WITH_ERRORS },
+                {   MSO_COMPLETE_STATUS, JobStatus.COMPLETED },
+                {   "SERVICE_FAILED", JobStatus.FAILED } };
     }
 
     @Test
@@ -197,7 +204,9 @@ public class AsyncInstantiationALaCarteApiTest3 extends AsyncInstantiationBase {
         return new Object[][]{
                 {MSO_COMPLETE_STATUS, MSO_COMPLETE_STATUS, JobStatus.COMPLETED},
                 {MSO_FAILED_STATUS, MSO_FAILED_STATUS, JobStatus.FAILED},
-                {MSO_COMPLETE_STATUS, MSO_FAILED_STATUS, JobStatus.COMPLETED_WITH_ERRORS}
+                {MSO_FAILED_STATUS, MSO_COMPLETE_STATUS,
+                        Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE.isActive() ?
+                                JobStatus.FAILED_AND_PAUSED : JobStatus.COMPLETED_WITH_ERRORS }
         };
     }
 
@@ -535,6 +544,8 @@ public class AsyncInstantiationALaCarteApiTest3 extends AsyncInstantiationBase {
                 return vidAuditStatusesFailed(jobId);
             case IN_PROGRESS:
                 return vidAuditStatusesInProgress(jobId);
+            case FAILED_AND_PAUSED:
+                return vidAuditStatusesFailedAndPaused(jobId);
         }
 
         return null;
diff --git a/vid-automation/src/test/resources/features.properties b/vid-automation/src/test/resources/features.properties
index a1a6cf733..35a044fdb 100644
--- a/vid-automation/src/test/resources/features.properties
+++ b/vid-automation/src/test/resources/features.properties
@@ -33,4 +33,4 @@ FLAG_1911_INSTANTIATION_ORDER_IN_ASYNC_ALACARTE = false
 FLAG_SHOW_ORCHESTRATION_TYPE = false
 FLAG_1911_INSTANTIATION_ORDER_BUTTON_IN_ASYNC_ALACARTE = false
 FLAG_A_LA_CARTE_PLATFORM_MULTI_SELECT = false
-
+FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE = false