Pause Upon VF Module Failure

Issue-ID: VID-862

Currently in VID, once a failure is received from SO on the
instantiation of a VF Module, it is assumed that other VF Modules may
continue with the execution without stopping. However, in most of the
cases, it is a good idea to stop further requests to SO if one of the VF
Module instantiation returns an error. This user story, makes sure this
functionality is achieved.

Change-Id: I07c51b81111f805b61c8b3714a65171d7cbaa254
Signed-off-by: PATTANAYAK, SAUMYA SWARUP (sp931a) <sp931a@att.com>

9 files changed, 68 insertions, 19 deletions

diff --git a/vid-app-common/src/main/java/org/onap/vid/job/Job.java b/vid-app-common/src/main/java/org/onap/vid/job/Job.java
index 9861d0703..230e70fa6 100644
--- a/vid-app-common/src/main/java/org/onap/vid/job/Job.java
+++ b/vid-app-common/src/main/java/org/onap/vid/job/Job.java
@@ -68,6 +68,7 @@ public interface Job {
         CREATING(false),
         PENDING_RESOURCE(false),
         COMPLETED_AND_PAUSED(true, false),
+        FAILED_AND_PAUSED(true, true),
         ;
 
         private final Boolean finalStatus;
diff --git a/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt b/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt
index 502d8344d..09357e519 100644
--- a/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt
+++ b/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt
@@ -25,9 +25,11 @@ import org.onap.portalsdk.core.service.DataAccessService
 import org.onap.vid.job.Job
 import org.onap.vid.job.Job.JobStatus.*
 import org.onap.vid.job.impl.JobDaoImpl
+import org.onap.vid.properties.Features
 import org.onap.vid.utils.DaoUtils
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Service
+import org.togglz.core.manager.FeatureManager
 import java.util.*
 import java.util.stream.Collectors
 import java.util.stream.Stream
@@ -35,7 +37,7 @@ import java.util.stream.Stream
 
 @Service
 class WatchChildrenJobsBL @Autowired
-constructor(private val dataAccessService: DataAccessService) {
+constructor(private val dataAccessService: DataAccessService, private val featureManager: FeatureManager) {
 
     fun retrieveChildrenJobsStatus(childrenJobsIds: List<String>): Job.JobStatus {
         val jobs = getAllChildrenJobs(childrenJobsIds)
@@ -47,14 +49,16 @@ constructor(private val dataAccessService: DataAccessService) {
                     else -> it.status
                 }}
 
-        return cumulateJobStatus(jobsStatuses)
+        return if(featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE))
+            cumulateJobStatusWithPauseOnFailure(jobsStatuses) else cumulateJobStatus(jobsStatuses)
 
     }
 
     fun cumulateJobStatus(childrenComulatedStatus: Job.JobStatus, fatherJobStatus: Job.JobStatus): Job.JobStatus {
-        return cumulateJobStatus(Stream.of(childrenComulatedStatus, fatherJobStatus))
+        return if(featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE))
+            cumulateJobStatusWithPauseOnFailure(Stream.of(childrenComulatedStatus, fatherJobStatus))
+                else cumulateJobStatus(Stream.of(childrenComulatedStatus, fatherJobStatus))
     }
-
     private fun cumulateJobStatus(jobsStatuses: Stream<Job.JobStatus>): Job.JobStatus {
 
         return jobsStatuses.reduce{ a, b ->
@@ -70,7 +74,21 @@ constructor(private val dataAccessService: DataAccessService) {
             }
         } .orElse(COMPLETED_WITH_NO_ACTION)
   }
+    private fun cumulateJobStatusWithPauseOnFailure(jobsStatuses: Stream<Job.JobStatus>): Job.JobStatus {
 
+        return jobsStatuses.reduce{ a, b ->
+            when {
+                a == FAILED_AND_PAUSED || b == FAILED_AND_PAUSED-> FAILED_AND_PAUSED
+                a == COMPLETED && b.isFailure -> FAILED_AND_PAUSED
+                b == COMPLETED && a.isFailure -> FAILED_AND_PAUSED
+                !a.isFinal || !b.isFinal -> IN_PROGRESS
+                a == COMPLETED_AND_PAUSED || b == COMPLETED_AND_PAUSED -> COMPLETED_AND_PAUSED
+                a == COMPLETED || b == COMPLETED -> COMPLETED
+                a.isFailure || b.isFailure -> FAILED
+                else ->  COMPLETED_WITH_NO_ACTION
+            }
+        } .orElse(COMPLETED_WITH_NO_ACTION)
+    }
     private fun getAllChildrenJobs(childrenJobsIds: List<String>): Map<UUID, JobDaoImpl> {
         val jobs:MutableList<JobDaoImpl> = dataAccessService.getList(JobDaoImpl::class.java, filterByJobIds(childrenJobsIds), null, DaoUtils.getPropsMap()) as MutableList<JobDaoImpl>
         return jobs.stream().collect(Collectors.toMap( { it.uuid }, { it }))
diff --git a/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java b/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java
index 51803891a..7391f41a9 100644
--- a/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java
+++ b/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java
@@ -45,12 +45,14 @@ import org.onap.vid.exceptions.GenericUncheckedException;
 import org.onap.vid.exceptions.OperationNotAllowedException;
 import org.onap.vid.job.Job;
 import org.onap.vid.job.JobsBrokerService;
+import org.onap.vid.properties.Features;
 import org.onap.vid.properties.VidProperties;
 import org.onap.vid.services.VersionService;
 import org.onap.vid.utils.DaoUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import org.togglz.core.manager.FeatureManager;
 
 @Service
 public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService {
@@ -64,19 +66,21 @@ public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService {
     private int pollingIntervalSeconds;
 
     private final VersionService versionService;
-
+    private final FeatureManager featureManager;
     @Autowired
     public JobsBrokerServiceInDatabaseImpl(DataAccessService dataAccessService,
                                            SessionFactory sessionFactory,
                                            @Value("0") int maxOpenedInstantiationRequestsToMso,
                                            @Value("10") int pollingIntervalSeconds,
-                                           VersionService versionService) {
+                                           VersionService versionService,
+                                           FeatureManager featureManager) {
         // tha @Value will inject conservative defaults; overridden in @PostConstruct from configuration
         this.dataAccessService = dataAccessService;
         this.sessionFactory = sessionFactory;
         this.maxOpenedInstantiationRequestsToMso = maxOpenedInstantiationRequestsToMso;
         this.pollingIntervalSeconds = pollingIntervalSeconds;
         this.versionService = versionService;
+        this.featureManager = featureManager;
     }
 
     @PostConstruct
@@ -199,7 +203,7 @@ public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService {
                 "and TEMPLATE_Id not in \n" +
                 "(select TEMPLATE_Id from vid_job where" +
                 "   TEMPLATE_Id IS NOT NULL and("+
-                "   (JOB_STATUS='FAILED' and DELETED_AT is null)" + // failed but not deleted
+                "   (JOB_STATUS IN('FAILED','FAILED_AND_PAUSED') and DELETED_AT is null)" + // failed but not deleted
                 "   or JOB_STATUS='IN_PROGRESS'" +
                 "   or TAKEN_BY IS NOT NULL))" + " \n " +
                 // prefer older jobs, but the earlier in each bulk
@@ -232,13 +236,24 @@ public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService {
                 "    and in_progress_count is NULL \n" +
                 //and that have valid templateId
                 "    and JOB.template_id is not NULL \n"+
-                ")\n" +
+
+                filterFailedStatusForPendingResource()
+
+                + ")" +
                 //INDEX_IN_BULK is for order them inside same templateId,
                 //template_id - for order between different templateId (just to be deterministic)
                 "order by INDEX_IN_BULK,JOB.template_id \n" +
                 "limit 1;";
     }
-
+    private String filterFailedStatusForPendingResource() {
+        String sql = "and JOB.template_id not in \n" +
+                "(select TEMPLATE_Id from vid_job where" +
+                "   TEMPLATE_Id IS NOT NULL and (JOB_STATUS IN('FAILED','FAILED_AND_PAUSED') "
+                + " AND JOB_TYPE NOT IN('NetworkInstantiation','InstanceGroup','InstanceGroupMember') and DELETED_AT is null)" + // failed but not deleted
+                "   or TAKEN_BY IS NOT NULL)";
+        return featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE) ?
+                sql : "";
+    }
 
     @Override
     public void pushBack(Job job) {
diff --git a/vid-app-common/src/main/java/org/onap/vid/properties/Features.java b/vid-app-common/src/main/java/org/onap/vid/properties/Features.java
index f863b3025..9d3faa062 100644
--- a/vid-app-common/src/main/java/org/onap/vid/properties/Features.java
+++ b/vid-app-common/src/main/java/org/onap/vid/properties/Features.java
@@ -102,6 +102,7 @@ public enum Features implements Feature {
     FLAG_2008_CREATE_VFMODULE_INSTANTIATION_ORDER_NUMBER,
     FLAG_2008_PAUSE_INSTANTIATION_ON_VFMODULE_POPUP,
     FLAG_2008_REMOVE_PAUSE_INSTANTIATION,
+    FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE,
     ;
 
     public boolean isActive() {
diff --git a/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java b/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java
index 91801973b..bf0344c6b 100644
--- a/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java
+++ b/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java
@@ -436,7 +436,7 @@ public class AsyncInstantiationBusinessLogicImpl implements
 
     private boolean isRetryEnabledForStatus(JobStatus jobStatus) {
         return jobStatus==JobStatus.COMPLETED_AND_PAUSED || (featureManager.isActive(Features.FLAG_1902_RETRY_JOB) &&
-                (jobStatus==JobStatus.COMPLETED_WITH_ERRORS || jobStatus==JobStatus.FAILED));
+                (jobStatus==JobStatus.COMPLETED_WITH_ERRORS || jobStatus==JobStatus.FAILED || jobStatus==JobStatus.FAILED_AND_PAUSED));
     }
 
     private void setServiceInfoStatus(ServiceInfo serviceInfo, JobStatus jobStatus) {
diff --git a/vid-app-common/src/main/webapp/WEB-INF/conf/dev.features.properties b/vid-app-common/src/main/webapp/WEB-INF/conf/dev.features.properties
index 92be1c74b..2c73281b2 100644
--- a/vid-app-common/src/main/webapp/WEB-INF/conf/dev.features.properties
+++ b/vid-app-common/src/main/webapp/WEB-INF/conf/dev.features.properties
@@ -34,3 +34,4 @@ FLAG_SHOW_ORCHESTRATION_TYPE = false
 FLAG_1911_INSTANTIATION_ORDER_BUTTON_IN_ASYNC_ALACARTE = false
 FLAG_2002_ANY_ALACARTE_BESIDES_EXCLUDED_NEW_INSTANTIATION_UI = false
 FLAG_GUILIN_CHANGEMG_SUBMIT_TO_SO = true
+FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE = false
\ No newline at end of file
diff --git a/vid-app-common/src/test/java/org/onap/vid/config/JobCommandsConfigWithMockedMso.java b/vid-app-common/src/test/java/org/onap/vid/config/JobCommandsConfigWithMockedMso.java
index 468fb41c3..ecec705a1 100644
--- a/vid-app-common/src/test/java/org/onap/vid/config/JobCommandsConfigWithMockedMso.java
+++ b/vid-app-common/src/test/java/org/onap/vid/config/JobCommandsConfigWithMockedMso.java
@@ -79,7 +79,7 @@ public class JobCommandsConfigWithMockedMso {
 
     @Bean
     public JobsBrokerService jobsBrokerService(DataAccessService dataAccessService, SessionFactory sessionFactory, VersionService versionService) {
-        return new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, 200, 0,versionService);
+        return new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, 200, 0,versionService,featureManager());
     }
 
     @Bean
@@ -314,7 +314,7 @@ public class JobCommandsConfigWithMockedMso {
 
     @Bean
     public WatchChildrenJobsBL watchChildrenJobsService(DataAccessService dataAccessService) {
-        return new WatchChildrenJobsBL(dataAccessService);
+        return new WatchChildrenJobsBL(dataAccessService,featureManager());
     }
 
 }
diff --git a/vid-app-common/src/test/java/org/onap/vid/job/command/WatchChildrenJobsBLTest.java b/vid-app-common/src/test/java/org/onap/vid/job/command/WatchChildrenJobsBLTest.java
index fad32bba6..a8613f7a7 100644
--- a/vid-app-common/src/test/java/org/onap/vid/job/command/WatchChildrenJobsBLTest.java
+++ b/vid-app-common/src/test/java/org/onap/vid/job/command/WatchChildrenJobsBLTest.java
@@ -37,10 +37,12 @@ import org.mockito.MockitoAnnotations;
 import org.onap.portalsdk.core.service.DataAccessService;
 import org.onap.vid.job.Job.JobStatus;
 import org.onap.vid.job.impl.JobDaoImpl;
+import org.onap.vid.properties.Features;
 import org.onap.vid.utils.DaoUtils;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
+import org.togglz.core.manager.FeatureManager;
 
 public class WatchChildrenJobsBLTest {
     @Mock
@@ -49,6 +51,9 @@ public class WatchChildrenJobsBLTest {
     @InjectMocks
     private WatchChildrenJobsBL watchChildrenJobsBL;
 
+    @Mock
+    private static FeatureManager featureManager;
+
     @BeforeClass
     public void initMocks() {
         MockitoAnnotations.initMocks(this);
@@ -80,16 +85,16 @@ public class WatchChildrenJobsBLTest {
                 {Arrays.asList(JobStatus.COMPLETED, JobStatus.COMPLETED), JobStatus.COMPLETED},
                 {Arrays.asList(JobStatus.COMPLETED, JobStatus.COMPLETED_WITH_NO_ACTION), JobStatus.COMPLETED},
                 {Arrays.asList(JobStatus.FAILED, JobStatus.COMPLETED_WITH_NO_ACTION), JobStatus.FAILED},
-                {Arrays.asList(JobStatus.FAILED, JobStatus.COMPLETED), JobStatus.COMPLETED_WITH_ERRORS},
+                {Arrays.asList(JobStatus.FAILED, JobStatus.COMPLETED), getErrorStatus()},
                 {Arrays.asList(JobStatus.RESOURCE_IN_PROGRESS, JobStatus.FAILED), JobStatus.IN_PROGRESS},
                 {Arrays.asList(JobStatus.PAUSE, JobStatus.FAILED), JobStatus.IN_PROGRESS},
                 {Arrays.asList(JobStatus.PENDING, JobStatus.FAILED), JobStatus.IN_PROGRESS},
                 {Arrays.asList(JobStatus.IN_PROGRESS, JobStatus.COMPLETED), JobStatus.IN_PROGRESS},
                 {Arrays.asList(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS),  JobStatus.IN_PROGRESS},
-                {Arrays.asList(JobStatus.COMPLETED, JobStatus.COMPLETED_WITH_ERRORS), JobStatus.COMPLETED_WITH_ERRORS},
-                {Arrays.asList(JobStatus.COMPLETED_WITH_ERRORS, JobStatus.FAILED), JobStatus.COMPLETED_WITH_ERRORS},
-                {Arrays.asList(JobStatus.COMPLETED_WITH_ERRORS, JobStatus.COMPLETED_WITH_ERRORS), JobStatus.COMPLETED_WITH_ERRORS},
-                {Arrays.asList(JobStatus.COMPLETED_WITH_ERRORS, JobStatus.COMPLETED_WITH_NO_ACTION), JobStatus.COMPLETED_WITH_ERRORS},
+                {Arrays.asList(JobStatus.COMPLETED, getErrorStatus()), getErrorStatus()},
+                {Arrays.asList(getErrorStatus(), JobStatus.FAILED), getErrorStatus()},
+                {Arrays.asList(getErrorStatus(), getErrorStatus()), getErrorStatus()},
+                {Arrays.asList(getErrorStatus(), JobStatus.COMPLETED_WITH_NO_ACTION), getErrorStatus()},
                 {Arrays.asList(JobStatus.COMPLETED_WITH_NO_ACTION, JobStatus.COMPLETED_WITH_NO_ACTION), JobStatus.COMPLETED_WITH_NO_ACTION},
                 {Arrays.asList(JobStatus.COMPLETED_AND_PAUSED, JobStatus.RESOURCE_IN_PROGRESS), JobStatus.IN_PROGRESS},
                 {Arrays.asList(JobStatus.COMPLETED_AND_PAUSED, JobStatus.COMPLETED), JobStatus.COMPLETED_AND_PAUSED},
@@ -117,4 +122,8 @@ public class WatchChildrenJobsBLTest {
     public void whenCumulate2JobStatus_thenResultAsExpected(List<JobStatus> jobs, JobStatus expectedChildrenJobsStatus) {
         assertEquals(expectedChildrenJobsStatus, watchChildrenJobsBL.cumulateJobStatus(jobs.get(0), jobs.get(1)));
     }
+    private static JobStatus getErrorStatus() {
+        return featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE) ?
+            JobStatus.FAILED_AND_PAUSED : JobStatus.COMPLETED_WITH_ERRORS;
+    }
 }
diff --git a/vid-app-common/src/test/java/org/onap/vid/services/JobsBrokerServiceTest.java b/vid-app-common/src/test/java/org/onap/vid/services/JobsBrokerServiceTest.java
index 00b2b959b..df8f5034b 100644
--- a/vid-app-common/src/test/java/org/onap/vid/services/JobsBrokerServiceTest.java
+++ b/vid-app-common/src/test/java/org/onap/vid/services/JobsBrokerServiceTest.java
@@ -103,6 +103,7 @@ import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
+import org.togglz.core.manager.FeatureManager;
 
 @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, JobAdapterConfig.class})
 public class JobsBrokerServiceTest extends AbstractTestNGSpringContextTests {
@@ -124,6 +125,9 @@ public class JobsBrokerServiceTest extends AbstractTestNGSpringContextTests {
     private static final String DELETE_SERVICE_NOT_EXIST_EXCEPTION_MESSAGE = "Service does not exist";
     private JobsBrokerService broker;
 
+    @Mock
+    private FeatureManager featureManager;
+
     @Inject
     JobAdapter jobAdapter;
     @Inject
@@ -144,7 +148,7 @@ public class JobsBrokerServiceTest extends AbstractTestNGSpringContextTests {
     public void initializeBroker() {
         MockitoAnnotations.initMocks(this);
         when(versionService.retrieveBuildNumber()).thenReturn("aBuildNumber");
-        broker = new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, 200, 0, versionService);
+        broker = new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, 200, 0, versionService, featureManager);
         ((JobsBrokerServiceInDatabaseImpl) broker).deleteAll();
     }
 
@@ -607,7 +611,7 @@ public class JobsBrokerServiceTest extends AbstractTestNGSpringContextTests {
 
     @Test(dataProvider = "jobs")
     public void givenSomeJobs_pullNextJob_returnNextOrNothingAsExpected(List<Jobber> jobbers, int msoLimit, int expectedIndexSelected, Job.JobStatus topic, String assertionReason) {
-        JobsBrokerServiceInDatabaseImpl aBroker = new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, msoLimit, 20, versionService);
+        JobsBrokerServiceInDatabaseImpl aBroker = new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, msoLimit, 20, versionService, featureManager);
         final List<JobDaoImpl> jobs = addJobsWithModifiedDate(jobbers, aBroker);
         Optional<Job> nextJob = aBroker.pull(topic, UUID.randomUUID().toString());
         boolean shouldAnyBeSelected = expectedIndexSelected >= 0;