Pause Upon VF Module Failure

Issue-ID: VID-862

Currently in VID, once a failure is received from SO on the
instantiation of a VF Module, it is assumed that other VF Modules may
continue with the execution without stopping. However, in most of the
cases, it is a good idea to stop further requests to SO if one of the VF
Module instantiation returns an error. This user story, makes sure this
functionality is achieved.

Change-Id: I07c51b81111f805b61c8b3714a65171d7cbaa254
Signed-off-by: PATTANAYAK, SAUMYA SWARUP (sp931a) <sp931a@att.com>

5 files changed, 45 insertions, 10 deletions

diff --git a/vid-app-common/src/main/java/org/onap/vid/job/Job.java b/vid-app-common/src/main/java/org/onap/vid/job/Job.java
index 9861d0703..230e70fa6 100644
--- a/vid-app-common/src/main/java/org/onap/vid/job/Job.java
+++ b/vid-app-common/src/main/java/org/onap/vid/job/Job.java
@@ -68,6 +68,7 @@ public interface Job {
         CREATING(false),
         PENDING_RESOURCE(false),
         COMPLETED_AND_PAUSED(true, false),
+        FAILED_AND_PAUSED(true, true),
         ;
 
         private final Boolean finalStatus;
diff --git a/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt b/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt
index 502d8344d..09357e519 100644
--- a/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt
+++ b/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt
@@ -25,9 +25,11 @@ import org.onap.portalsdk.core.service.DataAccessService
 import org.onap.vid.job.Job
 import org.onap.vid.job.Job.JobStatus.*
 import org.onap.vid.job.impl.JobDaoImpl
+import org.onap.vid.properties.Features
 import org.onap.vid.utils.DaoUtils
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Service
+import org.togglz.core.manager.FeatureManager
 import java.util.*
 import java.util.stream.Collectors
 import java.util.stream.Stream
@@ -35,7 +37,7 @@ import java.util.stream.Stream
 
 @Service
 class WatchChildrenJobsBL @Autowired
-constructor(private val dataAccessService: DataAccessService) {
+constructor(private val dataAccessService: DataAccessService, private val featureManager: FeatureManager) {
 
     fun retrieveChildrenJobsStatus(childrenJobsIds: List<String>): Job.JobStatus {
         val jobs = getAllChildrenJobs(childrenJobsIds)
@@ -47,14 +49,16 @@ constructor(private val dataAccessService: DataAccessService) {
                     else -> it.status
                 }}
 
-        return cumulateJobStatus(jobsStatuses)
+        return if(featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE))
+            cumulateJobStatusWithPauseOnFailure(jobsStatuses) else cumulateJobStatus(jobsStatuses)
 
     }
 
     fun cumulateJobStatus(childrenComulatedStatus: Job.JobStatus, fatherJobStatus: Job.JobStatus): Job.JobStatus {
-        return cumulateJobStatus(Stream.of(childrenComulatedStatus, fatherJobStatus))
+        return if(featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE))
+            cumulateJobStatusWithPauseOnFailure(Stream.of(childrenComulatedStatus, fatherJobStatus))
+                else cumulateJobStatus(Stream.of(childrenComulatedStatus, fatherJobStatus))
     }
-
     private fun cumulateJobStatus(jobsStatuses: Stream<Job.JobStatus>): Job.JobStatus {
 
         return jobsStatuses.reduce{ a, b ->
@@ -70,7 +74,21 @@ constructor(private val dataAccessService: DataAccessService) {
             }
         } .orElse(COMPLETED_WITH_NO_ACTION)
   }
+    private fun cumulateJobStatusWithPauseOnFailure(jobsStatuses: Stream<Job.JobStatus>): Job.JobStatus {
 
+        return jobsStatuses.reduce{ a, b ->
+            when {
+                a == FAILED_AND_PAUSED || b == FAILED_AND_PAUSED-> FAILED_AND_PAUSED
+                a == COMPLETED && b.isFailure -> FAILED_AND_PAUSED
+                b == COMPLETED && a.isFailure -> FAILED_AND_PAUSED
+                !a.isFinal || !b.isFinal -> IN_PROGRESS
+                a == COMPLETED_AND_PAUSED || b == COMPLETED_AND_PAUSED -> COMPLETED_AND_PAUSED
+                a == COMPLETED || b == COMPLETED -> COMPLETED
+                a.isFailure || b.isFailure -> FAILED
+                else ->  COMPLETED_WITH_NO_ACTION
+            }
+        } .orElse(COMPLETED_WITH_NO_ACTION)
+    }
     private fun getAllChildrenJobs(childrenJobsIds: List<String>): Map<UUID, JobDaoImpl> {
         val jobs:MutableList<JobDaoImpl> = dataAccessService.getList(JobDaoImpl::class.java, filterByJobIds(childrenJobsIds), null, DaoUtils.getPropsMap()) as MutableList<JobDaoImpl>
         return jobs.stream().collect(Collectors.toMap( { it.uuid }, { it }))
diff --git a/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java b/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java
index 51803891a..7391f41a9 100644
--- a/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java
+++ b/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java
@@ -45,12 +45,14 @@ import org.onap.vid.exceptions.GenericUncheckedException;
 import org.onap.vid.exceptions.OperationNotAllowedException;
 import org.onap.vid.job.Job;
 import org.onap.vid.job.JobsBrokerService;
+import org.onap.vid.properties.Features;
 import org.onap.vid.properties.VidProperties;
 import org.onap.vid.services.VersionService;
 import org.onap.vid.utils.DaoUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import org.togglz.core.manager.FeatureManager;
 
 @Service
 public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService {
@@ -64,19 +66,21 @@ public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService {
     private int pollingIntervalSeconds;
 
     private final VersionService versionService;
-
+    private final FeatureManager featureManager;
     @Autowired
     public JobsBrokerServiceInDatabaseImpl(DataAccessService dataAccessService,
                                            SessionFactory sessionFactory,
                                            @Value("0") int maxOpenedInstantiationRequestsToMso,
                                            @Value("10") int pollingIntervalSeconds,
-                                           VersionService versionService) {
+                                           VersionService versionService,
+                                           FeatureManager featureManager) {
         // tha @Value will inject conservative defaults; overridden in @PostConstruct from configuration
         this.dataAccessService = dataAccessService;
         this.sessionFactory = sessionFactory;
         this.maxOpenedInstantiationRequestsToMso = maxOpenedInstantiationRequestsToMso;
         this.pollingIntervalSeconds = pollingIntervalSeconds;
         this.versionService = versionService;
+        this.featureManager = featureManager;
     }
 
     @PostConstruct
@@ -199,7 +203,7 @@ public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService {
                 "and TEMPLATE_Id not in \n" +
                 "(select TEMPLATE_Id from vid_job where" +
                 "   TEMPLATE_Id IS NOT NULL and("+
-                "   (JOB_STATUS='FAILED' and DELETED_AT is null)" + // failed but not deleted
+                "   (JOB_STATUS IN('FAILED','FAILED_AND_PAUSED') and DELETED_AT is null)" + // failed but not deleted
                 "   or JOB_STATUS='IN_PROGRESS'" +
                 "   or TAKEN_BY IS NOT NULL))" + " \n " +
                 // prefer older jobs, but the earlier in each bulk
@@ -232,13 +236,24 @@ public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService {
                 "    and in_progress_count is NULL \n" +
                 //and that have valid templateId
                 "    and JOB.template_id is not NULL \n"+
-                ")\n" +
+
+                filterFailedStatusForPendingResource()
+
+                + ")" +
                 //INDEX_IN_BULK is for order them inside same templateId,
                 //template_id - for order between different templateId (just to be deterministic)
                 "order by INDEX_IN_BULK,JOB.template_id \n" +
                 "limit 1;";
     }
-
+    private String filterFailedStatusForPendingResource() {
+        String sql = "and JOB.template_id not in \n" +
+                "(select TEMPLATE_Id from vid_job where" +
+                "   TEMPLATE_Id IS NOT NULL and (JOB_STATUS IN('FAILED','FAILED_AND_PAUSED') "
+                + " AND JOB_TYPE NOT IN('NetworkInstantiation','InstanceGroup','InstanceGroupMember') and DELETED_AT is null)" + // failed but not deleted
+                "   or TAKEN_BY IS NOT NULL)";
+        return featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE) ?
+                sql : "";
+    }
 
     @Override
     public void pushBack(Job job) {
diff --git a/vid-app-common/src/main/java/org/onap/vid/properties/Features.java b/vid-app-common/src/main/java/org/onap/vid/properties/Features.java
index f863b3025..9d3faa062 100644
--- a/vid-app-common/src/main/java/org/onap/vid/properties/Features.java
+++ b/vid-app-common/src/main/java/org/onap/vid/properties/Features.java
@@ -102,6 +102,7 @@ public enum Features implements Feature {
     FLAG_2008_CREATE_VFMODULE_INSTANTIATION_ORDER_NUMBER,
     FLAG_2008_PAUSE_INSTANTIATION_ON_VFMODULE_POPUP,
     FLAG_2008_REMOVE_PAUSE_INSTANTIATION,
+    FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE,
     ;
 
     public boolean isActive() {
diff --git a/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java b/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java
index 91801973b..bf0344c6b 100644
--- a/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java
+++ b/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java
@@ -436,7 +436,7 @@ public class AsyncInstantiationBusinessLogicImpl implements
 
     private boolean isRetryEnabledForStatus(JobStatus jobStatus) {
         return jobStatus==JobStatus.COMPLETED_AND_PAUSED || (featureManager.isActive(Features.FLAG_1902_RETRY_JOB) &&
-                (jobStatus==JobStatus.COMPLETED_WITH_ERRORS || jobStatus==JobStatus.FAILED));
+                (jobStatus==JobStatus.COMPLETED_WITH_ERRORS || jobStatus==JobStatus.FAILED || jobStatus==JobStatus.FAILED_AND_PAUSED));
     }
 
     private void setServiceInfoStatus(ServiceInfo serviceInfo, JobStatus jobStatus) {