From 62c72edd91b2d5485683274298b8249670663519 Mon Sep 17 00:00:00 2001 From: "PATTANAYAK, SAUMYA SWARUP (sp931a)" Date: Thu, 13 Aug 2020 12:35:01 -0400 Subject: Pause Upon VF Module Failure Issue-ID: VID-862 Currently in VID, once a failure is received from SO on the instantiation of a VF Module, it is assumed that other VF Modules may continue with the execution without stopping. However, in most of the cases, it is a good idea to stop further requests to SO if one of the VF Module instantiation returns an error. This user story, makes sure this functionality is achieved. Change-Id: I07c51b81111f805b61c8b3714a65171d7cbaa254 Signed-off-by: PATTANAYAK, SAUMYA SWARUP (sp931a) --- .../src/main/java/org/onap/vid/job/Job.java | 1 + .../onap/vid/job/command/WatchChildrenJobsBL.kt | 26 ++++++++++++++++++---- .../job/impl/JobsBrokerServiceInDatabaseImpl.java | 25 ++++++++++++++++----- .../java/org/onap/vid/properties/Features.java | 1 + .../AsyncInstantiationBusinessLogicImpl.java | 2 +- .../webapp/WEB-INF/conf/dev.features.properties | 1 + .../vid/config/JobCommandsConfigWithMockedMso.java | 4 ++-- .../vid/job/command/WatchChildrenJobsBLTest.java | 19 +++++++++++----- .../onap/vid/services/JobsBrokerServiceTest.java | 8 +++++-- 9 files changed, 68 insertions(+), 19 deletions(-) (limited to 'vid-app-common') diff --git a/vid-app-common/src/main/java/org/onap/vid/job/Job.java b/vid-app-common/src/main/java/org/onap/vid/job/Job.java index 9861d0703..230e70fa6 100644 --- a/vid-app-common/src/main/java/org/onap/vid/job/Job.java +++ b/vid-app-common/src/main/java/org/onap/vid/job/Job.java @@ -68,6 +68,7 @@ public interface Job { CREATING(false), PENDING_RESOURCE(false), COMPLETED_AND_PAUSED(true, false), + FAILED_AND_PAUSED(true, true), ; private final Boolean finalStatus; diff --git a/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt b/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt index 502d8344d..09357e519 100644 --- a/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt +++ b/vid-app-common/src/main/java/org/onap/vid/job/command/WatchChildrenJobsBL.kt @@ -25,9 +25,11 @@ import org.onap.portalsdk.core.service.DataAccessService import org.onap.vid.job.Job import org.onap.vid.job.Job.JobStatus.* import org.onap.vid.job.impl.JobDaoImpl +import org.onap.vid.properties.Features import org.onap.vid.utils.DaoUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import org.togglz.core.manager.FeatureManager import java.util.* import java.util.stream.Collectors import java.util.stream.Stream @@ -35,7 +37,7 @@ import java.util.stream.Stream @Service class WatchChildrenJobsBL @Autowired -constructor(private val dataAccessService: DataAccessService) { +constructor(private val dataAccessService: DataAccessService, private val featureManager: FeatureManager) { fun retrieveChildrenJobsStatus(childrenJobsIds: List): Job.JobStatus { val jobs = getAllChildrenJobs(childrenJobsIds) @@ -47,14 +49,16 @@ constructor(private val dataAccessService: DataAccessService) { else -> it.status }} - return cumulateJobStatus(jobsStatuses) + return if(featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE)) + cumulateJobStatusWithPauseOnFailure(jobsStatuses) else cumulateJobStatus(jobsStatuses) } fun cumulateJobStatus(childrenComulatedStatus: Job.JobStatus, fatherJobStatus: Job.JobStatus): Job.JobStatus { - return cumulateJobStatus(Stream.of(childrenComulatedStatus, fatherJobStatus)) + return if(featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE)) + cumulateJobStatusWithPauseOnFailure(Stream.of(childrenComulatedStatus, fatherJobStatus)) + else cumulateJobStatus(Stream.of(childrenComulatedStatus, fatherJobStatus)) } - private fun cumulateJobStatus(jobsStatuses: Stream): Job.JobStatus { return jobsStatuses.reduce{ a, b -> @@ -70,7 +74,21 @@ constructor(private val dataAccessService: DataAccessService) { } } .orElse(COMPLETED_WITH_NO_ACTION) } + private fun cumulateJobStatusWithPauseOnFailure(jobsStatuses: Stream): Job.JobStatus { + return jobsStatuses.reduce{ a, b -> + when { + a == FAILED_AND_PAUSED || b == FAILED_AND_PAUSED-> FAILED_AND_PAUSED + a == COMPLETED && b.isFailure -> FAILED_AND_PAUSED + b == COMPLETED && a.isFailure -> FAILED_AND_PAUSED + !a.isFinal || !b.isFinal -> IN_PROGRESS + a == COMPLETED_AND_PAUSED || b == COMPLETED_AND_PAUSED -> COMPLETED_AND_PAUSED + a == COMPLETED || b == COMPLETED -> COMPLETED + a.isFailure || b.isFailure -> FAILED + else -> COMPLETED_WITH_NO_ACTION + } + } .orElse(COMPLETED_WITH_NO_ACTION) + } private fun getAllChildrenJobs(childrenJobsIds: List): Map { val jobs:MutableList = dataAccessService.getList(JobDaoImpl::class.java, filterByJobIds(childrenJobsIds), null, DaoUtils.getPropsMap()) as MutableList return jobs.stream().collect(Collectors.toMap( { it.uuid }, { it })) diff --git a/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java b/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java index 51803891a..7391f41a9 100644 --- a/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java +++ b/vid-app-common/src/main/java/org/onap/vid/job/impl/JobsBrokerServiceInDatabaseImpl.java @@ -45,12 +45,14 @@ import org.onap.vid.exceptions.GenericUncheckedException; import org.onap.vid.exceptions.OperationNotAllowedException; import org.onap.vid.job.Job; import org.onap.vid.job.JobsBrokerService; +import org.onap.vid.properties.Features; import org.onap.vid.properties.VidProperties; import org.onap.vid.services.VersionService; import org.onap.vid.utils.DaoUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.togglz.core.manager.FeatureManager; @Service public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService { @@ -64,19 +66,21 @@ public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService { private int pollingIntervalSeconds; private final VersionService versionService; - + private final FeatureManager featureManager; @Autowired public JobsBrokerServiceInDatabaseImpl(DataAccessService dataAccessService, SessionFactory sessionFactory, @Value("0") int maxOpenedInstantiationRequestsToMso, @Value("10") int pollingIntervalSeconds, - VersionService versionService) { + VersionService versionService, + FeatureManager featureManager) { // tha @Value will inject conservative defaults; overridden in @PostConstruct from configuration this.dataAccessService = dataAccessService; this.sessionFactory = sessionFactory; this.maxOpenedInstantiationRequestsToMso = maxOpenedInstantiationRequestsToMso; this.pollingIntervalSeconds = pollingIntervalSeconds; this.versionService = versionService; + this.featureManager = featureManager; } @PostConstruct @@ -199,7 +203,7 @@ public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService { "and TEMPLATE_Id not in \n" + "(select TEMPLATE_Id from vid_job where" + " TEMPLATE_Id IS NOT NULL and("+ - " (JOB_STATUS='FAILED' and DELETED_AT is null)" + // failed but not deleted + " (JOB_STATUS IN('FAILED','FAILED_AND_PAUSED') and DELETED_AT is null)" + // failed but not deleted " or JOB_STATUS='IN_PROGRESS'" + " or TAKEN_BY IS NOT NULL))" + " \n " + // prefer older jobs, but the earlier in each bulk @@ -232,13 +236,24 @@ public class JobsBrokerServiceInDatabaseImpl implements JobsBrokerService { " and in_progress_count is NULL \n" + //and that have valid templateId " and JOB.template_id is not NULL \n"+ - ")\n" + + + filterFailedStatusForPendingResource() + + + ")" + //INDEX_IN_BULK is for order them inside same templateId, //template_id - for order between different templateId (just to be deterministic) "order by INDEX_IN_BULK,JOB.template_id \n" + "limit 1;"; } - + private String filterFailedStatusForPendingResource() { + String sql = "and JOB.template_id not in \n" + + "(select TEMPLATE_Id from vid_job where" + + " TEMPLATE_Id IS NOT NULL and (JOB_STATUS IN('FAILED','FAILED_AND_PAUSED') " + + " AND JOB_TYPE NOT IN('NetworkInstantiation','InstanceGroup','InstanceGroupMember') and DELETED_AT is null)" + // failed but not deleted + " or TAKEN_BY IS NOT NULL)"; + return featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE) ? + sql : ""; + } @Override public void pushBack(Job job) { diff --git a/vid-app-common/src/main/java/org/onap/vid/properties/Features.java b/vid-app-common/src/main/java/org/onap/vid/properties/Features.java index f863b3025..9d3faa062 100644 --- a/vid-app-common/src/main/java/org/onap/vid/properties/Features.java +++ b/vid-app-common/src/main/java/org/onap/vid/properties/Features.java @@ -102,6 +102,7 @@ public enum Features implements Feature { FLAG_2008_CREATE_VFMODULE_INSTANTIATION_ORDER_NUMBER, FLAG_2008_PAUSE_INSTANTIATION_ON_VFMODULE_POPUP, FLAG_2008_REMOVE_PAUSE_INSTANTIATION, + FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE, ; public boolean isActive() { diff --git a/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java b/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java index 91801973b..bf0344c6b 100644 --- a/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java +++ b/vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java @@ -436,7 +436,7 @@ public class AsyncInstantiationBusinessLogicImpl implements private boolean isRetryEnabledForStatus(JobStatus jobStatus) { return jobStatus==JobStatus.COMPLETED_AND_PAUSED || (featureManager.isActive(Features.FLAG_1902_RETRY_JOB) && - (jobStatus==JobStatus.COMPLETED_WITH_ERRORS || jobStatus==JobStatus.FAILED)); + (jobStatus==JobStatus.COMPLETED_WITH_ERRORS || jobStatus==JobStatus.FAILED || jobStatus==JobStatus.FAILED_AND_PAUSED)); } private void setServiceInfoStatus(ServiceInfo serviceInfo, JobStatus jobStatus) { diff --git a/vid-app-common/src/main/webapp/WEB-INF/conf/dev.features.properties b/vid-app-common/src/main/webapp/WEB-INF/conf/dev.features.properties index 92be1c74b..2c73281b2 100644 --- a/vid-app-common/src/main/webapp/WEB-INF/conf/dev.features.properties +++ b/vid-app-common/src/main/webapp/WEB-INF/conf/dev.features.properties @@ -34,3 +34,4 @@ FLAG_SHOW_ORCHESTRATION_TYPE = false FLAG_1911_INSTANTIATION_ORDER_BUTTON_IN_ASYNC_ALACARTE = false FLAG_2002_ANY_ALACARTE_BESIDES_EXCLUDED_NEW_INSTANTIATION_UI = false FLAG_GUILIN_CHANGEMG_SUBMIT_TO_SO = true +FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE = false \ No newline at end of file diff --git a/vid-app-common/src/test/java/org/onap/vid/config/JobCommandsConfigWithMockedMso.java b/vid-app-common/src/test/java/org/onap/vid/config/JobCommandsConfigWithMockedMso.java index 468fb41c3..ecec705a1 100644 --- a/vid-app-common/src/test/java/org/onap/vid/config/JobCommandsConfigWithMockedMso.java +++ b/vid-app-common/src/test/java/org/onap/vid/config/JobCommandsConfigWithMockedMso.java @@ -79,7 +79,7 @@ public class JobCommandsConfigWithMockedMso { @Bean public JobsBrokerService jobsBrokerService(DataAccessService dataAccessService, SessionFactory sessionFactory, VersionService versionService) { - return new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, 200, 0,versionService); + return new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, 200, 0,versionService,featureManager()); } @Bean @@ -314,7 +314,7 @@ public class JobCommandsConfigWithMockedMso { @Bean public WatchChildrenJobsBL watchChildrenJobsService(DataAccessService dataAccessService) { - return new WatchChildrenJobsBL(dataAccessService); + return new WatchChildrenJobsBL(dataAccessService,featureManager()); } } diff --git a/vid-app-common/src/test/java/org/onap/vid/job/command/WatchChildrenJobsBLTest.java b/vid-app-common/src/test/java/org/onap/vid/job/command/WatchChildrenJobsBLTest.java index fad32bba6..a8613f7a7 100644 --- a/vid-app-common/src/test/java/org/onap/vid/job/command/WatchChildrenJobsBLTest.java +++ b/vid-app-common/src/test/java/org/onap/vid/job/command/WatchChildrenJobsBLTest.java @@ -37,10 +37,12 @@ import org.mockito.MockitoAnnotations; import org.onap.portalsdk.core.service.DataAccessService; import org.onap.vid.job.Job.JobStatus; import org.onap.vid.job.impl.JobDaoImpl; +import org.onap.vid.properties.Features; import org.onap.vid.utils.DaoUtils; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import org.togglz.core.manager.FeatureManager; public class WatchChildrenJobsBLTest { @Mock @@ -49,6 +51,9 @@ public class WatchChildrenJobsBLTest { @InjectMocks private WatchChildrenJobsBL watchChildrenJobsBL; + @Mock + private static FeatureManager featureManager; + @BeforeClass public void initMocks() { MockitoAnnotations.initMocks(this); @@ -80,16 +85,16 @@ public class WatchChildrenJobsBLTest { {Arrays.asList(JobStatus.COMPLETED, JobStatus.COMPLETED), JobStatus.COMPLETED}, {Arrays.asList(JobStatus.COMPLETED, JobStatus.COMPLETED_WITH_NO_ACTION), JobStatus.COMPLETED}, {Arrays.asList(JobStatus.FAILED, JobStatus.COMPLETED_WITH_NO_ACTION), JobStatus.FAILED}, - {Arrays.asList(JobStatus.FAILED, JobStatus.COMPLETED), JobStatus.COMPLETED_WITH_ERRORS}, + {Arrays.asList(JobStatus.FAILED, JobStatus.COMPLETED), getErrorStatus()}, {Arrays.asList(JobStatus.RESOURCE_IN_PROGRESS, JobStatus.FAILED), JobStatus.IN_PROGRESS}, {Arrays.asList(JobStatus.PAUSE, JobStatus.FAILED), JobStatus.IN_PROGRESS}, {Arrays.asList(JobStatus.PENDING, JobStatus.FAILED), JobStatus.IN_PROGRESS}, {Arrays.asList(JobStatus.IN_PROGRESS, JobStatus.COMPLETED), JobStatus.IN_PROGRESS}, {Arrays.asList(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS), JobStatus.IN_PROGRESS}, - {Arrays.asList(JobStatus.COMPLETED, JobStatus.COMPLETED_WITH_ERRORS), JobStatus.COMPLETED_WITH_ERRORS}, - {Arrays.asList(JobStatus.COMPLETED_WITH_ERRORS, JobStatus.FAILED), JobStatus.COMPLETED_WITH_ERRORS}, - {Arrays.asList(JobStatus.COMPLETED_WITH_ERRORS, JobStatus.COMPLETED_WITH_ERRORS), JobStatus.COMPLETED_WITH_ERRORS}, - {Arrays.asList(JobStatus.COMPLETED_WITH_ERRORS, JobStatus.COMPLETED_WITH_NO_ACTION), JobStatus.COMPLETED_WITH_ERRORS}, + {Arrays.asList(JobStatus.COMPLETED, getErrorStatus()), getErrorStatus()}, + {Arrays.asList(getErrorStatus(), JobStatus.FAILED), getErrorStatus()}, + {Arrays.asList(getErrorStatus(), getErrorStatus()), getErrorStatus()}, + {Arrays.asList(getErrorStatus(), JobStatus.COMPLETED_WITH_NO_ACTION), getErrorStatus()}, {Arrays.asList(JobStatus.COMPLETED_WITH_NO_ACTION, JobStatus.COMPLETED_WITH_NO_ACTION), JobStatus.COMPLETED_WITH_NO_ACTION}, {Arrays.asList(JobStatus.COMPLETED_AND_PAUSED, JobStatus.RESOURCE_IN_PROGRESS), JobStatus.IN_PROGRESS}, {Arrays.asList(JobStatus.COMPLETED_AND_PAUSED, JobStatus.COMPLETED), JobStatus.COMPLETED_AND_PAUSED}, @@ -117,4 +122,8 @@ public class WatchChildrenJobsBLTest { public void whenCumulate2JobStatus_thenResultAsExpected(List jobs, JobStatus expectedChildrenJobsStatus) { assertEquals(expectedChildrenJobsStatus, watchChildrenJobsBL.cumulateJobStatus(jobs.get(0), jobs.get(1))); } + private static JobStatus getErrorStatus() { + return featureManager.isActive(Features.FLAG_2008_PAUSE_VFMODULE_INSTANTIATION_FAILURE) ? + JobStatus.FAILED_AND_PAUSED : JobStatus.COMPLETED_WITH_ERRORS; + } } diff --git a/vid-app-common/src/test/java/org/onap/vid/services/JobsBrokerServiceTest.java b/vid-app-common/src/test/java/org/onap/vid/services/JobsBrokerServiceTest.java index 00b2b959b..df8f5034b 100644 --- a/vid-app-common/src/test/java/org/onap/vid/services/JobsBrokerServiceTest.java +++ b/vid-app-common/src/test/java/org/onap/vid/services/JobsBrokerServiceTest.java @@ -103,6 +103,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import org.togglz.core.manager.FeatureManager; @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, JobAdapterConfig.class}) public class JobsBrokerServiceTest extends AbstractTestNGSpringContextTests { @@ -124,6 +125,9 @@ public class JobsBrokerServiceTest extends AbstractTestNGSpringContextTests { private static final String DELETE_SERVICE_NOT_EXIST_EXCEPTION_MESSAGE = "Service does not exist"; private JobsBrokerService broker; + @Mock + private FeatureManager featureManager; + @Inject JobAdapter jobAdapter; @Inject @@ -144,7 +148,7 @@ public class JobsBrokerServiceTest extends AbstractTestNGSpringContextTests { public void initializeBroker() { MockitoAnnotations.initMocks(this); when(versionService.retrieveBuildNumber()).thenReturn("aBuildNumber"); - broker = new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, 200, 0, versionService); + broker = new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, 200, 0, versionService, featureManager); ((JobsBrokerServiceInDatabaseImpl) broker).deleteAll(); } @@ -607,7 +611,7 @@ public class JobsBrokerServiceTest extends AbstractTestNGSpringContextTests { @Test(dataProvider = "jobs") public void givenSomeJobs_pullNextJob_returnNextOrNothingAsExpected(List jobbers, int msoLimit, int expectedIndexSelected, Job.JobStatus topic, String assertionReason) { - JobsBrokerServiceInDatabaseImpl aBroker = new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, msoLimit, 20, versionService); + JobsBrokerServiceInDatabaseImpl aBroker = new JobsBrokerServiceInDatabaseImpl(dataAccessService, sessionFactory, msoLimit, 20, versionService, featureManager); final List jobs = addJobsWithModifiedDate(jobbers, aBroker); Optional nextJob = aBroker.pull(topic, UUID.randomUUID().toString()); boolean shouldAnyBeSelected = expectedIndexSelected >= 0; -- cgit 1.2.3-korg