From 6ad41e3ccd398a2721f41ad61c80b7bb03f7d127 Mon Sep 17 00:00:00 2001 From: Ittay Stern Date: Mon, 31 Dec 2018 17:21:27 +0200 Subject: Merge from ECOMP's repository Main Features -------------- - Async-Instantiation jobs mechanism major update; still WIP (package `org.onap.vid.job`) - New features in View/Edit: Activate fabric configuration; show related networks; soft delete - Support AAI service-tree traversal (`AAIServiceTree`) - In-memory cache for SDC models and certain A&AI queries (`CacheProviderWithLoadingCache`) - Upgrade TOSCA Parser and add parsing options; fix malformed TOSCA models - Resolve Cloud-Owner values for MSO - Pass X-ONAP headers to MSO Infrastructure -------------- - Remove codehaus' jackson mapper; use soley fasterxml 2.9.7 - Surefire invokes both TestNG and JUnit tests - Support Kotlin source files - AaiController2 which handles errors in a "Spring manner" - Inline generated-sources and remove jsonschema2pojo Quality -------- - Cumulative bug fixes (A&AI API, UI timeouts, and many more) - Many Sonar issues cleaned-up - Some unused classes removed - Minor changes in vid-automation project, allowing some API verification to run Hard Merges ------------ - HTTP Clients (MSO, A&AI, WebConfig, OutgoingRequestHeadersTest) - Moved `package org.onap.vid.controllers` to `controller`, without plural -- just to keep semantic sync with ECOMP. Reference commit in ECOMP: 3d1141625 Issue-ID: VID-378 Change-Id: I9c8d1e74caa41815891d441fc0760bb5f29c5788 Signed-off-by: Ittay Stern --- .../impl/AsyncInstantiationIntegrationTest.java | 925 +++++++++++++++++++++ 1 file changed, 925 insertions(+) create mode 100644 vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java (limited to 'vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java') diff --git a/vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java b/vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java new file mode 100644 index 000000000..f7f83bdfe --- /dev/null +++ b/vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java @@ -0,0 +1,925 @@ +package org.onap.vid.job.impl; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang3.StringUtils; +import org.mockito.Mockito; +import org.onap.portalsdk.core.service.DataAccessService; +import org.onap.portalsdk.core.util.SystemProperties; +import org.onap.vid.asdc.AsdcCatalogException; +import org.onap.vid.config.DataSourceConfig; +import org.onap.vid.config.JobCommandsConfigWithMockedMso; +import org.onap.vid.config.MockedAaiClientAndFeatureManagerConfig; +import org.onap.vid.job.Job; +import org.onap.vid.job.Job.JobStatus; +import org.onap.vid.job.JobType; +import org.onap.vid.job.JobsBrokerService; +import org.onap.vid.job.command.CommandUtils; +import org.onap.vid.job.command.InternalState; +import org.onap.vid.model.Action; +import org.onap.vid.model.NameCounter; +import org.onap.vid.model.RequestReferencesContainer; +import org.onap.vid.model.ServiceInfo; +import org.onap.vid.model.serviceInstantiation.ServiceInstantiation; +import org.onap.vid.mso.RestMsoImplementation; +import org.onap.vid.mso.RestObject; +import org.onap.vid.mso.model.RequestReferences; +import org.onap.vid.mso.rest.AsyncRequestStatus; +import org.onap.vid.properties.Features; +import org.onap.vid.services.AsyncInstantiationBaseTest; +import org.onap.vid.services.AsyncInstantiationBusinessLogic; +import org.onap.vid.utils.DaoUtils; +import org.springframework.test.context.ContextConfiguration; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.togglz.core.manager.FeatureManager; + +import javax.inject.Inject; +import javax.ws.rs.ProcessingException; +import java.lang.reflect.Method; +import java.util.*; +import java.util.function.BiConsumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.*; +import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartEquals; +import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasProperty; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.core.Every.everyItem; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.endsWith; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; +import static org.onap.vid.job.Job.JobStatus.*; +import static org.onap.vid.model.JobAuditStatus.SourceStatus.MSO; +import static org.onap.vid.model.JobAuditStatus.SourceStatus.VID; +import static org.testng.AssertJUnit.*; + +//it's more like integration test than UT +//But it's very hard to test in API test so I use UT +@ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, MockedAaiClientAndFeatureManagerConfig.class, JobCommandsConfigWithMockedMso.class}) +public class AsyncInstantiationIntegrationTest extends AsyncInstantiationBaseTest { + + private static final String FAILED_STR = "FAILED"; + private static final String COMPLETE_STR = "COMPLETE"; + private static final String IN_PROGRESS_STR = "IN_PROGRESS"; + private static final String REQUESTED = "REQUESTED"; + private static final String PENDING_MANUAL_TASK = "PENDING_MANUAL_TASK"; + public static final String RAW_DATA_FROM_MSO = "RAW DATA FROM MSO"; + private static String USER_ID = "123"; + public static String REQUEST_ID = UUID.randomUUID().toString(); + public static String SERVICE_INSTANCE_ID = UUID.randomUUID().toString(); + + @Inject + private JobsBrokerService jobsBrokerService; + + @Inject + private JobWorker jobWorker; + + @Inject + private FeatureManager featureManager; + + @Inject + private AsyncInstantiationBusinessLogic asyncInstantiationBL; + + @Inject + private RestMsoImplementation restMso; + + @Inject + private DataAccessService dataAccessService; + + @Inject + private CommandUtils commandUtils; + + @BeforeClass + void initServicesInfoService() { + createInstanceParamsMaps(); + } + + @BeforeMethod + void clearDb() { + dataAccessService.deleteDomainObjects(ServiceInfo.class, "1=1", DaoUtils.getPropsMap()); + dataAccessService.deleteDomainObjects(JobDaoImpl.class, "1=1", DaoUtils.getPropsMap()); + dataAccessService.deleteDomainObjects(NameCounter.class, "1=1", DaoUtils.getPropsMap()); + } + + @BeforeMethod + void defineMocks() { + mockAaiClientAnyNameFree(); + } + + //@Test + public void whenPushNewBulk_thenAllServicesAreInPending() { + + pushMacroBulk(); + List serviceInfoList = asyncInstantiationBL.getAllServicesInfo(); + assertThat( serviceInfoList, everyItem(hasProperty("jobStatus", is(PENDING)))); + } + + private List pushMacroBulk() { + ServiceInstantiation serviceInstantiation = generateMockMacroServiceInstantiationPayload(false, + createVnfList(instanceParamsMapWithoutParams, Collections.EMPTY_LIST, true), + 3, true,PROJECT_NAME, true); + return asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID); + } + + private UUID pushALaCarteWithVnf() { + ServiceInstantiation serviceInstantiation = generateALaCarteWithVnfsServiceInstantiationPayload(); + List uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID); + assertThat(uuids, hasSize(1)); + return uuids.get(0); + } + + private UUID pushALaCarteUpdateWithGroups() { + ServiceInstantiation serviceInstantiation = generateALaCarteUpdateWith1ExistingGroup2NewGroupsPayload(); + List uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID); + assertThat(uuids, hasSize(1)); + return uuids.get(0); + } + + public static RestObject createResponse(int statusCode) { + return createResponse(statusCode, SERVICE_INSTANCE_ID, REQUEST_ID); + } + + public static RestObject createResponse(int statusCode, String instanceId, String requestId) { + RequestReferences requestReferences = new RequestReferences(); + requestReferences.setRequestId(requestId); + requestReferences.setInstanceId(instanceId); + RestObject restObject = new RestObject<>(); + restObject.set(new RequestReferencesContainer(requestReferences)); + restObject.setStatusCode(statusCode); + restObject.setRaw(RAW_DATA_FROM_MSO); + return restObject; + } + + ImmutableList statusesToStrings(JobStatus... jobStatuses) { + return Stream.of(jobStatuses).map( + Enum::toString).collect(ImmutableList.toImmutableList()); + } + + /* + Make sure service state is in progress once request has sent to MSO + Make sure service state is in progress once request has sent to MSO and MSO status is in_progress + Make sure service state is Failed once we got from MSO failure state, and that job's are not collected any more. + Make sure service state is Completed successfully once we got from MSO complete, and that next job is peeked. + Once a service in the bulk is failed, other services moved to Stopped, and no other jobs from the bulk are peeked. + */ + //@Test + public void testStatusesOfMacroServiceInBulkDuringBulkLifeCycle() { + when(restMso.PostForObject(any(), any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(200)); + ImmutableList> expectedStatusesForVid = ImmutableList.of( + statusesToStrings(PENDING, IN_PROGRESS, COMPLETED), + statusesToStrings(PENDING, IN_PROGRESS, FAILED), + statusesToStrings(PENDING, STOPPED) + ); + + ImmutableList> expectedStatusesForMso = ImmutableList.of( + ImmutableList.of(REQUESTED, IN_PROGRESS_STR, "not a state", FAILED_STR ,COMPLETE_STR), + ImmutableList.of(REQUESTED, FAILED_STR), + ImmutableList.of() + ); + + List uuids = pushMacroBulk(); + pullPendingJobAndAssertJobStatus(JobStatus.IN_PROGRESS, PENDING); + + //assert that when get ProcessingException from restMso, status remain the same + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenThrow(new ProcessingException("fake message")); + Job job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS); + UUID firstHandledJobUUID = job.getUuid(); + listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job); + + //assert that when get IN_PROGRESS status from restMso, status remain IN_PROGRESS + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR)); + job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS); + listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job); + + //assert that when get unrecognized status from restMso, status remain IN_PROGRESS + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject("not a state")); + job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS); + listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job); + + //assert that when get non 200 status code during IN_PROGRESS, status remain IN_PROGRESS + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR, 404)); + job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS); + listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job); + + //when get job COMPLETE from MSO, service status become COMPLETED + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED); + List serviceInfoList = listServicesAndAssertStatus(COMPLETED, PENDING, job); + + + //for use later in the test + Map expectedJobStatusMap = serviceInfoList.stream().collect( + Collectors.toMap(ServiceInfo::getJobId, x-> PENDING)); + expectedJobStatusMap.put(job.getUuid(), COMPLETED); + + //when handling another PENDING job, statuses are : COMPLETED, IN_PROGRESS, PENDING + job = pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS); + assertThat(job.getUuid(), not(equalTo(firstHandledJobUUID))); //assert different job was handled now + expectedJobStatusMap.put(job.getUuid(), JobStatus.IN_PROGRESS); + listServicesAndAssertStatus(expectedJobStatusMap); + + //when get FAILED status from MSO statuses are : COMPLETED, FAILED, STOPPED + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR)); + job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED); + expectedJobStatusMap.put(job.getUuid(), JobStatus.FAILED); + expectedJobStatusMap = expectedJobStatusMap.entrySet().stream().collect(Collectors.toMap( + e -> e.getKey(), e -> e.getValue() == PENDING ? JobStatus.STOPPED : e.getValue() + )); + + listServicesAndAssertStatus(expectedJobStatusMap); + IntStream.range(0, uuids.size()).forEach(i -> { + UUID uuid = uuids.get(i); + List msoStatuses = asyncInstantiationBL.getAuditStatuses(uuid, MSO).stream().map(x -> x.getJobStatus()).collect(Collectors.toList()); + List vidStatuses = asyncInstantiationBL.getAuditStatuses(uuid, VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList()); + assertThat(msoStatuses, is(expectedStatusesForMso.get(i))); + assertThat(vidStatuses, is(expectedStatusesForVid.get(i))); + }); + // + assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent()); + assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent()); + } + + + @DataProvider + public static Object[][] AlaCarteStatuses(Method test) { + return new Object[][]{ + {COMPLETE_STR, JobStatus.COMPLETED, JobStatus.COMPLETED}, + {FAILED_STR, JobStatus.COMPLETED_WITH_ERRORS, JobStatus.FAILED}, + }; + } + + /* + Make sure service state is in progress once request has sent to MSO + Make sure service state is watching until state changes to complemented + Make sure service state is watching until vnf state changes to completed + Make sure service state is Completed successfully once we got from MSO complete for the vnf job. + status Creating + */ + //@Test(dataProvider = "AlaCarteStatuses") + public void testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules(String msoVnfStatus, JobStatus expectedServiceStatus, JobStatus expectedVnfStatus) { + /* + [v] + push alacarte with 1 vnf + [v] verify STATUS pending + [v] + pull+execute (should post to MSO) + [v] verify STATUS in progress + [v] + pull+execute (should GET completed from MSO) + [v] verify STATUS in progress; TYPE watching + [v] verify job#2 *new* VNF job STATUS creating + [v] + pull+execute job#2 (should post to MSO) + [v] verify job#2 STATUS resource in progress + [v] verify job#1 STATUS in progress + [v] + pull+execute job#2 (should GET completed from MSO) + [v] verify job#2 STATUS completed + [v] + pull+execute job#1 + [v] verify job#1 STATUS completed + + * not looking on audit (yet) + */ + when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true); + when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(false); + final String SERVICE_REQUEST_ID = UUID.randomUUID().toString(); + final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString(); + final String VNF_REQUEST_ID = UUID.randomUUID().toString(); + + + //push alacarte with 1 vnf, verify STATUS pending + UUID uuid = pushALaCarteWithVnf(); + singleServicesAndAssertStatus(JobStatus.PENDING, uuid); + + //mock mso to answer 200 of create service instance request, verify STATUS in progress + when(restMso.PostForObject(any(), endsWith("serviceInstances"), eq(RequestReferencesContainer.class))).thenReturn( + createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID)); + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.PENDING, JobStatus.IN_PROGRESS, JobType.InProgressStatus); + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + + //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching + reset(restMso); + when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, JobType.Watching); + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + + //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress + reset(restMso); + when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), eq(RequestReferencesContainer.class))).thenReturn( + createResponse(200, UUID.randomUUID().toString(), VNF_REQUEST_ID)); + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.VnfInProgressStatus); + + //verify service job STATUS in progress + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, JobType.Watching); + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + + //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status, + //job status shall be final (COMPLETE/COMPLETE_WITH_ERRORS) + reset(restMso); + when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn( + asyncRequestStatusResponseAsRestObject(msoVnfStatus)); + pullJobProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, expectedVnfStatus, false); + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, expectedServiceStatus, true); + singleServicesAndAssertStatus(expectedServiceStatus, uuid); + + } + + /* + this test is almost duplication of testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules. + + IgnoringVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is off + WithVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is on + + We shall consider later to remove testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules + And union these tests to single one. + */ + + //@Test + public void testALaCarteLifeCycle1Vnf2VfModules() { + + + String msoVnfStatus = COMPLETE_STR; + JobStatus expectedServiceStatus = IN_PROGRESS; + JobStatus expectedVnfStatus = RESOURCE_IN_PROGRESS; + when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true); + when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true); + final String SERVICE_REQUEST_ID = UUID.randomUUID().toString(); + final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString(); + final String VNF_REQUEST_ID = UUID.randomUUID().toString(); + final String VNF_INSTANCE_ID = UUID.randomUUID().toString(); + final String VG_REQUEST_ID = UUID.randomUUID().toString(); + final String VG_INSTANCE_ID = UUID.randomUUID().toString(); + final String VF_MODULE_REQUEST_ID = UUID.randomUUID().toString(); + final String VF_MODULE_REQUEST_ID2 = UUID.randomUUID().toString(); + + + //push alacarte with 1 vnf, verify STATUS pending + UUID uuid = pushALaCarteWithVnf(); + singleServicesAndAssertStatus(JobStatus.PENDING, uuid); + + /*---------- service -----------*/ + + //mock mso to answer 200 of create service instance request, verify STATUS in progress + when(restMso.PostForObject(any(), endsWith("serviceInstances"), eq(RequestReferencesContainer.class))).thenReturn( + createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID)); + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.PENDING, JobStatus.IN_PROGRESS, JobType.InProgressStatus); + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + + //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching + reset(restMso); + when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, JobType.Watching); + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + + /*---------- vnf -----------*/ + + //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress + reset(restMso); + when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), eq(RequestReferencesContainer.class))).thenReturn( + createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID)); + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.VnfInProgressStatus); + + //verify service job STATUS in progress + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, JobType.Watching); + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + + //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status, + //job status shall be final (COMPLETE/COMPLETE_WITH_ERRORS) + reset(restMso); + when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + + try { + reset(commandUtils); + when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_0_MODEL_VERSION_ID)).thenReturn(true); + when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_1_MODEL_VERSION_ID)).thenReturn(false); + } catch (AsdcCatalogException e) { + + } + + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.RESOURCE_IN_PROGRESS, JobStatus.RESOURCE_IN_PROGRESS, JobType.WatchingBaseModule); + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, true); + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + + /*---------- vf Module without volume group name (base) -----------*/ + + //vg name not exist, so vf module created immediately + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching); + + //verify vnf/volumeGroup job STATUS still watching with resource in progress + pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching); + + //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress + reset(restMso); + when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), eq(RequestReferencesContainer.class))).thenReturn( + createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID)); + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus); + + //mock mso to answer for vf module orchestration request + reset(restMso); + when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn( + asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus, JobStatus.COMPLETED, JobType.ResourceInProgressStatus); + + //verify volume group become completed + pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching, JobStatus.COMPLETED, JobType.Watching); + + //vnf become watching after volume group completed, and new volume group created + pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.WatchingBaseModule, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching); + + /*---------- volume group & vf module (non base) -----------*/ + + /*---------- volume group -----------*/ + + //mock mso to answer 200 of create volumeGroup instance request, pull+execute volumeGroup job, STATUS resource in progress + reset(restMso); + when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), eq(RequestReferencesContainer.class))).thenReturn( + createResponse(200, VG_INSTANCE_ID, VG_REQUEST_ID)); + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.VolumeGroupInProgressStatus); + + //verify vnf job STATUS still watching with resource in progress + pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching); + + //mock mso to answer for volume group orchestration request + reset(restMso); + when(restMso.GetForObject(endsWith(VG_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn( + asyncRequestStatusResponseAsRestObject(msoVnfStatus)); + pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.VolumeGroupInProgressStatus, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching); + + /*---------- vfModule -----------*/ + + //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress + reset(restMso); + when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), eq(RequestReferencesContainer.class))).thenReturn( + createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID2)); + pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus); + + //mock mso to answer for vf module orchestration request + reset(restMso); + when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID2), eq(AsyncRequestStatus.class))).thenReturn( + asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus, JobStatus.COMPLETED, JobType.ResourceInProgressStatus); + + //execute twice - 1 for parent volume group, 1 for parent vnf + pullAllJobProcessAndPushBackByType(JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching , JobStatus.COMPLETED); + + singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid); + pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.COMPLETED, true); + singleServicesAndAssertStatus(JobStatus.COMPLETED, uuid); + } + + //@Test + public void testBadAaiResponseForSearchNamesAndBackToNormal() { + when(aaiClient.isNodeTypeExistsByName(any(), any())).thenThrow(aaiNodeQueryBadResponseException()); + pushMacroBulk(); //JOB shall become IN_PROGRESS but service info is still pending + Job job = pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS, true); + listServicesAndAssertStatus(PENDING, PENDING, job); + + //JOB shall remain in IN_PROGRESS + job = pullJobProcessAndPushBack( JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, true); + //make sure the job command is still ServiceInstantiation + assertThat(job.getType(), is(JobType.MacroServiceInstantiation)); + listServicesAndAssertStatus(PENDING, PENDING, job); + + //simulate AAI back to normal, AAI return name is free, and MSO return good response + Mockito.reset(aaiClient); // must forget the "thenThrow" + when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(false); + when(restMso.PostForObject(any(),any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(200)); + job = pullJobProcessAndPushBack( JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, true); + listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job); + + //when get job COMPLETE from MSO, service status become COMPLETED + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED); + listServicesAndAssertStatus(COMPLETED, PENDING, job); + } + + //@Test + public void testAaiResponseNameUsedTillMaxRetries() { + when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(true); + asyncInstantiationBL.setMaxRetriesGettingFreeNameFromAai(10); + pushMacroBulk(); + //JOB shall become IN_PROGRESS but service info is still pending + Job job = pullJobProcessAndPushBack(PENDING, JobStatus.FAILED, true); + listServicesAndAssertStatus(JobStatus.FAILED, JobStatus.STOPPED, job); + } + + private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus) { + return pullJobProcessAndPushBack(topic, expectedNextJobStatus, true); + } + + //return the pulled job (and not the pushed job) + private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus, boolean pullingAssertion) { + Optional job = pullJob(topic, pullingAssertion); + + Job nextJob = jobWorker.executeJobAndGetNext(job.get()); + + try { + assertThat("next job not ok: " + nextJob.getData(), nextJob.getStatus(), is(expectedNextJobStatus)); + + if (pullingAssertion) { + //assert another pulling on same topic return no result (before push back) + assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent()); + } + + } finally { + jobsBrokerService.pushBack(nextJob); // push back to let retries - even if any assertion failure + } + assertThat(jobsBrokerService.peek(job.get().getUuid()).getStatus(), is(expectedNextJobStatus)); + return job.get(); + } + + private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus, + JobType expectedNextJobType) { + Job job = pullJobProcessAndPushBack(topic, expectedNextJobStatus, false); + assertThat("job not ok: " + job.getData(), job.getType(), is(expectedNextJobType)); + return job; + } + + private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus, + JobType expectedNextJobType, int retries) { + return retryWithAssertionsLimit(retries, () -> { + return pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType); + }); + } + + private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus, + JobType expectedNextJobType, Action actionPhase, InternalState internalState, int retries) { + return retryWithAssertionsLimit(retries, () -> { + Job job = pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType); + assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("actionPhase", actionPhase.name()))); + if (internalState != null) { + assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("internalState", internalState.name()))); + } + return job; + }); + } + + private Job retryWithAssertionsLimit(int retries, Supplier supplier) { + java.util.Stack history = new Stack<>(); + + do { + try { + return supplier.get(); + } catch (AssertionError assertionError) { + history.push(assertionError); + } + } while (history.size() < retries); + + // No success: + throw new AssertionError("No luck while all of these assertion errors: " + history.stream() + .map(Throwable::getMessage) + .map(s -> s.replace('\n', ' ')) + .map(s -> s.replaceAll("\\s{2,}"," ")) + .distinct() + .collect(joining("\n ", "\n ", "")), history.peek()); + } + + private Job pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus topic, JobType expectedCurrentJobType, JobStatus expectedNextJobStatus, + JobType expectedNextJobType) { + List pulledJobs = new ArrayList<>(); + Job lastJob = null; + while (lastJob == null || lastJob.getType() != expectedCurrentJobType) { + lastJob = pullJob(topic, false).get(); + if (lastJob.getType() != expectedCurrentJobType) { + pulledJobs.add(lastJob); + } + } + + Job nextJob = jobWorker.executeJobAndGetNext(lastJob); + assertThat(nextJob.getStatus(), is(expectedNextJobStatus)); + assertThat(nextJob.getType(), is(expectedNextJobType)); + + jobsBrokerService.pushBack(nextJob); + assertThat(jobsBrokerService.peek(nextJob.getUuid()).getStatus(), is(expectedNextJobStatus)); + + pulledJobs.forEach(job -> + jobsBrokerService.pushBack(job) + ); + + return nextJob; + } + + private void pullAllJobProcessAndPushBackByType(JobStatus topic, JobType commandType, JobStatus expectedFinalStatus) { + Map jobStatusMap = new HashMap<>(); + Optional job = pullJob(topic, false); + for (int i=0; i<1000 && job.isPresent() && job.get().getType() == commandType; i++) { + Job nextJob = jobWorker.executeJobAndGetNext(job.get()); + jobStatusMap.put(nextJob.getUuid(), nextJob.getStatus()); + jobsBrokerService.pushBack(nextJob); + job = jobsBrokerService.pull(topic, UUID.randomUUID().toString()); + } + assertThat(jobStatusMap.values(), everyItem(is(expectedFinalStatus))); + + } + + private Optional pullJob(JobStatus topic, boolean pullingAssertion) { + if (pullingAssertion) { + //assert pulling on inverse topic return no result + assertFalse(jobsBrokerService.pull(inverseTopic(topic), randomUuid()).isPresent()); + } + + Optional job = jobsBrokerService.pull(topic, randomUuid()); + assertTrue("no job fetched", job.isPresent()); + + if (pullingAssertion) { + //assert another pulling on same topic return no result + assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent()); + } + + return job; + } + + private JobStatus inverseTopic(JobStatus topic) { + return topic==JobStatus.IN_PROGRESS ? PENDING : JobStatus.IN_PROGRESS; + } + + + //@Test + public void whenPushNewBulk_andGetNoResponseFromMsoOnCreation_thenServiceMoveToFailedAndOtherToStopped() { + when(restMso.PostForObject(any(), any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(500)); + pushBulkPullPendingJobAndAssertJobStatus(JobStatus.FAILED, JobStatus.STOPPED); + } + + //@Test + public void whenMsoStatusIsPendingManualTask_ThenJobStatusIsPaused() { + when(restMso.PostForObject(any(), any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(200)); + + Job firstJob = pushBulkPullPendingJobAndAssertJobStatus(JobStatus.IN_PROGRESS, PENDING); + + //assert that when get ProcessingException from restMso, status remain the same + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(PENDING_MANUAL_TASK)); + Job job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS); + listServicesAndAssertStatus(PAUSE, PENDING, job); + + //The paused job is pulled and remain in pause state. Other jobs from bulk remain pending + job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS); + listServicesAndAssertStatus(PAUSE, PENDING, job); + + //the job get IN_PROGRESS response (simulate activate operation) and status changed to IN_PROGRESS + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR)); + job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS); + listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job); + + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED); + listServicesAndAssertStatus(COMPLETED, PENDING, job); + + //Pulling PENDING job return another job + assertThat(jobsBrokerService.pull(PENDING, randomUuid()).get().getUuid(), not(equalTo(job.getUuid()))); + + + ImmutableList expectedStatusesForMso = ImmutableList.of(REQUESTED, PENDING_MANUAL_TASK, IN_PROGRESS_STR, COMPLETE_STR); + List msoStatuses = asyncInstantiationBL.getAuditStatuses(firstJob.getUuid(), MSO).stream().map(x -> x.getJobStatus()).collect(Collectors.toList()); + assertThat(msoStatuses, is(expectedStatusesForMso)); + + ImmutableList expectedStatusesForVid = statusesToStrings(PENDING, IN_PROGRESS, PAUSE, IN_PROGRESS, COMPLETED); + List vidStatuses = asyncInstantiationBL.getAuditStatuses(firstJob.getUuid(), VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList()); + assertThat(vidStatuses, is(expectedStatusesForVid)); + } + + private Job pushBulkPullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) { + pushMacroBulk(); + return pullPendingJobAndAssertJobStatus(pulledJobStatus, otherJobsStatus); + } + + private Job pullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) { + Job job = pullJobProcessAndPushBack(PENDING, pulledJobStatus, false); + listServicesAndAssertStatus(pulledJobStatus, otherJobsStatus, job); + return job; + } + + //@Test + public void test2BulksLifeCyclesAreIndependent() { + pushMacroBulk(); + when(restMso.PostForObject(any(), any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(200)); + //push 2nd job, then when pulling first job the job become in_progress, other jobs (from 2 bulks) remain pending + Job firstJob = pushBulkPullPendingJobAndAssertJobStatus(JobStatus.IN_PROGRESS, PENDING); + + //assert we can pull another job from pending from other template id + Job secondJob = pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS, false); + assertThat(firstJob.getTemplateId(), not(equalTo(secondJob.getTemplateId()))); + + //assert no more PENDING jobs to pull + assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent()); + + //when get FAILED status from MSO statuses for failed bulk are: FAILED, STOPPED, for other bulk: IN_PROGRESS, 2 pending + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR)); + Job failedJob = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED, false); + Map> servicesByTemplateId = + asyncInstantiationBL.getAllServicesInfo() + .stream().collect(groupingBy(ServiceInfo::getTemplateId)); + assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob); + Job successJob = failedJob.getUuid().equals(firstJob.getUuid()) ? secondJob : firstJob; + assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), JobStatus.IN_PROGRESS, PENDING, successJob); + + //yet no more PENDING jobs to pull + assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent()); + + //assert that job from non failed bulk can progress. + //When completed, failed bulk statuses: FAILED, STOPPED. Succeeded bulk statuses are : COMPLETED, 2 pending + when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED, false); + servicesByTemplateId = + asyncInstantiationBL.getAllServicesInfo() + .stream().collect(groupingBy(ServiceInfo::getTemplateId)); + assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob); + assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, PENDING, successJob); + + //advance other jobs of succeeded bulk till al of them reach to COMPLETED + pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS, false); + pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED, false); + pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS, false); + pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED, false); + servicesByTemplateId = + asyncInstantiationBL.getAllServicesInfo() + .stream().collect(groupingBy(ServiceInfo::getTemplateId)); + assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob); + assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, COMPLETED, successJob); + + //assert no more PENDING jobs nor IN_PROGRESS jobs to pull + assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent()); + assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent()); + } + + public void deploy2NewGroupsToServiceWith1ExistingGroup() { + + /* + new feature: skip service (existing impl) and skip group (new impl) + service+group aren't touched, 2 groups ARE created + + [v] success if all GROUPs success + + Next test variation should: + [ ] error if all GROUPs error + [ ] completed with error if 1 GROUP error + + + [v] + service with 3 groups - 1 action=none, 2 action=create; service's action=none + [v] verify STATUS pending + [v] + pull+execute (should NOT post to MSO) + [v] verify STATUS in progress; TYPE watching + ... + [v] verify job#2 *new* GROUP job STATUS completed with no action TYPE group INTERNAL STATE terminal PHASE delete + [v] verify job#3 *new* GROUP job STATUS completed with no action TYPE group INTERNAL STATE terminal PHASE delete + [v] verify job#4 *new* GROUP job STATUS completed with no action TYPE group INTERNAL STATE terminal PHASE delete + + [v] + pull+execute job#1 (should NOT post to MSO) + [v] verify STATUS in progress; TYPE watching + [v] verify job#5 *new* GROUP job STATUS creating TYPE group INTERNAL STATE initial PHASE create + [v] verify job#6 *new* GROUP job STATUS creating TYPE group INTERNAL STATE initial PHASE create + [v] verify job#7 *new* GROUP job STATUS creating TYPE group INTERNAL STATE initial PHASE create + + [v] + pull+execute job#5 (should NOT post to MSO) + [v] verify job#5 STATUS completed with no action TYPE group INTERNAL STATE terminal PHASE create + [v] + pull+execute job#1 + [v] verify job#1 STATUS in progress; TYPE watching + + [v] + pull+execute job#6 (should post to MSO) + [v] verify job#6 STATUS resource in progress + [v] + pull+execute job#1 + [v] verify job#1 STATUS in progress; TYPE watching + [v] + pull+execute job#6 (should get from MSO) + [v] verify job#6 STATUS completed + [v] + pull+execute job#1 + [v] verify job#1 STATUS in progress; TYPE watching + + [v] + pull+execute job#7 (should post to MSO) + [v] verify job#7 STATUS resource in progress + [v] + pull+execute job#1 + [v] verify job#1 STATUS in progress; TYPE watching + [v] + pull+execute job#7 (should get from MSO) + [v] verify job#7 STATUS completed + [v] + pull+execute job#1 + [v] verify job#1 STATUS completed + + */ + + final String GROUP1_REQUEST_ID = UUID.randomUUID().toString(); + final String GROUP1_INSTANCE_ID = UUID.randomUUID().toString(); + final String GROUP2_REQUEST_ID = UUID.randomUUID().toString(); + final String GROUP2_INSTANCE_ID = UUID.randomUUID().toString(); + + // Utility method + final BiConsumer verify_Job1InProgress = (phase, nextJobStatus) -> { + pullJobProcessAndPushBackWithTypeAssertion(IN_PROGRESS, nextJobStatus, JobType.ALaCarteService, phase, InternalState.WATCHING, 2); + }; + + //service with 3 groups - 1 action=none, 2 action=create; service's action=none + UUID uuid = pushALaCarteUpdateWithGroups(); + singleServicesAndAssertStatus(PENDING, uuid); + + // take from pending, put in-progress -> 3 delete-child were born + pullJobProcessAndPushBackWithTypeAssertion(PENDING, IN_PROGRESS, JobType.ALaCarteService, Action.Delete, InternalState.WATCHING, 1); + verifyQueueSizes(ImmutableMap.of( + IN_PROGRESS, 1, CREATING, 3 + )); + + Stream.of(1, 2, 3).forEach(i -> { + // take each child creating, put in-progress + verify_Job1InProgress.accept(Action.Delete, IN_PROGRESS); + pullJobProcessAndPushBackWithTypeAssertion(CREATING, RESOURCE_IN_PROGRESS, JobType.InstanceGroup, Action.Delete, null, 1); + + // execute each in-progress -> job is completed + verify_Job1InProgress.accept(Action.Delete, IN_PROGRESS); + pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED/*_WITH_NO_ACTION*/, JobType.InstanceGroup,1); + }); + verifyQueueSizes(ImmutableMap.of( + IN_PROGRESS, 1, COMPLETED, 3 + )); + + // take job #1 from phase delete to phase create -> 3 create-child were born + verify_Job1InProgress.accept(Action.Create, IN_PROGRESS); + verifyQueueSizes(ImmutableMap.of( + IN_PROGRESS, 1, CREATING, 3, COMPLETED, 3 + )); + + // prepare MSO mock + when(restMso.PostForObject(any(), endsWith("instanceGroups"), eq(RequestReferencesContainer.class))) + .thenReturn(createResponse(200, GROUP1_INSTANCE_ID, GROUP1_REQUEST_ID)) + .thenReturn(createResponse(200, GROUP2_INSTANCE_ID, GROUP2_REQUEST_ID)) + .thenReturn(null); + when(restMso.GetForObject(argThat(uri -> StringUtils.endsWithAny(uri, GROUP1_REQUEST_ID, GROUP2_REQUEST_ID)), eq(AsyncRequestStatus.class))). + thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR)); + + // take first "none" child from creating to completed + // note there's no concrete mechanism that makes the first child be + // the "action=None" case, but that's what happens, and following line + // relies on that fact. + pullJobProcessAndPushBackWithTypeAssertion(CREATING, COMPLETED_WITH_NO_ACTION, JobType.InstanceGroupInstantiation, 1); + + // take each of next two children from creating to in-progress, then to completed + // verify job #1 is watching, and MSO is getting requests + Stream.of(1, 2).forEach(i -> { + verify_Job1InProgress.accept(Action.Create, IN_PROGRESS); + pullJobProcessAndPushBackWithTypeAssertion(CREATING, RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus); + verify(restMso, times(i)).PostForObject(any(), any(), any()); + + verify_Job1InProgress.accept(Action.Create, IN_PROGRESS); + pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED, JobType.ResourceInProgressStatus); + verify(restMso, times(i)).GetForObject(any(), any()); + }); + + // job #1 is done as all children are done + verify_Job1InProgress.accept(Action.Create, COMPLETED); + verifyQueueSizes(ImmutableMap.of(COMPLETED, 7)); + } + + private void verifyQueueSizes(ImmutableMap expected) { + final Collection peek = jobsBrokerService.peek(); + final Map jobTypes = peek.stream().collect(groupingBy(Job::getStatus, counting())); + assertThat(jobTypes, is(expected)); + } + + private List listServicesAndAssertStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus, Job job) { + List serviceInfoList = asyncInstantiationBL.getAllServicesInfo(); + assertServicesStatus(serviceInfoList, pulledJobStatus, otherJobsStatus, job); + + return serviceInfoList; + } + + private ServiceInfo singleServicesAndAssertStatus(JobStatus expectedStatus, UUID jobUUID) { + List serviceInfoList = asyncInstantiationBL.getAllServicesInfo(); + assertEquals(1, serviceInfoList.size()); + ServiceInfo serviceInfo = serviceInfoList.get(0); + assertThat(serviceInfo.getJobStatus(), is(expectedStatus)); + assertThat(serviceInfo.getJobId(), is(jobUUID)); + return serviceInfo; + } + + private void assertServicesStatus(List serviceInfoList, JobStatus pulledJobStatus, JobStatus otherJobsStatus, Job job) { + serviceInfoList.forEach(si->{ + if (si.getJobId().equals(job.getUuid())) { + assertThat(si.getJobStatus(), is(pulledJobStatus)); + } + else { + assertThat(si.getJobStatus(), is(otherJobsStatus)); + } + }); + } + + private void listServicesAndAssertStatus(Map expectedJobStatusMap) { + Map actualStatuses = asyncInstantiationBL.getAllServicesInfo() + .stream().collect(Collectors.toMap(ServiceInfo::getJobId, ServiceInfo::getJobStatus)); + assertThat(actualStatuses.entrySet(), equalTo(expectedJobStatusMap.entrySet())); + } + + private String randomUuid() { + return UUID.randomUUID().toString(); + } +} -- cgit 1.2.3-korg