1 files changed, 573 insertions, 0 deletions
diff --git a/vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java b/vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java
new file mode 100644
index 000000000..2a6b012fe
--- /dev/null
+++ b/vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java
@@ -0,0 +1,573 @@
+package org.onap.vid.api;
+
+import static java.lang.Boolean.FALSE;
+import static java.lang.Boolean.TRUE;
+import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toMap;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.hasSize;
+import static org.onap.simulator.presetGenerator.presets.mso.PresetMSOServiceInstanceGen2WithNames.Keys;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+import static vid.automation.test.utils.ExtendedHamcrestMatcher.hasItemsFromCollection;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.Uninterruptibles;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.Description;
+import org.hamcrest.MatcherAssert;
+import org.onap.simulator.presetGenerator.presets.BasePresets.BaseMSOPreset;
+import org.onap.simulator.presetGenerator.presets.BasePresets.BasePreset;
+import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetCloudOwnersByCloudRegionId;
+import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetSubscribersGet;
+import org.onap.simulator.presetGenerator.presets.ecompportal_att.PresetGetSessionSlotCheckIntervalGet;
+import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseCreateInstancePost;
+import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseDelete;
+import org.onap.simulator.presetGenerator.presets.mso.PresetMSOCreateServiceInstanceGen2WithNames;
+import org.onap.simulator.presetGenerator.presets.mso.PresetMSOOrchestrationRequestGet;
+import org.onap.vid.model.asyncInstantiation.JobAuditStatus;
+import org.onap.vid.model.asyncInstantiation.ServiceInfo;
+import org.onap.vid.model.mso.MsoResponseWrapper2;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.RestTemplate;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import vid.automation.test.infra.Features;
+import vid.automation.test.infra.Wait;
+import vid.automation.test.model.JobStatus;
+import vid.automation.test.model.ServiceAction;
+import vid.automation.test.services.AsyncJobsService;
+import vid.automation.test.services.SimulatorApi;
+
+public class AsyncInstantiationBase extends BaseMsoApiTest {
+
+    public static final String CREATE_BULK_OF_ALACARTE_REQUEST_WITH_VNF = "asyncInstantiation/vidRequestCreateALaCarteWithVnf.json";
+    protected static final String CREATE_BULK_OF_MACRO_REQUEST = "asyncInstantiation/vidRequestCreateBulkOfMacro.json";
+
+    protected static final String MSO_BASE_ERROR =
+            "Received error from SDN-C: java.lang.IllegalArgumentException: All keys must be specified for class org."+
+            "opendaylight.yang.gen.v1.org.onap.sdnc.northbound.generic.resource.rev170824.vf.module.assignments.vf."+
+            "module.assignments.vms.VmKey. Missing key is getVmType. Supplied key is VmKey [].";
+    protected static final String MSO_ERROR = MSO_BASE_ERROR + StringUtils.repeat(" and a lot of sentences for long message", 60);
+
+    @BeforeClass
+    protected void muteAndDropNameCounter() {
+        AsyncJobsService asyncJobsService = new AsyncJobsService();
+        asyncJobsService.muteAllAsyncJobs();
+        asyncJobsService.dropAllFromNameCounter();
+    }
+
+    @AfterMethod
+    protected void muteAllAsyncJobs() {
+        AsyncJobsService asyncJobsService = new AsyncJobsService();
+        asyncJobsService.muteAllAsyncJobs();
+    }
+
+    @DataProvider
+    public static Object[][] trueAndFalse() {
+            return new Object[][]{{TRUE},{FALSE}};
+    }
+
+    protected String getCreateBulkUri() {
+        return uri.toASCIIString() + "/asyncInstantiation/bulk";
+    }
+
+    protected String getHideServiceUri(String jobId) {
+        return uri.toASCIIString() + "/asyncInstantiation/hide/"+jobId;
+    }
+
+    protected String getServiceInfoUrl() {
+        return uri.toASCIIString() + "/asyncInstantiation";
+    }
+
+    protected String getJobAuditUrl() {
+        return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}?source={SOURCE}";
+    }
+
+    protected String getMsoJobAuditUrl() {
+        return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}/mso";
+    }
+
+    protected String getDeleteServiceUrl(String uuid) {
+        return uri.toASCIIString() + "/asyncInstantiation/job/" + uuid;
+    }
+
+    protected String getInstanceAuditInfoUrl() {
+        return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{TYPE}/{INSTANCE_ID}/mso";
+    }
+
+    protected String getRetryJobUrl() {
+        return uri.toASCIIString() + "/asyncInstantiation/retry/{JOB_ID}";
+    }
+    protected String getTopologyForRetryUrl() {
+        return uri.toASCIIString() + "/asyncInstantiation/bulkForRetry/{JOB_ID}";
+    }
+
+
+    protected String getRetryJobWithChangedDataUrl() {
+        return uri.toASCIIString() + "/asyncInstantiation/retryJobWithChangedData/{JOB_ID}";
+    }
+
+    protected boolean getExpectedRetryEnabled(JobStatus jobStatus) {
+        return Features.FLAG_1902_RETRY_JOB.isActive() && (jobStatus==JobStatus.FAILED || jobStatus==JobStatus.COMPLETED_WITH_ERRORS);
+    }
+
+    public List<BasePreset> getPresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
+
+        final ImmutableList.Builder<BasePreset> basePresetBuilder = new ImmutableList.Builder<>();
+        basePresetBuilder
+                .add(new PresetGetSessionSlotCheckIntervalGet())
+                .add(new PresetAAIGetSubscribersGet())
+                .addAll(presetOnDeleteList)
+                .addAll(presetOnCreateList)
+                .addAll(presetInProgressList);
+        return basePresetBuilder.build();
+    }
+
+    public List<BasePreset> getDeletePresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
+        return getPresets(presetOnDeleteList, emptyList(), presetInProgressList);
+    }
+
+    public List<BasePreset> getPresets(List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
+        return getPresets(emptyList(), presetOnCreateList, presetInProgressList);
+    }
+
+    public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName) {
+        assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, userName, null, ServiceAction.INSTANTIATE);
+    }
+
+    public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName, String instanceId, ServiceAction action) {
+        assertExpectedStatusAndServiceInfo(jobStatus, jobId, new ServiceInfo(
+                userName, jobStatus, false,
+                "038d99af-0427-42c2-9d15-971b99b9b489", "Lucine Sarika", "zasaki",
+                "de738e5f-3704-4a14-b98f-3bf86ac0c0a0", "voloyakane-senamo",
+                "c85f0e80-0636-44a4-8cb2-4ec00d056e79", "Hedvika Wendelin",
+                "a93f8383-707e-43fa-8191-a6e69a1aab17", null,
+                "TYLER SILVIA", "SILVIA ROBBINS",
+                instanceId, serviceInstanceName,
+                "e3c34d88-a216-4f1d-a782-9af9f9588705", "gayawabawe", "5.1",
+                jobId, null, action, false)
+        );
+    }
+
+    public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName) {
+        assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, "us16807000");
+    }
+
+    protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses) {
+        assertAuditStatuses(jobId, expectedVidStatuses, expectedMsoStatuses, 15);
+    }
+
+    protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses, long timeoutInSeconds) {
+        assertAndRetryIfNeeded(() -> {
+            final List<JobAuditStatus> auditVidStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name());
+            assertThat(auditVidStatuses, hasItemsFromCollection(expectedVidStatuses));
+            if (expectedMsoStatuses!=null) {
+                final List<JobAuditStatus> auditMsoStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
+                assertThat(auditMsoStatuses, containsInAnyOrder(expectedMsoStatuses.toArray()));
+            }
+        }, timeoutInSeconds);
+    }
+
+    protected void assertAndRetryIfNeeded(Runnable asserter, long timeoutInSeconds) {
+        final Instant expiry = Instant.now().plusSeconds(timeoutInSeconds);
+        while (true) {
+            try {
+                asserter.run();
+                break; // we're cool, assertion passed
+            } catch (AssertionError fail) {
+                Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+                if (Instant.now().isAfter(expiry)) {
+                    throw fail;
+                } else {
+                    System.out.println("retrying after: " + fail);
+                }
+            }
+        }
+    }
+
+    protected ImmutableList<JobAuditStatus> vidAuditStatusesCompletedWithErrors(String jobId) {
+        return ImmutableList.of(
+                vidAuditStatus(jobId, "PENDING", false),
+                vidAuditStatus(jobId, "IN_PROGRESS", false),
+                vidAuditStatus(jobId, "COMPLETED_WITH_ERRORS", true)
+        );
+    }
+
+    protected ImmutableList<JobAuditStatus> vidAuditStatusesFailed(String jobId) {
+        return ImmutableList.of(
+                vidAuditStatus(jobId, "PENDING", false),
+                vidAuditStatus(jobId, "IN_PROGRESS", false),
+                vidAuditStatus(jobId, "FAILED", true)
+        );
+    }
+
+    protected JobAuditStatus vidAuditStatus(String jobId, String jobStatus, boolean isFinal) {
+        return new JobAuditStatus(UUID.fromString(jobId), jobStatus, JobAuditStatus.SourceStatus.VID, null, null, isFinal);
+    }
+
+    public static class JobIdAndStatusMatcher extends BaseMatcher<ServiceInfo> {
+        protected String expectedJobId;
+
+        public JobIdAndStatusMatcher(String expectedJobId) {
+            this.expectedJobId = expectedJobId;
+        }
+
+        @Override
+        public boolean matches(Object item) {
+            if (!(item instanceof ServiceInfo)) {
+                return false;
+            }
+            ServiceInfo serviceInfo = (ServiceInfo) item;
+            return expectedJobId.equals(serviceInfo.jobId);
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            description.appendText("failed to find job with uuid ")
+                    .appendValue(expectedJobId);
+        }
+    }
+
+
+
+    protected Map<Keys,String> generateNames() {
+        return Stream.of(Keys.values()).collect(
+                Collectors.toMap(x->x, x -> UUID.randomUUID().toString().replace("-","")));
+    }
+
+    protected ImmutableList<BasePreset> addPresetsForCreateBulkOfCreateInstances(int bulkSize, Map<Keys, String> names){
+        ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
+        ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
+                .add(new PresetGetSessionSlotCheckIntervalGet())
+                .add(new PresetAAIGetSubscribersGet())
+                .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
+                .addAll(msoBulkPresets)
+                .add(new PresetMSOOrchestrationRequestGet())
+                .build();
+        return presets;
+
+    }
+
+    protected ImmutableList<BasePreset> generateMsoCreateBulkPresets(int bulkSize, Map<Keys, String> names) {
+        return IntStream.rangeClosed(0, bulkSize-1).
+                mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
+                .collect(ImmutableList.toImmutableList());
+    }
+
+    protected ResponseEntity<List<JobAuditStatus>> auditStatusCall(String url) {
+        return restTemplate.exchange(
+                url,
+                org.springframework.http.HttpMethod.GET,
+                null,
+                new ParameterizedTypeReference<List<JobAuditStatus>>() {});
+    }
+
+    @DataProvider
+    public static Object[][] auditSources() {
+        return new Object[][]{{JobAuditStatus.SourceStatus.VID},{JobAuditStatus.SourceStatus.MSO}};
+    }
+
+
+
+    protected List<String> createBulkAndWaitForBeCompleted(int bulkSize){
+        Map<Keys, String> names = generateNames();
+        ImmutableList<BasePreset> presets = addPresetsForCreateBulkOfCreateInstances(bulkSize, names);
+        final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
+        Assert.assertEquals(jobIds.size(),bulkSize);
+
+        assertTrue(String.format("Not all services with ids: %s are in state completed after 30 sec",
+                jobIds.stream().collect(joining(","))),
+
+                Wait.waitFor(y-> serviceListCall().getBody().stream()
+                        .filter(si -> jobIds.contains(si.jobId))
+                        .filter(si -> si.jobStatus==JobStatus.COMPLETED)
+                        .count() == bulkSize,
+                null, 30, 1 ));
+        return jobIds;
+    }
+
+    protected List<JobAuditStatus> getJobMsoAuditStatusForAlaCarte(String jobUUID, String requestId, String serviceInstanceId){
+        String url = getMsoJobAuditUrl().replace("{JOB_ID}",jobUUID);
+
+        if(!StringUtils.isEmpty(requestId)) {
+            url = url + "?requestId=" + requestId;
+            if(!StringUtils.isEmpty(serviceInstanceId)) {
+                url = url + "&serviceInstanceId=" + serviceInstanceId;
+            }
+        }
+        return callAuditStatus(url);
+    }
+
+    protected List<JobAuditStatus> getAuditStatuses(String jobUUID, String source){
+        String url = getJobAuditUrl().replace("{JOB_ID}",jobUUID).replace("{SOURCE}", source);
+        return callAuditStatus(url);
+    }
+
+    protected List<JobAuditStatus> getAuditStatusesForInstance(String type, String instanceId){
+        String url = getInstanceAuditInfoUrl().replace("{TYPE}",type).replace("{INSTANCE_ID}", instanceId);
+        return callAuditStatus(url);
+    }
+
+    private List<JobAuditStatus> callAuditStatus(String url) {
+        ResponseEntity<List<JobAuditStatus>> statusesResponse = auditStatusCall(url);
+        assertThat(statusesResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
+        return statusesResponse.getBody();
+    }
+
+    protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(){
+        return addBulkAllPendingButOneInProgress(3);
+    }
+
+    protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(int bulkSize){
+        Map<Keys, String> names = generateNames();
+        ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
+        ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
+                .add(new PresetGetSessionSlotCheckIntervalGet())
+                .add(new PresetAAIGetSubscribersGet())
+                .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
+                .addAll(msoBulkPresets)
+                .add(new PresetMSOOrchestrationRequestGet("IN_PROGRESS"))
+                .build();
+        final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
+
+        // wait for single IN_PROGRESS, so statuses will stop from changing
+        Wait.waitFor(foo -> serviceListCall().getBody().stream()
+                        .filter(si -> jobIds.contains(si.jobId))
+                        .anyMatch(si -> si.jobStatus.equals(JobStatus.IN_PROGRESS)),
+                null, 20, 1);
+
+        final Map<String, JobStatus> statusMapBefore = serviceListCall().getBody().stream()
+                .filter(si -> jobIds.contains(si.jobId))
+                .collect(toMap(si -> si.jobId, si -> si.jobStatus));
+
+        assertThat(jobIds, hasSize(bulkSize));
+
+
+        return statusMapBefore;
+    }
+
+    protected String deleteOneJobHavingTheStatus(Map<String, JobStatus> jobIdToStatus, JobStatus jobStatus) {
+        final String jobToDelete = jobIdToStatus.entrySet().stream()
+                .filter(entry -> entry.getValue().equals(jobStatus))
+                .map(Map.Entry::getKey)
+                .findFirst().orElseThrow(() -> new AssertionError("no job in " + jobStatus + " state: " + jobIdToStatus));
+
+
+        restTemplate.delete(getDeleteServiceUrl(jobToDelete));
+
+        return jobToDelete;
+    }
+
+
+    protected MsoResponseWrapper2 hideService(String jobId) {
+        MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getHideServiceUri(jobId), "");
+        return responseWrapper2;
+    }
+
+    protected List<String> createBulkOfInstancesAndAssert(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, JobStatus finalState, Map<Keys, String> names){
+        List<String> jobIds = createBulkOfMacroInstances(presets, isPause, bulkSize, names);
+        Assert.assertEquals(jobIds.size(), bulkSize);
+        for(String jobId: jobIds) {
+            assertExpectedStatusAndServiceInfo(isPause, finalState, names, jobId);
+        }
+
+        return jobIds;
+    }
+
+    protected void assertExpectedStatusAndServiceInfo(boolean isPause, JobStatus finalState, Map<Keys, String> names, String jobId) {
+        assertExpectedStatusAndServiceInfo(finalState, jobId, new ServiceInfo("us16807000", JobStatus.IN_PROGRESS, isPause, "someID",
+                "someName", "myProject", "NFT1", "NFTJSSSS-NFT1", "greatTenant", "greatTenant", "hvf3", null,
+                "mySubType", "a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb", null, names.get(Keys.SERVICE_NAME),
+                "5c9e863f-2716-467b-8799-4a67f378dcaa", "AIM_TRANSPORT_00004", "1.0", jobId, null, ServiceAction.INSTANTIATE, false));
+    }
+
+    protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, ServiceInfo expectedServiceInfo) {
+        assertExpectedStatusAndServiceInfo(finalState, jobId, false, expectedServiceInfo);
+    }
+
+    protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, boolean longWait, ServiceInfo expectedServiceInfo) {
+        JobInfoChecker<Integer> jobInfoChecker = new JobInfoChecker<>(
+                restTemplate, ImmutableSet.of(JobStatus.PENDING, JobStatus.IN_PROGRESS, finalState), jobId, expectedServiceInfo);
+        boolean result = jobInfoChecker.test(null);
+        assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
+
+        jobInfoChecker.setExpectedJobStatus(ImmutableSet.of(finalState));
+        if (ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE).contains(finalState) && expectedServiceInfo.serviceInstanceId==null) {
+            expectedServiceInfo.serviceInstanceId = BaseMSOPreset.DEFAULT_INSTANCE_ID;
+        }
+        result = Wait.waitFor(jobInfoChecker, null, 30, longWait ? 2 : 1);
+        assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
+    }
+
+    protected List<String> createBulkOfMacroInstances(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, Map<Keys, String> names) {
+        SimulatorApi.registerExpectationFromPresets(presets, SimulatorApi.RegistrationStrategy.CLEAR_THEN_SET);
+        return createBulkOfInstances(isPause, bulkSize, names, CREATE_BULK_OF_MACRO_REQUEST);
+    }
+
+    public List<String> createBulkOfInstances(boolean isPause, int bulkSize, Map<Keys, String> names, String requestDetailsFileName){
+
+        String requestBody = TestUtils.convertRequest(objectMapper, requestDetailsFileName);
+        requestBody = requestBody.replace("\"IS_PAUSE_VALUE\"", String.valueOf(isPause)).replace("\"BULK_SIZE\"", String.valueOf(bulkSize));
+        for (Map.Entry<Keys, String> e : names.entrySet()) {
+            requestBody = requestBody.replace(e.getKey().name(), e.getValue());
+        }
+        MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getCreateBulkUri(), requestBody);
+        assertNotNull(responseWrapper2);
+        return (List<String>)responseWrapper2.getEntity();
+    }
+
+    protected List<String> retryJob(String jobId) {
+        ResponseEntity<String> retryBulkPayload = getRetryBulk(jobId);
+        return retryJobWithChangedData(jobId, retryBulkPayload.getBody());
+    }
+
+    protected List<String> retryJobWithChangedData(String jobId, String requestBody) {
+        String retryUri = getRetryJobWithChangedDataUrl();
+        retryUri = retryUri.replace("{JOB_ID}", jobId);
+        MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(HttpMethod.POST, retryUri, requestBody);
+        assertNotNull(responseWrapper2);
+        return (List<String>)responseWrapper2.getEntity();
+    }
+
+    protected ResponseEntity<String> getRetryBulk(String jobId) {
+        String retryUri = getTopologyForRetryUrl();
+        retryUri = retryUri.replace("{JOB_ID}", jobId);
+        return restTemplateErrorAgnostic.getForEntity(retryUri, String.class);
+    }
+
+    protected Object getResourceAuditInfo(String trackById) {
+        return restTemplate.getForObject(buildUri("/asyncInstantiation/auditStatusForRetry/{trackById}"), Object.class, trackById);
+    }
+
+    public class JobInfoChecker<Integer> implements Predicate<Integer> {
+
+        protected final RestTemplate restTemplate;
+        protected Set<JobStatus> expectedJobStatus;
+        protected ServiceInfo expectedServiceInfo;
+        protected final String jobId;
+        protected JobStatus lastStatus;
+
+        public JobInfoChecker(RestTemplate restTemplate, Set<JobStatus> expectedJobStatus, String jobId, ServiceInfo expectedServiceInfo) {
+            this.restTemplate = restTemplate;
+            this.expectedJobStatus = expectedJobStatus;
+            this.jobId = jobId;
+            this.expectedServiceInfo = expectedServiceInfo;
+        }
+
+        public void setExpectedJobStatus(Set<JobStatus> expectedJobStatus) {
+            this.expectedJobStatus = expectedJobStatus;
+        }
+
+        @Override
+        public boolean test(Integer integer) {
+            ResponseEntity<List<ServiceInfo>> serviceListResponse = serviceListCall();
+            assertThat(serviceListResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
+            assertThat(serviceListResponse.getBody(), hasItem(new JobIdAndStatusMatcher(jobId)));
+            ServiceInfo serviceInfoFromDB = serviceListResponse.getBody().stream()
+                    .filter(serviceInfo -> serviceInfo.jobId.equals(jobId))
+                    .findFirst().orElse(null);
+            Assert.assertNotNull(serviceInfoFromDB);
+            Assert.assertEquals(serviceInfoDataReflected(expectedServiceInfo), serviceInfoDataReflected(serviceInfoFromDB));
+            assertTrue("actual service instance doesn't contain template service name:" + expectedServiceInfo.serviceInstanceName,
+                    serviceInfoFromDB.serviceInstanceName.contains(expectedServiceInfo.serviceInstanceName));
+
+            if (expectedServiceInfo.serviceInstanceId != null && ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE, JobStatus.COMPLETED_WITH_ERRORS).contains(serviceInfoFromDB.jobStatus)) {
+                MatcherAssert.assertThat("service instance id is wrong", serviceInfoFromDB.serviceInstanceId, CoreMatchers.is(expectedServiceInfo.serviceInstanceId));
+            }
+            if (expectedJobStatus.size()==1) {
+                assertEquals("job status is wrong", getExpectedRetryEnabled((JobStatus)(expectedJobStatus.toArray()[0])), serviceInfoFromDB.isRetryEnabled);
+            }
+            lastStatus = serviceInfoFromDB.jobStatus;
+            return expectedJobStatus.contains(serviceInfoFromDB.jobStatus);
+        }
+    }
+
+    protected ResponseEntity<List<ServiceInfo>> serviceListCall() {
+        return restTemplate.exchange(
+                getServiceInfoUrl(),
+                org.springframework.http.HttpMethod.GET,
+                null,
+                new ParameterizedTypeReference<List<ServiceInfo>>() {});
+    }
+
+    //serialize fields except of fields we cannot know ahead of time
+    protected static String serviceInfoDataReflected(ServiceInfo service1) {
+        return new ReflectionToStringBuilder(service1, ToStringStyle.SHORT_PREFIX_STYLE)
+                .setExcludeFieldNames("jobStatus", "templateId", "statusModifiedDate", "createdBulkDate", "serviceInstanceId", "serviceInstanceName", "isRetryEnabled")
+                .toString();
+    }
+
+    protected void addBulkPendingWithCustomList(List<BasePreset> customPresets){
+        Map<Keys, String> names = generateNames();
+        final int bulkSize = 2 + customPresets.size();
+
+        List<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
+        ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
+                .add(new PresetGetSessionSlotCheckIntervalGet())
+                .add(new PresetAAIGetSubscribersGet())
+                .addAll(msoBulkPresets)
+                .addAll(customPresets)
+                .build();
+
+        List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
+        Assert.assertEquals(jobIds.size(),bulkSize);
+    }
+
+    protected void verifyAuditStatuses(String jobId, List<String> statuses, JobAuditStatus.SourceStatus source) {
+        int statusesSize = statuses.size();
+        AtomicReference<List<JobAuditStatus>> actualAudits = new AtomicReference<>();
+        if (source.equals(JobAuditStatus.SourceStatus.VID)) {
+            actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name()));
+            org.junit.Assert.assertEquals("Received number of VID statuses is not as expected", statusesSize, actualAudits.get().size());
+        } else {
+            boolean isStatusedSizeAsExpected = Wait.waitFor(x-> {
+                actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name()));
+                return actualAudits.get().size() == statusesSize;
+            },null,5,1);
+            org.junit.Assert.assertTrue("Received number of MSO statuses is not as expected. Expected: " + statusesSize + ". Received: " + actualAudits.get().size(), isStatusedSizeAsExpected);
+        }
+        IntStream.range(0, statusesSize).forEach(i-> org.junit.Assert.assertEquals(source + " status #" + i + " is not as expected", statuses.get(i), actualAudits.get().get(i).getJobStatus()));
+    }
+
+    protected void verifyInstanceAuditStatuses(List<JobAuditStatus> expectedStatuses, List<JobAuditStatus> actualStatuses) {
+        final int expectedSize = expectedStatuses.size();
+        assertTrue("Expected statuses size is "+ expectedSize +", actual size is "+actualStatuses.size(), new Integer(expectedSize).equals(actualStatuses.size()));
+        IntStream.range(0, expectedSize).forEach(i-> {
+
+            final JobAuditStatus expectedStatus = expectedStatuses.get(i);
+            final JobAuditStatus actualStatus = actualStatuses.get(i);
+            org.junit.Assert.assertEquals("MSO status #" + i + " is not as expected", expectedStatus.getJobStatus(), actualStatus.getJobStatus());
+            org.junit.Assert.assertEquals("MSO requestId #" + i + " is not as expected", expectedStatus.getRequestId(), actualStatus.getRequestId());
+            org.junit.Assert.assertEquals("MSO additionalInfo #" + i + " is not as expected", expectedStatus.getAdditionalInfo(), actualStatus.getAdditionalInfo());
+            org.junit.Assert.assertEquals("MSO jobID #" + i + " is not as expected", expectedStatus.getJobId(), actualStatus.getJobId());
+            org.junit.Assert.assertEquals("MSO instanceName #" + i + " is not as expected", expectedStatus.getInstanceName(), actualStatus.getInstanceName());
+            org.junit.Assert.assertEquals("MSO instanceType  #" + i + " is not as expected", expectedStatus.getInstanceType(), actualStatus.getInstanceType());
+        });
+    }
+}