Renaming vid-automation #4

Change-Id: I907b9a6c199302d748918e236ee2945d56f4dd26
Issue-ID: VID-205
Signed-off-by: kurczews <krzysztof.kurczewski@nokia.com>

1 files changed, 524 insertions, 0 deletions

diff --git a/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationApiTest.java b/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationApiTest.java
new file mode 100644
index 000000000..1a0e1c9b7
--- /dev/null
+++ b/vid-automation/src/test/java/org/onap/vid/api/AsyncInstantiationApiTest.java
@@ -0,0 +1,524 @@
+package org.onap.vid.api;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import net.bytebuddy.utility.RandomString;
+import net.javacrumbs.jsonunit.JsonAssert;
+import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.Description;
+import org.onap.simulator.presetGenerator.presets.BasePresets.BasePreset;
+import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetSubscribersGet;
+import org.onap.simulator.presetGenerator.presets.aai.PresetAAISearchNodeQueryEmptyResult;
+import org.onap.simulator.presetGenerator.presets.ecompportal_att.PresetGetSessionSlotCheckIntervalGet;
+import org.onap.simulator.presetGenerator.presets.mso.PresetMSOAssignServiceInstanceGen2WithNames;
+import org.onap.simulator.presetGenerator.presets.mso.PresetMSOCreateServiceInstanceGen2WithNames;
+import org.onap.simulator.presetGenerator.presets.mso.PresetMSOOrchestrationRequestGet;
+import org.onap.simulator.presetGenerator.presets.mso.PresetMSOOrchestrationRequestGetErrorResponse;
+import org.onap.simulator.presetGenerator.presets.mso.PresetMSOServiceInstanceGen2ErrorResponse;
+import org.onap.vid.model.asyncInstantiation.JobAuditStatus;
+import org.onap.vid.model.asyncInstantiation.ServiceInfo;
+import org.onap.vid.model.mso.MsoResponseWrapper2;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.HttpClientErrorException;
+
+import org.springframework.web.client.RestTemplate;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+import vid.automation.test.infra.FeatureTogglingTest;
+import vid.automation.test.infra.Features;
+import vid.automation.test.infra.Wait;
+import vid.automation.test.model.JobStatus;
+import vid.automation.test.services.SimulatorApi;
+import java.util.*;
+
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static java.lang.Boolean.FALSE;
+import static java.lang.Boolean.TRUE;
+import static java.util.stream.Collectors.*;
+import static org.hamcrest.CoreMatchers.*;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasProperty;
+import static org.hamcrest.Matchers.hasSize;
+import static org.onap.simulator.presetGenerator.presets.mso.PresetMSOServiceInstanceGen2WithNames.Keys;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.AssertJUnit.assertTrue;
+
+@FeatureTogglingTest({Features.FLAG_ASYNC_JOBS, Features.FLAG_ASYNC_INSTANTIATION})
+public class AsyncInstantiationApiTest extends BaseMsoApiTest {
+
+    private static final String CREATE_BULK_OF_MACRO_REQUEST = "asyncInstantiation/vidRequestCreateBulkOfMacro.json";
+
+    @DataProvider
+    public static Object[][] trueAndFalse() {
+            return new Object[][]{{TRUE},{FALSE}};
+    }
+
+    private String getCreateBulkUri() {
+        return uri.toASCIIString() + "/asyncInstantiation/bulk";
+    }
+
+    private String getHideServiceUri(String jobId) {
+        return uri.toASCIIString() + "/asyncInstantiation/hide/"+jobId;
+    }
+
+    private String getServiceInfoUrl() {
+        return uri.toASCIIString() + "/asyncInstantiation";
+    }
+
+    private String getJobAuditUrl() {
+        return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}?source={SOURCE}";
+    }
+
+    private String getDeleteServiceUrl(String uuid) {
+        return uri.toASCIIString() + "/asyncInstantiation/job/" + uuid;
+    }
+
+    public static class JobIdAndStatusMatcher extends BaseMatcher<ServiceInfo> {
+        private String expectedJobId;
+
+        public JobIdAndStatusMatcher(String expectedJobId) {
+            this.expectedJobId = expectedJobId;
+        }
+
+        @Override
+        public boolean matches(Object item) {
+            if (!(item instanceof ServiceInfo)) {
+                return false;
+            }
+            ServiceInfo serviceInfo = (ServiceInfo) item;
+            return expectedJobId.equals(serviceInfo.jobId);
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            description.appendText("failed to find job with uuid ")
+                    .appendValue(expectedJobId);
+        }
+    }
+
+
+
+    @Test
+    public void createBulkOfCreateInstances(){
+        Map<Keys, String> names = generateNames();
+        final int bulkSize = 3;
+        ImmutableList<BasePreset> presets = addPresetsForCreateBulkOfCreateInstances(bulkSize, names);
+        createBulkOfInstancesAndAssert(presets, false, bulkSize, JobStatus.COMPLETED, names);
+    }
+
+    private Map<Keys,String> generateNames() {
+        return Stream.of(Keys.values()).collect(
+                Collectors.toMap(x->x, x -> UUID.randomUUID().toString().replace("-","")));
+    }
+
+    private ImmutableList<BasePreset> addPresetsForCreateBulkOfCreateInstances(int bulkSize, Map<Keys, String> names){
+        ImmutableList<BasePreset> msoBulkPresets = IntStream.rangeClosed(1,bulkSize).
+                mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
+                .collect(ImmutableList.toImmutableList());
+        ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
+                .add(new PresetGetSessionSlotCheckIntervalGet())
+                .add(new PresetAAIGetSubscribersGet())
+                .add(new PresetAAISearchNodeQueryEmptyResult())
+                .addAll(msoBulkPresets)
+                .add(new PresetMSOOrchestrationRequestGet())
+                .build();
+        return presets;
+
+    }
+
+    private ResponseEntity<List<JobAuditStatus>> auditStatusCall(String url) {
+        return restTemplate.exchange(
+                url,
+                org.springframework.http.HttpMethod.GET,
+                null,
+                new ParameterizedTypeReference<List<JobAuditStatus>>() {});
+    }
+
+    @DataProvider
+    public static Object[][] auditSources() {
+        return new Object[][]{{JobAuditStatus.SourceStatus.VID},{JobAuditStatus.SourceStatus.MSO}};
+    }
+
+
+    @Test(dataProvider = "auditSources")
+   public void getAuditStatus_nonExistingJobId_returnsEmptyList(JobAuditStatus.SourceStatus source){
+       List<JobAuditStatus> audits = getAuditStatuses(UUID.randomUUID().toString(), source.name());
+       Assert.assertEquals(audits.size(),0);
+   }
+
+    @Test(expectedExceptions = HttpClientErrorException.class)
+    public void getAuditStatus_nonExistingSource_returnsError() {
+        try {
+            getAuditStatuses(UUID.randomUUID().toString(), new RandomString(8).nextString());
+        } catch (HttpClientErrorException e) {
+            Assert.assertEquals(e.getResponseBodyAsString(),"The parameter source must have a value among : MSO, VID");
+            assertThat(e.getStatusCode(), is(HttpStatus.BAD_REQUEST));
+            throw e;
+        }
+    }
+
+    @Test()
+    public void simulateBulkRequest_getAuditStatus_auditStatusesReturnedAccordingSource() {
+        final int bulkSize = 2;
+        final List<String> jobIds = createBulkAndWaitForBeCompleted(bulkSize);
+
+        for(String jobId: jobIds) {
+            List<JobAuditStatus> actualVidAudits = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name());
+            List<JobAuditStatus> expectedVidAudits = Stream.of(JobStatus.PENDING, JobStatus.IN_PROGRESS, JobStatus.COMPLETED)
+                    .map(status->new JobAuditStatus(UUID.fromString(jobId),
+                            status.name(),
+                            JobAuditStatus.SourceStatus.VID,
+                            null,
+                            null,
+                            status.equals(JobStatus.COMPLETED))).collect(toList());
+            assertThat(actualVidAudits, is(expectedVidAudits));
+
+            List<JobAuditStatus> actualMsoAudits = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
+            List<JobAuditStatus> expectedMsoAudits = Stream.of("REQUESTED", "COMPLETE")
+                    .map(status-> new JobAuditStatus(UUID.fromString(jobId),
+                            status,
+                            JobAuditStatus.SourceStatus.MSO,
+                            UUID.fromString("c0011670-0e1a-4b74-945d-8bf5aede1d9c"),
+                            status.equals("COMPLETE") ? "Service Instance was created successfully." : null,
+                            false)).collect(toList());
+            assertThat(actualMsoAudits, is(expectedMsoAudits));
+        }
+    }
+
+    protected List<String> createBulkAndWaitForBeCompleted(int bulkSize){
+        Map<Keys, String> names = generateNames();
+        ImmutableList<BasePreset> presets = addPresetsForCreateBulkOfCreateInstances(bulkSize, names);
+        final List<String> jobIds = createBulkOfInstances(presets, false, bulkSize, names);
+        Assert.assertEquals(jobIds.size(),bulkSize);
+
+        assertTrue(String.format("Not all services with ids: %s are in state completed after 30 sec",
+                jobIds.stream().collect(joining(","))),
+
+                Wait.waitFor(y-> serviceListCall().getBody().stream()
+                        .filter(si -> jobIds.contains(si.jobId))
+                        .filter(si -> si.jobStatus==JobStatus.COMPLETED)
+                        .count() == bulkSize,
+                null, 30, 1 ));
+        return jobIds;
+    }
+
+    private List<JobAuditStatus> getAuditStatuses(String jobUUID, String source){
+        String url = getJobAuditUrl().replace("{JOB_ID}",jobUUID).replace("{SOURCE}", source);
+        ResponseEntity<List<JobAuditStatus>> statusesResponse = auditStatusCall(url);
+        assertThat(statusesResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
+        return statusesResponse.getBody();
+    }
+
+    @Test(expectedExceptions = HttpClientErrorException.class)
+    public void addBulkAndDeleteInProgress_deletionIsRejected(){
+        try {
+            final Map<String, JobStatus> jobs = addBulkAllPendingButOneInProgress();
+            deleteOneJobHavingTheStatus(jobs, JobStatus.IN_PROGRESS);
+        } catch (HttpClientErrorException e) {
+            JsonAssert.assertJsonPartEquals(
+                    "Service status does not allow deletion from the queue (Request id: null)",
+                    e.getResponseBodyAsString(),
+                    "message"
+            );
+            assertThat(e.getStatusCode(), is(HttpStatus.METHOD_NOT_ALLOWED));
+
+            throw e;
+        }
+    }
+
+    @Test
+    public void addBulkAndDeletePending_deletedIsHiddenFromServiceInfoResults(){
+        Map<String, JobStatus> statusesBefore = addBulkAllPendingButOneInProgress();
+
+        final String deletedJob = deleteOneJobHavingTheStatus(statusesBefore, JobStatus.PENDING);
+
+        final Map<String, JobStatus> statusesNow = serviceListCall().getBody().stream()
+                .filter(si -> statusesBefore.keySet().contains(si.jobId))
+                .collect(toMap(si -> si.jobId, si -> si.jobStatus));
+
+        statusesBefore.remove(deletedJob);
+        assertThat("deleted job shall not be present in StatusInfo response", statusesNow, is(statusesBefore));
+    }
+
+    private Map<String, JobStatus> addBulkAllPendingButOneInProgress(){
+        return addBulkAllPendingButOneInProgress(3);
+    }
+    
+    private Map<String, JobStatus> addBulkAllPendingButOneInProgress(int bulkSize){
+        Map<Keys, String> names = generateNames();
+        ImmutableList<BasePreset> msoBulkPresets = IntStream.rangeClosed(1,bulkSize)
+                .mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
+                .collect(ImmutableList.toImmutableList());
+        ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
+                .add(new PresetGetSessionSlotCheckIntervalGet())
+                .add(new PresetAAISearchNodeQueryEmptyResult())
+                .add(new PresetAAIGetSubscribersGet())
+                .addAll(msoBulkPresets)
+                .add(new PresetMSOOrchestrationRequestGet("IN_PROGRESS"))
+                .build();
+        final List<String> jobIds = createBulkOfInstances(presets, false, bulkSize, names);
+
+        // wait for single IN_PROGRESS, so statuses will stop from changing
+        Wait.waitFor(foo -> serviceListCall().getBody().stream()
+                        .filter(si -> jobIds.contains(si.jobId))
+                        .anyMatch(si -> si.jobStatus.equals(JobStatus.IN_PROGRESS)),
+                null, 20, 1);
+
+        final Map<String, JobStatus> statusMapBefore = serviceListCall().getBody().stream()
+                .filter(si -> jobIds.contains(si.jobId))
+                .collect(toMap(si -> si.jobId, si -> si.jobStatus));
+
+        assertThat(jobIds, hasSize(bulkSize));
+
+
+        return statusMapBefore;
+    }
+
+    private String deleteOneJobHavingTheStatus(Map<String, JobStatus> jobIdToStatus, JobStatus jobStatus) {
+        final String jobToDelete = jobIdToStatus.entrySet().stream()
+                .filter(entry -> entry.getValue().equals(jobStatus))
+                .map(Map.Entry::getKey)
+                .findFirst().orElseThrow(() -> new AssertionError("no job in " + jobStatus + " state: " + jobIdToStatus));
+
+
+        restTemplate.delete(getDeleteServiceUrl(jobToDelete));
+
+        return jobToDelete;
+    }
+
+    @Test(invocationCount = 3)
+    public void createBulkOfCreateInstancesWithSinglePreset_firstOneInProgressOtherArePending(){
+        final int bulkSize = 3;
+        Map<String, JobStatus> statusMap = addBulkAllPendingButOneInProgress(bulkSize);
+        Set<String> jobIds = statusMap.keySet();
+
+        final Map<JobStatus, List<ServiceInfo>> statuses = serviceListCall().getBody().stream()
+                .filter(si -> jobIds.contains(si.jobId))
+                .collect(groupingBy(ServiceInfo::getJobStatus));
+
+        // Variable "statuses" contains two lists by status:
+        // IN_PROGRESS:  The ultimate first job - named with _001 - is always the only one in progress
+        // PENDING:      The other two jobs - named with _002 and _003 - are the still pending
+        assertThat(jobIds, hasSize(bulkSize));
+        assertThat(statuses.get(JobStatus.IN_PROGRESS), hasSize(1));
+        assertThat(statuses.get(JobStatus.IN_PROGRESS), everyItem(hasProperty("serviceInstanceName", endsWith("_001"))));
+
+        assertThat(statuses.get(JobStatus.PENDING), hasSize(bulkSize - 1));
+    }
+
+
+    @Test(dataProvider = "trueAndFalse" )
+    public void whenServiceInBulkFailed_otherServicesAreStopped(Boolean isPresetForCreate){
+        Map<Keys, String> names = generateNames();
+        final int bulkSize = 3;
+
+        //if there is a preset for create,  service shall failed during in_progress (upon get status)
+        //it there is no preset for create, service shall failed during pending (upon create request)
+        List<BasePreset> msoBulkPresets = isPresetForCreate ?
+                IntStream.rangeClosed(1,bulkSize)
+                        .mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
+                        .collect(ImmutableList.toImmutableList()) :
+                new LinkedList<>();
+        ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
+                .add(new PresetGetSessionSlotCheckIntervalGet())
+                .add(new PresetAAIGetSubscribersGet())
+                .add(new PresetAAISearchNodeQueryEmptyResult())
+                .addAll(msoBulkPresets)
+                .add(new PresetMSOOrchestrationRequestGet("FAILED"))
+                .build();
+        List<String> jobIds = createBulkOfInstances(presets, false, bulkSize, names);
+        Assert.assertEquals(jobIds.size(),bulkSize);
+        boolean result = Wait.waitFor(x->{
+                List<ServiceInfo> serviceInfoList = serviceListCall().getBody();
+                Map<JobStatus, Long> statusCount = serviceInfoList.stream().filter(si->jobIds.contains(si.jobId)).collect(groupingBy(ServiceInfo::getJobStatus, counting()));
+                return Objects.equals(statusCount.get(JobStatus.FAILED), 1L) && Objects.equals(statusCount.get(JobStatus.STOPPED), 2L);
+            }, null, 15, 1);
+        assertTrue(String.format("failed to get jobs [%s] to state of: 1 failed and 2 stopped ",
+                String.join(",", jobIds)),result);
+    }
+
+    @Test
+    public void createBulkOfAssignInstances(){
+        Map<Keys, String> names = generateNames();
+        final int bulkSize = 2;
+        ImmutableList<BasePreset> msoBulkPresets = IntStream.rangeClosed(1,bulkSize)
+                .mapToObj(i-> new PresetMSOAssignServiceInstanceGen2WithNames(names, i))
+                .collect(ImmutableList.toImmutableList());
+        ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
+                .add(new PresetGetSessionSlotCheckIntervalGet())
+                .add(new PresetAAIGetSubscribersGet())
+                .add(new PresetAAISearchNodeQueryEmptyResult())
+                .addAll(msoBulkPresets)
+                .add(new PresetMSOOrchestrationRequestGet())
+                .build();
+        createBulkOfInstancesAndAssert(presets, true, bulkSize, JobStatus.COMPLETED, names);
+    }  
+
+    @Test
+    public void tryToCreateBulkOfAssignInstancesErrorResponseFromMso(){
+        ImmutableList<BasePreset> presets = ImmutableList.of(
+                new PresetGetSessionSlotCheckIntervalGet(),
+                new PresetAAIGetSubscribersGet(),
+                new PresetAAISearchNodeQueryEmptyResult(),
+                new PresetMSOServiceInstanceGen2ErrorResponse(406));
+
+        List<String> jobIds = createBulkOfInstancesAndAssert(presets, true,1, JobStatus.FAILED, generateNames());
+        String jobId  = jobIds.get(0);
+        List<JobAuditStatus> actualMsoAudits = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
+        JobAuditStatus expectedMsoAudit = new JobAuditStatus(UUID.fromString(jobId),"FAILED",JobAuditStatus.SourceStatus.MSO,
+                        null,
+                        "Http Code:406, \"messageId\":\"SVC0002\",\"text\":\"JSON Object Mapping Request\"" ,
+                        false);
+        assertThat(actualMsoAudits.get(0), is(expectedMsoAudit));
+    }
+
+    @Test
+    public void whenHideCompletedService_thenServiceNotReturnInServiceList(){
+        List<String> services = createBulkAndWaitForBeCompleted(2);
+        hideService(services.get(0));
+        List<String> serviceInfoList = serviceListCall().getBody().stream().map(ServiceInfo::getJobId).collect(toList());
+        assertThat(serviceInfoList, hasItem(services.get(1)));
+        assertThat(serviceInfoList, not(hasItem(services.get(0))));
+    }
+
+    private MsoResponseWrapper2 hideService(String jobId) {
+        MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getHideServiceUri(jobId), "");
+        return responseWrapper2;
+    }
+
+    private List<String> createBulkOfInstancesAndAssert(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, JobStatus finalState, Map<Keys, String> names){
+        List<String> jobIds = createBulkOfInstances(presets, isPause, bulkSize, names);
+        Assert.assertEquals(jobIds.size(),bulkSize);
+        for(String jobId: jobIds) {
+            ServiceInfo expectedServiceInfo = new ServiceInfo("ab2222", JobStatus.IN_PROGRESS, isPause, "someID",
+                    "someName", "myProject", "NFT1", "NFTJSSSS-NFT1", "greatTenant", "greatTenant", "mtn3", null,
+                    "mySubType", "a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb", null, names.get(Keys.SERVICE_NAME),
+                    "300adb1e-9b0c-4d52-bfb5-fa5393c4eabb", "AIM_TRANSPORT_00004", "1.0", jobId, null);
+            JobInfoChecker jobInfoChecker = new JobInfoChecker(
+                    restTemplate, ImmutableSet.of(JobStatus.PENDING, JobStatus.IN_PROGRESS, finalState), jobId, expectedServiceInfo);
+            boolean result = jobInfoChecker.test(null);
+            assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
+
+            jobInfoChecker.setExpectedJobStatus(ImmutableSet.of(finalState));
+            if (ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE).contains(finalState)) {
+                expectedServiceInfo.serviceInstanceId = "f8791436-8d55-4fde-b4d5-72dd2cf13cfb";
+            }
+            result = Wait.waitFor(jobInfoChecker, null, 20, 1);
+            assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
+        }
+
+        return jobIds;
+    }
+
+    private List<String> createBulkOfInstances(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, Map<Keys, String> names){
+
+        SimulatorApi.registerExpectationFromPresets(presets, SimulatorApi.RegistrationStrategy.CLEAR_THEN_SET);
+
+        String requestBody = TestUtils.convertRequest(objectMapper, CREATE_BULK_OF_MACRO_REQUEST);
+        requestBody = requestBody.replace("\"IS_PAUSE_VALUE\"", String.valueOf(isPause)).replace("\"BULK_SIZE\"", String.valueOf(bulkSize));
+        for (Map.Entry<Keys, String> e : names.entrySet()) {
+            requestBody = requestBody.replace(e.getKey().name(), e.getValue());
+        }
+        MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getCreateBulkUri(), requestBody);
+        assertNotNull(responseWrapper2);
+        return (List<String>)responseWrapper2.getEntity();
+    }
+
+    public class JobInfoChecker<Integer> implements Predicate<Integer> {
+
+        private final RestTemplate restTemplate;
+        private Set<JobStatus> expectedJobStatus;
+        private ServiceInfo expectedServiceInfo;
+        private final String jobId;
+        private JobStatus lastStatus;
+
+        public JobInfoChecker(RestTemplate restTemplate, Set<JobStatus> expectedJobStatus, String jobId, ServiceInfo expectedServiceInfo) {
+            this.restTemplate = restTemplate;
+            this.expectedJobStatus = expectedJobStatus;
+            this.jobId = jobId;
+            this.expectedServiceInfo = expectedServiceInfo;
+        }
+
+        public void setExpectedJobStatus(Set<JobStatus> expectedJobStatus) {
+            this.expectedJobStatus = expectedJobStatus;
+        }
+
+        @Override
+        public boolean test(Integer integer) {
+            ResponseEntity<List<ServiceInfo>> serviceListResponse = serviceListCall();
+            assertThat(serviceListResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
+            assertThat(serviceListResponse.getBody(), hasItem(new JobIdAndStatusMatcher(jobId)));
+            ServiceInfo serviceInfoFromDB = serviceListResponse.getBody().stream()
+                    .filter(serviceInfo -> serviceInfo.jobId.equals(jobId))
+                    .findFirst().orElse(null);
+            Assert.assertNotNull(serviceInfoFromDB);
+            Assert.assertEquals(serviceInfoDataReflected(serviceInfoFromDB), serviceInfoDataReflected(expectedServiceInfo));
+            assertTrue("actual service instance doesn't contain template service name:" + expectedServiceInfo.serviceInstanceName,
+                    serviceInfoFromDB.serviceInstanceName.contains(expectedServiceInfo.serviceInstanceName));
+            if (serviceInfoFromDB.jobStatus==JobStatus.IN_PROGRESS || serviceInfoFromDB.jobStatus==JobStatus.COMPLETED) {
+                assertTrue("actual service instance doesn't contain template service name and trailing numbers:" + expectedServiceInfo.serviceInstanceName,
+                        serviceInfoFromDB.serviceInstanceName.contains(expectedServiceInfo.serviceInstanceName+"_00"));
+            }
+
+            if (expectedServiceInfo.serviceInstanceId != null) {
+                assertThat(serviceInfoFromDB.serviceInstanceId, is(expectedServiceInfo.serviceInstanceId));
+            }
+            lastStatus = serviceInfoFromDB.jobStatus;
+            return expectedJobStatus.contains(serviceInfoFromDB.jobStatus);
+        }
+    }
+
+    private ResponseEntity<List<ServiceInfo>> serviceListCall() {
+        return restTemplate.exchange(
+                getServiceInfoUrl(),
+                org.springframework.http.HttpMethod.GET,
+                null,
+                new ParameterizedTypeReference<List<ServiceInfo>>() {});
+    }
+
+    //serialize fields except of fields we cannot know ahead of time
+    private static String serviceInfoDataReflected(ServiceInfo service1) {
+        return new ReflectionToStringBuilder(service1, ToStringStyle.SHORT_PREFIX_STYLE)
+                .setExcludeFieldNames("jobStatus", "templateId", "statusModifiedDate", "createdBulkDate", "serviceInstanceId", "serviceInstanceName")
+                .toString();
+    }
+
+    @Test
+    public void errorResponseInGetStatusFromMso_getAuditStatusFromMso_errorMsgExistInAdditionalInfo(){
+        Map<Keys, String> names = generateNames();
+        ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
+                .add(new PresetGetSessionSlotCheckIntervalGet())
+                .add(new PresetAAIGetSubscribersGet())
+                .add(new PresetAAISearchNodeQueryEmptyResult())
+                .add(new PresetMSOAssignServiceInstanceGen2WithNames(names, 1))
+                .add(new PresetMSOOrchestrationRequestGetErrorResponse(406))
+                .build();
+
+        final List<String> jobIds = createBulkOfInstancesAndAssert(presets, true,1, JobStatus.IN_PROGRESS, names);
+        String jobId = jobIds.get(0);
+        Wait.waitFor(y-> getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name()).stream()
+                        .anyMatch(si -> si.getJobStatus().equals("FAILED")),
+                null, 10, 1 );
+        List<JobAuditStatus> actualMsoAudits = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
+        List<JobAuditStatus> expectedMsoAudits = Stream.of("REQUESTED", "FAILED")
+                .map(status -> new JobAuditStatus(UUID.fromString(jobId),
+                        status,
+                        JobAuditStatus.SourceStatus.MSO,
+                        UUID.fromString("c0011670-0e1a-4b74-945d-8bf5aede1d9c"),
+                        status.equals("FAILED") ? "Http Code:406, \"messageId\":\"SVC0002\",\"text\":\"JSON Object Mapping Request\"" : null,
+                        false)).collect(toList());
+        assertThat(actualMsoAudits, is(expectedMsoAudits));
+
+    }
+
+}