diff options
27 files changed, 818 insertions, 192 deletions
diff --git a/cps-application/src/main/resources/application.yml b/cps-application/src/main/resources/application.yml index 9c8c1ecd5..4f08bb61a 100644 --- a/cps-application/src/main/resources/application.yml +++ b/cps-application/src/main/resources/application.yml @@ -189,11 +189,18 @@ logging: ncmp: dmi: httpclient: - connectionTimeoutInSeconds: 30 - readTimeoutInSeconds: 30 - writeTimeoutInSeconds: 30 - maximumConnectionsTotal: 100 - maximumInMemorySizeInMegabytes: 16 + data-services: + connectionTimeoutInSeconds: 30 + readTimeoutInSeconds: 30 + writeTimeoutInSeconds: 30 + maximumConnectionsTotal: 100 + maximumInMemorySizeInMegabytes: 16 + model-services: + connectionTimeoutInSeconds: 30 + readTimeoutInSeconds: 30 + writeTimeoutInSeconds: 30 + maximumConnectionsTotal: 100 + maximumInMemorySizeInMegabytes: 16 auth: username: ${DMI_USERNAME:cpsuser} password: ${DMI_PASSWORD:cpsr0cks!} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/client/DmiRestClient.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/client/DmiRestClient.java index ef48c43d2..7878c5d0b 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/client/DmiRestClient.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/client/DmiRestClient.java @@ -26,6 +26,7 @@ import static org.onap.cps.ncmp.api.NcmpResponseStatus.UNABLE_TO_READ_RESOURCE_D import static org.onap.cps.ncmp.api.NcmpResponseStatus.UNKNOWN_ERROR; import static org.springframework.http.HttpStatus.BAD_REQUEST; import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; +import static org.springframework.http.HttpStatus.REQUEST_TIMEOUT; import com.fasterxml.jackson.databind.JsonNode; import java.net.URI; @@ -37,7 +38,9 @@ import org.onap.cps.ncmp.api.impl.config.DmiProperties; import org.onap.cps.ncmp.api.impl.exception.DmiClientRequestException; import org.onap.cps.ncmp.api.impl.exception.InvalidDmiResourceUrlException; import org.onap.cps.ncmp.api.impl.operations.OperationType; +import org.onap.cps.ncmp.api.impl.operations.RequiredDmiService; import org.onap.cps.utils.JsonObjectMapper; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -45,6 +48,7 @@ import org.springframework.stereotype.Component; import org.springframework.web.client.HttpServerErrorException; import org.springframework.web.reactive.function.BodyInserters; import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; import org.springframework.web.reactive.function.client.WebClientResponseException; @Component @@ -55,31 +59,43 @@ public class DmiRestClient { private static final String HEALTH_CHECK_URL_EXTENSION = "/actuator/health"; private static final String NOT_SPECIFIED = ""; private static final String NO_AUTHORIZATION = null; - private final WebClient webClient; + private final DmiProperties dmiProperties; private final JsonObjectMapper jsonObjectMapper; + @Qualifier("dataServicesWebClient") + private final WebClient dataServicesWebClient; + @Qualifier("modelServicesWebClient") + private final WebClient modelServicesWebClient; + @Qualifier("healthChecksWebClient") + private final WebClient healthChecksWebClient; /** - * Sends POST operation to DMI with json body containing module references. + * Sends a POST operation to the DMI with a JSON body containing module references. * - * @param dmiResourceUrl dmi resource url - * @param requestBodyAsJsonString json data body - * @param operationType the type of operation being executed (for error reporting only) - * @param authorization contents of Authorization header, or null if not present - * @return response entity of type String + * @param requiredDmiService Determines if the required service is for a data or model operation. + * @param dmiUrl The DMI resource URL. + * @param requestBodyAsJsonString JSON data body. + * @param operationType The type of operation being executed (for error reporting only). + * @param authorization Contents of the Authorization header, or null if not present. + * @return ResponseEntity containing the response from the DMI. + * @throws DmiClientRequestException If there is an error during the DMI request. */ - public ResponseEntity<Object> postOperationWithJsonData(final String dmiResourceUrl, + public ResponseEntity<Object> postOperationWithJsonData(final RequiredDmiService requiredDmiService, + final String dmiUrl, final String requestBodyAsJsonString, final OperationType operationType, final String authorization) { + final WebClient webClient = requiredDmiService.equals(RequiredDmiService.DATA) + ? dataServicesWebClient : modelServicesWebClient; try { - return ResponseEntity.ok(webClient.post().uri(toUri(dmiResourceUrl)) + return webClient.post() + .uri(toUri(dmiUrl)) .headers(httpHeaders -> configureHttpHeaders(httpHeaders, authorization)) .body(BodyInserters.fromValue(requestBodyAsJsonString)) .retrieve() - .bodyToMono(Object.class) + .toEntity(Object.class) .onErrorMap(httpError -> handleDmiClientException(httpError, operationType.getOperationName())) - .block()); + .block(); } catch (final HttpServerErrorException e) { throw handleDmiClientException(e, operationType.getOperationName()); } @@ -88,31 +104,31 @@ public class DmiRestClient { /** * Get DMI plugin health status. * - * @param dmiPluginBaseUrl the base URL of the dmi-plugin - * @return plugin health status ("UP" is all OK, "" (not-specified) in case of any exception) + * @param dmiUrl the base URL of the dmi-plugin + * @return plugin health status ("UP" is all OK, "" (not-specified) in case of any exception) */ - public String getDmiHealthStatus(final String dmiPluginBaseUrl) { + public String getDmiHealthStatus(final String dmiUrl) { try { - final JsonNode responseHealthStatus = webClient.get() - .uri(toUri(dmiPluginBaseUrl + HEALTH_CHECK_URL_EXTENSION)) + final URI dmiHealthCheckUri = toUri(dmiUrl + HEALTH_CHECK_URL_EXTENSION); + final JsonNode responseHealthStatus = healthChecksWebClient.get() + .uri(dmiHealthCheckUri) .headers(httpHeaders -> configureHttpHeaders(httpHeaders, NO_AUTHORIZATION)) .retrieve() .bodyToMono(JsonNode.class).block(); return responseHealthStatus == null ? NOT_SPECIFIED : - responseHealthStatus.get("status").asText(); + responseHealthStatus.path("status").asText(); } catch (final Exception e) { - log.warn("Failed to retrieve health status from {}. Error Message: {}", dmiPluginBaseUrl, e.getMessage()); + log.warn("Failed to retrieve health status from {}. Error Message: {}", dmiUrl, e.getMessage()); return NOT_SPECIFIED; } } - private HttpHeaders configureHttpHeaders(final HttpHeaders httpHeaders, final String authorization) { + private void configureHttpHeaders(final HttpHeaders httpHeaders, final String authorization) { if (dmiProperties.isDmiBasicAuthEnabled()) { httpHeaders.setBasicAuth(dmiProperties.getAuthUsername(), dmiProperties.getAuthPassword()); } else if (authorization != null && authorization.toLowerCase(Locale.getDefault()).startsWith("bearer ")) { httpHeaders.add(HttpHeaders.AUTHORIZATION, authorization); } - return httpHeaders; } private static URI toUri(final String dmiResourceUrl) { @@ -123,22 +139,31 @@ public class DmiRestClient { } } - private DmiClientRequestException handleDmiClientException(final Throwable e, final String operationType) { - final String exceptionMessage = "Unable to " + operationType + " resource data."; - if (e instanceof WebClientResponseException wcre) { - if (wcre.getStatusCode().isSameCodeAs(HttpStatus.REQUEST_TIMEOUT)) { - throw new DmiClientRequestException(wcre.getStatusCode().value(), wcre.getMessage(), - jsonObjectMapper.asJsonString(wcre.getResponseBodyAsString()), DMI_SERVICE_NOT_RESPONDING); + private DmiClientRequestException handleDmiClientException(final Throwable throwable, final String operationType) { + if (throwable instanceof WebClientResponseException webClientResponseException) { + if (webClientResponseException.getStatusCode().isSameCodeAs(REQUEST_TIMEOUT)) { + throw new DmiClientRequestException(webClientResponseException.getStatusCode().value(), + webClientResponseException.getMessage(), + jsonObjectMapper.asJsonString(webClientResponseException.getResponseBodyAsString()), + DMI_SERVICE_NOT_RESPONDING); } - throw new DmiClientRequestException(wcre.getStatusCode().value(), wcre.getMessage(), - jsonObjectMapper.asJsonString(wcre.getResponseBodyAsString()), UNABLE_TO_READ_RESOURCE_DATA); + throw new DmiClientRequestException(webClientResponseException.getStatusCode().value(), + webClientResponseException.getMessage(), + jsonObjectMapper.asJsonString(webClientResponseException.getResponseBodyAsString()), + UNABLE_TO_READ_RESOURCE_DATA); } - if (e instanceof HttpServerErrorException httpServerErrorException) { + final String exceptionMessage = "Unable to " + operationType + " resource data."; + if (throwable instanceof WebClientRequestException webClientRequestException) { + throw new DmiClientRequestException(HttpStatus.SERVICE_UNAVAILABLE.value(), + webClientRequestException.getMessage(), + exceptionMessage, DMI_SERVICE_NOT_RESPONDING); + } + if (throwable instanceof HttpServerErrorException httpServerErrorException) { throw new DmiClientRequestException(httpServerErrorException.getStatusCode().value(), exceptionMessage, httpServerErrorException.getResponseBodyAsString(), DMI_SERVICE_NOT_RESPONDING); } - throw new DmiClientRequestException(INTERNAL_SERVER_ERROR.value(), exceptionMessage, e.getMessage(), + throw new DmiClientRequestException(INTERNAL_SERVER_ERROR.value(), exceptionMessage, throwable.getMessage(), UNKNOWN_ERROR); } } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/DmiWebClientConfiguration.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/DmiWebClientConfiguration.java index 2e84f7f69..2c0b70262 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/DmiWebClientConfiguration.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/DmiWebClientConfiguration.java @@ -34,6 +34,10 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.netty.http.client.HttpClient; import reactor.netty.resources.ConnectionProvider; +/** + * Configures and creates a WebClient bean that triggers an initialization (warmup) of the host name resolver and + * loads the necessary native libraries to avoid the extra time needed to load resources for first request. + */ @Configuration @RequiredArgsConstructor public class DmiWebClientConfiguration { @@ -41,30 +45,83 @@ public class DmiWebClientConfiguration { private final HttpClientConfiguration httpClientConfiguration; /** - * Configures and create a WebClient bean that triggers an initialization (warmup) of the host name resolver and - * loads the necessary native libraries to avoid the extra time needed to load resources for first request. + * Configures and create a WebClient bean for DMI data service. + * + * @return a WebClient instance for data services. + */ + @Bean + public WebClient dataServicesWebClient() { + final HttpClientConfiguration.DataServices httpClientConfiguration + = this.httpClientConfiguration.getDataServices(); + + final HttpClient httpClient = createHttpClient("dataConnectionPool", + httpClientConfiguration.getMaximumConnectionsTotal(), + httpClientConfiguration.getConnectionTimeoutInSeconds(), + httpClientConfiguration.getReadTimeoutInSeconds(), + httpClientConfiguration.getWriteTimeoutInSeconds()); + return buildAndGetWebClient(httpClient, httpClientConfiguration.getMaximumInMemorySizeInMegabytes()); + } + + /** + * Configures and creates a WebClient bean for DMI model service. + * + * @return a WebClient instance for model services. + */ + @Bean + public WebClient modelServicesWebClient() { + final HttpClientConfiguration.ModelServices httpClientConfiguration + = this.httpClientConfiguration.getModelServices(); + + final HttpClient httpClient = createHttpClient("modelConnectionPool", + httpClientConfiguration.getMaximumConnectionsTotal(), + httpClientConfiguration.getConnectionTimeoutInSeconds(), + httpClientConfiguration.getReadTimeoutInSeconds(), + httpClientConfiguration.getWriteTimeoutInSeconds()); + return buildAndGetWebClient(httpClient, httpClientConfiguration.getMaximumInMemorySizeInMegabytes()); + } + + /** + * Configures and creates a WebClient bean for DMI health service. * - * @return a WebClient instance. + * @return a WebClient instance for health checks. */ @Bean - public WebClient webClient() { - final ConnectionProvider dmiWebClientConnectionProvider = ConnectionProvider.create( - "dmiWebClientConnectionPool", httpClientConfiguration.getMaximumConnectionsTotal()); + public WebClient healthChecksWebClient() { + final HttpClientConfiguration.HealthCheckServices httpClientConfiguration + = this.httpClientConfiguration.getHealthCheckServices(); + + final HttpClient httpClient = createHttpClient("healthConnectionPool", + httpClientConfiguration.getMaximumConnectionsTotal(), + httpClientConfiguration.getConnectionTimeoutInSeconds(), + httpClientConfiguration.getReadTimeoutInSeconds(), + httpClientConfiguration.getWriteTimeoutInSeconds()); + return buildAndGetWebClient(httpClient, httpClientConfiguration.getMaximumInMemorySizeInMegabytes()); + } + + private static HttpClient createHttpClient(final String connectionProviderName, + final Integer maximumConnectionsTotal, + final Integer connectionTimeoutInSeconds, + final Integer readTimeoutInSeconds, + final Integer writeTimeoutInSeconds) { + final ConnectionProvider dmiWebClientConnectionProvider = ConnectionProvider.create(connectionProviderName, + maximumConnectionsTotal); final HttpClient httpClient = HttpClient.create(dmiWebClientConnectionProvider) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, - httpClientConfiguration.getConnectionTimeoutInSeconds() * 1000) - .doOnConnected(connection -> connection.addHandlerLast(new ReadTimeoutHandler( - httpClientConfiguration.getReadTimeoutInSeconds(), TimeUnit.SECONDS)) - .addHandlerLast(new WriteTimeoutHandler( - httpClientConfiguration.getWriteTimeoutInSeconds(), TimeUnit.SECONDS))); + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, connectionTimeoutInSeconds * 1000) + .doOnConnected(connection -> connection.addHandlerLast(new ReadTimeoutHandler(readTimeoutInSeconds, + TimeUnit.SECONDS)).addHandlerLast(new WriteTimeoutHandler(writeTimeoutInSeconds, + TimeUnit.SECONDS))); httpClient.warmup().block(); + return httpClient; + } + + private static WebClient buildAndGetWebClient(final HttpClient httpClient, + final Integer maximumInMemorySizeInMegabytes) { return WebClient.builder() .defaultHeaders(header -> header.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)) .defaultHeaders(header -> header.set(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)) .clientConnector(new ReactorClientHttpConnector(httpClient)) .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize( - httpClientConfiguration.getMaximumInMemorySizeInMegabytes() * 1024 * 1024)) - .build(); + maximumInMemorySizeInMegabytes * 1024 * 1024)).build(); } } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/HttpClientConfiguration.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/HttpClientConfiguration.java index d2521d9d1..62432f6ca 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/HttpClientConfiguration.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/HttpClientConfiguration.java @@ -30,9 +30,37 @@ import org.springframework.stereotype.Component; @Component @ConfigurationProperties(prefix = "ncmp.dmi.httpclient") public class HttpClientConfiguration { - private Integer maximumConnectionsTotal = 100; - private Integer connectionTimeoutInSeconds = 30; - private Integer readTimeoutInSeconds = 30; - private Integer writeTimeoutInSeconds = 30; - private Integer maximumInMemorySizeInMegabytes = 1; + + private final DataServices dataServices = new DataServices(); + private final ModelServices modelServices = new ModelServices(); + private final HealthCheckServices healthCheckServices = new HealthCheckServices(); + + @Getter + @Setter + public static class DataServices { + private Integer maximumConnectionsTotal = 100; + private Integer connectionTimeoutInSeconds = 30; + private Integer readTimeoutInSeconds = 30; + private Integer writeTimeoutInSeconds = 30; + private Integer maximumInMemorySizeInMegabytes = 1; + } + + @Getter + @Setter + public static class ModelServices { + private Integer maximumConnectionsTotal = 100; + private Integer connectionTimeoutInSeconds = 30; + private Integer readTimeoutInSeconds = 30; + private Integer writeTimeoutInSeconds = 30; + private Integer maximumInMemorySizeInMegabytes = 1; + } + + @Getter + public static class HealthCheckServices { + private final Integer maximumConnectionsTotal = 10; + private final Integer connectionTimeoutInSeconds = 30; + private final Integer readTimeoutInSeconds = 30; + private final Integer writeTimeoutInSeconds = 30; + private final Integer maximumInMemorySizeInMegabytes = 1; + } } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/operations/DmiDataOperations.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/operations/DmiDataOperations.java index 637087909..978855569 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/operations/DmiDataOperations.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/operations/DmiDataOperations.java @@ -23,6 +23,7 @@ package org.onap.cps.ncmp.api.impl.operations; import static org.onap.cps.ncmp.api.impl.operations.DatastoreType.PASSTHROUGH_RUNNING; import static org.onap.cps.ncmp.api.impl.operations.OperationType.READ; +import static org.onap.cps.ncmp.api.impl.operations.RequiredDmiService.DATA; import io.micrometer.core.annotation.Timed; import java.util.Collection; @@ -87,13 +88,9 @@ public class DmiDataOperations { final CmHandleState cmHandleState = yangModelCmHandle.getCompositeState().getCmHandleState(); validateIfCmHandleStateReady(yangModelCmHandle, cmHandleState); final String jsonRequestBody = getDmiRequestBody(READ, requestId, null, null, yangModelCmHandle); - - final String dmiUrl = getDmiResourceDataUrl(cmResourceAddress.datastoreName(), - yangModelCmHandle, - cmResourceAddress.resourceIdentifier(), - optionsParamInQuery, - topicParamInQuery); - return dmiRestClient.postOperationWithJsonData(dmiUrl, jsonRequestBody, READ, authorization); + final String dmiUrl = getDmiResourceDataUrl(cmResourceAddress.datastoreName(), yangModelCmHandle, + cmResourceAddress.resourceIdentifier(), optionsParamInQuery, topicParamInQuery); + return dmiRestClient.postOperationWithJsonData(DATA, dmiUrl, jsonRequestBody, READ, authorization); } /** @@ -114,7 +111,7 @@ public class DmiDataOperations { final String jsonRequestBody = getDmiRequestBody(READ, requestId, null, null, yangModelCmHandle); final String dmiUrl = getDmiResourceDataUrl(datastoreName, yangModelCmHandle, "/", null, null); - return dmiRestClient.postOperationWithJsonData(dmiUrl, jsonRequestBody, READ, null); + return dmiRestClient.postOperationWithJsonData(DATA, dmiUrl, jsonRequestBody, READ, null); } /** @@ -171,7 +168,7 @@ public class DmiDataOperations { yangModelCmHandle); final String dmiUrl = getDmiResourceDataUrl(PASSTHROUGH_RUNNING.getDatastoreName(), yangModelCmHandle, resourceId, null, null); - return dmiRestClient.postOperationWithJsonData(dmiUrl, jsonRequestBody, operationType, authorization); + return dmiRestClient.postOperationWithJsonData(DATA, dmiUrl, jsonRequestBody, operationType, authorization); } private YangModelCmHandle getYangModelCmHandle(final String cmHandleId) { @@ -251,7 +248,8 @@ public class DmiDataOperations { .operations(dmiDataOperationRequestBodies).build(); final String dmiDataOperationRequestAsJsonString = jsonObjectMapper.asJsonString(dmiDataOperationRequest); try { - dmiRestClient.postOperationWithJsonData(dmiUrl, dmiDataOperationRequestAsJsonString, READ, authorization); + dmiRestClient.postOperationWithJsonData(DATA, dmiUrl, dmiDataOperationRequestAsJsonString, READ, + authorization); } catch (final DmiClientRequestException e) { handleTaskCompletionException(e, dmiUrl, dmiDataOperationRequestBodies); } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/operations/DmiModelOperations.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/operations/DmiModelOperations.java index 78d27b54b..77dfcb7a2 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/operations/DmiModelOperations.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/operations/DmiModelOperations.java @@ -111,7 +111,7 @@ public class DmiModelOperations { .variablePathSegment("cmHandleId", cmHandle) .variablePathSegment("resourceName", resourceName) .build(dmiServiceName, dmiProperties.getDmiBasePath()); - return dmiRestClient.postOperationWithJsonData(dmiUrl, jsonRequestBody, OperationType.READ, null); + return dmiRestClient.postOperationWithJsonData(MODEL, dmiUrl, jsonRequestBody, OperationType.READ, null); } private static String getRequestBodyToFetchYangResources(final Collection<ModuleReference> newModuleReferences, diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/async/DataOperationEventConsumerSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/async/DataOperationEventConsumerSpec.groovy index 369b496ca..b095f904a 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/async/DataOperationEventConsumerSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/async/DataOperationEventConsumerSpec.groovy @@ -20,6 +20,8 @@ package org.onap.cps.ncmp.api.impl.async +import static org.onap.cps.ncmp.api.impl.events.mapper.CloudEventMapper.toTargetEvent + import com.fasterxml.jackson.databind.ObjectMapper import io.cloudevents.CloudEvent import io.cloudevents.kafka.CloudEventDeserializer @@ -42,8 +44,6 @@ import org.springframework.test.annotation.DirtiesContext import org.testcontainers.spock.Testcontainers import java.time.Duration -import static org.onap.cps.ncmp.api.impl.events.mapper.CloudEventMapper.toTargetEvent - @SpringBootTest(classes = [EventsPublisher, DataOperationEventConsumer, RecordFilterStrategies, JsonObjectMapper, ObjectMapper]) @Testcontainers @DirtiesContext diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/client/DmiRestClientSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/client/DmiRestClientSpec.groovy index d2dce06b0..2c2212773 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/client/DmiRestClientSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/client/DmiRestClientSpec.groovy @@ -21,9 +21,11 @@ package org.onap.cps.ncmp.api.impl.client -import static org.onap.cps.ncmp.api.impl.operations.OperationType.READ -import static org.onap.cps.ncmp.api.impl.operations.OperationType.PATCH +import static org.onap.cps.ncmp.api.NcmpResponseStatus.UNKNOWN_ERROR import static org.onap.cps.ncmp.api.impl.operations.OperationType.CREATE +import static org.onap.cps.ncmp.api.impl.operations.OperationType.PATCH +import static org.onap.cps.ncmp.api.impl.operations.OperationType.READ +import static org.onap.cps.ncmp.api.impl.operations.RequiredDmiService.DATA import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE import static org.onap.cps.ncmp.api.NcmpResponseStatus.DMI_SERVICE_NOT_RESPONDING import static org.onap.cps.ncmp.api.NcmpResponseStatus.UNABLE_TO_READ_RESOURCE_DATA @@ -31,109 +33,102 @@ import static org.onap.cps.ncmp.api.NcmpResponseStatus.UNABLE_TO_READ_RESOURCE_D import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode -import org.onap.cps.ncmp.api.impl.config.DmiWebClientConfiguration -import org.onap.cps.ncmp.api.impl.exception.InvalidDmiResourceUrlException import org.onap.cps.ncmp.api.impl.exception.DmiClientRequestException +import org.onap.cps.ncmp.api.impl.exception.InvalidDmiResourceUrlException +import org.onap.cps.ncmp.api.impl.config.DmiProperties import org.onap.cps.ncmp.utils.TestUtils import org.onap.cps.utils.JsonObjectMapper -import org.spockframework.spring.SpringBean -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpHeaders +import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity -import org.springframework.test.context.ContextConfiguration import org.springframework.web.client.HttpServerErrorException import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.WebClientResponseException +import org.springframework.web.reactive.function.client.WebClientRequestException import reactor.core.publisher.Mono import spock.lang.Specification -import org.springframework.web.reactive.function.client.WebClientResponseException -import org.onap.cps.ncmp.api.impl.config.DmiProperties -@SpringBootTest -@ContextConfiguration(classes = [DmiProperties, DmiRestClient, ObjectMapper]) class DmiRestClientSpec extends Specification { static final NO_AUTH_HEADER = null - static final BASIC_AUTH_HEADER = 'Basic c29tZS11c2VyOnNvbWUtcGFzc3dvcmQ=' + static final BASIC_AUTH_HEADER = 'Basic c29tZSB1c2VyOnNvbWUgcGFzc3dvcmQ=' static final BEARER_AUTH_HEADER = 'Bearer my-bearer-token' - @Autowired - DmiProperties dmiProperties + def mockDataServicesWebClient = Mock(WebClient) + def mockModelServicesWebClient = Mock(WebClient) + def mockHealthChecksWebClient = Mock(WebClient) - @Autowired - DmiRestClient objectUnderTest + def mockRequestBody = Mock(WebClient.RequestBodyUriSpec) + def mockResponse = Mock(WebClient.ResponseSpec) - @SpringBean - WebClient mockWebClient = Mock(WebClient); + def responseBody = [message: 'Success'] + def mockDmiProperties = Mock(DmiProperties) - @SpringBean JsonObjectMapper jsonObjectMapper = new JsonObjectMapper(new ObjectMapper()) - def mockRequestBodyUriSpec = Mock(WebClient.RequestBodyUriSpec) - def mockResponseSpec = Mock(WebClient.ResponseSpec) - def mockResponseEntity = Mock(ResponseEntity) + DmiRestClient objectUnderTest = new DmiRestClient(mockDmiProperties, jsonObjectMapper, mockDataServicesWebClient, mockModelServicesWebClient, mockHealthChecksWebClient) def setup() { - mockRequestBodyUriSpec.uri(_) >> mockRequestBodyUriSpec - mockRequestBodyUriSpec.headers(_) >> mockRequestBodyUriSpec - mockRequestBodyUriSpec.retrieve() >> mockResponseSpec + mockRequestBody.uri(_) >> mockRequestBody + mockRequestBody.headers(_) >> mockRequestBody + mockRequestBody.body(_) >> mockRequestBody + mockRequestBody.retrieve() >> mockResponse } - def 'DMI POST operation with JSON.'() { + def 'DMI POST Operation with JSON for status #httpStatusCode'() { given: 'the web client returns a valid response entity for the expected parameters' - mockWebClient.post() >> mockRequestBodyUriSpec - mockRequestBodyUriSpec.body(_) >> mockRequestBodyUriSpec - def monoSpec = Mono.just(mockResponseEntity) - mockResponseSpec.bodyToMono(Object.class) >> monoSpec - monoSpec.block() >> mockResponseEntity + mockDataServicesWebClient.post() >> mockRequestBody + mockResponse.toEntity(Object.class) >> Mono.just(new ResponseEntity<>(responseBody, httpStatusCode)) when: 'POST operation is invoked' - def response = objectUnderTest.postOperationWithJsonData('/my/url', 'some json', READ, null) + def response = objectUnderTest.postOperationWithJsonData(DATA, '/my/url', 'some json', READ, NO_AUTH_HEADER) then: 'the output of the method is equal to the output from the test template' - assert response.statusCode.value() == 200 - assert response.hasBody() + assert response.statusCode == httpStatusCode + assert response.body == responseBody + where: 'the following status codes are used' + httpStatusCode << [HttpStatus.OK, HttpStatus.CREATED, HttpStatus.ACCEPTED] } def 'Failing DMI POST operation for server error'() { given: 'the web client throws an exception' - mockWebClient.post() >> { throw new HttpServerErrorException(SERVICE_UNAVAILABLE, null, null, null) } + mockDataServicesWebClient.post() >> { throw new HttpServerErrorException(SERVICE_UNAVAILABLE, null, null, null) } when: 'POST operation is invoked' - objectUnderTest.postOperationWithJsonData('/some', 'some json', READ, null) + objectUnderTest.postOperationWithJsonData(DATA, '/some', 'some json', READ, NO_AUTH_HEADER) then: 'a http client exception is thrown' def thrown = thrown(DmiClientRequestException) and: 'the exception has the relevant details from the error response' - assert thrown.ncmpResponseStatus.code == '102' - assert thrown.httpStatusCode == 503 + thrown.ncmpResponseStatus.code == '102' + thrown.httpStatusCode == 503 } def 'Failing DMI POST operation due to invalid dmi resource url.'() { when: 'POST operation is invoked with invalid dmi resource url' - objectUnderTest.postOperationWithJsonData('/invalid dmi url', null, null, null) + objectUnderTest.postOperationWithJsonData(DATA, '/invalid dmi url', null, null, NO_AUTH_HEADER) then: 'invalid dmi resource url exception is thrown' def thrown = thrown(InvalidDmiResourceUrlException) and: 'the exception has the relevant details from the error response' - assert thrown.httpStatus == 400 - assert thrown.message == 'Invalid dmi resource url: /invalid dmi url' + thrown.httpStatus == 400 + thrown.message == 'Invalid dmi resource url: /invalid dmi url' where: 'the following operations are executed' operation << [CREATE, READ, PATCH] } def 'Dmi service sends client error response when #scenario'() { given: 'the web client unable to return response entity but error' - mockWebClient.post() >> mockRequestBodyUriSpec - mockRequestBodyUriSpec.body(_) >> mockRequestBodyUriSpec - def monoSpec = Mono.error(new WebClientResponseException('message', httpStatusCode, null, null, null, null)) - mockResponseSpec.bodyToMono(Object.class) >> monoSpec + mockDataServicesWebClient.post() >> mockRequestBody + mockResponse.toEntity(Object.class) >> Mono.error(exceptionType) when: 'POST operation is invoked' - objectUnderTest.postOperationWithJsonData('/my/url', 'some json', READ, null) + objectUnderTest.postOperationWithJsonData(DATA, '/my/url', 'some json', READ, NO_AUTH_HEADER) then: 'a http client exception is thrown' def thrown = thrown(DmiClientRequestException) and: 'the exception has the relevant details from the error response' - assert thrown.ncmpResponseStatus.code == expectedNcmpResponseStatusCode + assert thrown.ncmpResponseStatus == expectedNcmpResponseStatusCode assert thrown.httpStatusCode == httpStatusCode where: 'the following errors occur' - scenario | httpStatusCode | expectedNcmpResponseStatusCode - 'dmi request timeout' | 408 | DMI_SERVICE_NOT_RESPONDING.code - 'other error code' | 500 | UNABLE_TO_READ_RESOURCE_DATA.code + scenario | httpStatusCode | exceptionType || expectedNcmpResponseStatusCode + 'dmi service unavailable' | 503 | new WebClientRequestException(new RuntimeException('some-error'), null, null, new HttpHeaders()) || DMI_SERVICE_NOT_RESPONDING + 'dmi request timeout' | 408 | new WebClientResponseException('message', httpStatusCode, 'statusText', null, null, null) || DMI_SERVICE_NOT_RESPONDING + 'dmi server error' | 500 | new WebClientResponseException('message', httpStatusCode, 'statusText', null, null, null) || UNABLE_TO_READ_RESOURCE_DATA + 'unknown error' | 500 | new Throwable('message') || UNKNOWN_ERROR } def 'Dmi trust level is determined by spring boot health status'() { @@ -141,19 +136,18 @@ class DmiRestClientSpec extends Specification { def dmiPluginHealthCheckResponseJsonData = TestUtils.getResourceFileContent('dmiPluginHealthCheckResponse.json') def jsonNode = jsonObjectMapper.convertJsonString(dmiPluginHealthCheckResponseJsonData, JsonNode.class) ((ObjectNode) jsonNode).put('status', 'my status') - def monoResponse = Mono.just(jsonNode) - mockWebClient.get() >> mockRequestBodyUriSpec - mockResponseSpec.bodyToMono(_) >> monoResponse - monoResponse.block() >> jsonNode + mockHealthChecksWebClient.get() >> mockRequestBody + mockResponse.bodyToMono(JsonNode.class) >> Mono.just(jsonNode) when: 'get trust level of the dmi plugin' def result = objectUnderTest.getDmiHealthStatus('some/url') - then: 'the status value from the json is return' + then: 'the status value from the json is returned' assert result == 'my status' } def 'Failing to get dmi plugin health status #scenario'() { given: 'rest template with #scenario' - mockWebClient.get() >> healthStatusResponse + mockHealthChecksWebClient.get() >> mockRequestBody + mockResponse.bodyToMono(_) >> healthStatusResponse when: 'attempt to get health status of the dmi plugin' def result = objectUnderTest.getDmiHealthStatus('some url') then: 'result will be empty' @@ -161,15 +155,18 @@ class DmiRestClientSpec extends Specification { where: 'the following responses are used' scenario | healthStatusResponse 'null' | null - 'exception' | {throw new Exception()} + 'exception' | { throw new Exception() } } def 'DMI auth header #scenario'() { when: 'Specific dmi properties are provided' - dmiProperties.dmiBasicAuthEnabled = authEnabled + mockDmiProperties.dmiBasicAuthEnabled >> authEnabled + mockDmiProperties.authUsername >> 'some user' + mockDmiProperties.authPassword >> 'some password' then: 'http headers to conditionally have Authorization header' - def authHeaderValues = objectUnderTest.configureHttpHeaders(new HttpHeaders(), ncmpAuthHeader).getOrEmpty('Authorization') - def outputAuthHeader = (authHeaderValues == null ? null : authHeaderValues[0]) + def httpHeaders = new HttpHeaders() + objectUnderTest.configureHttpHeaders(httpHeaders, ncmpAuthHeader) + def outputAuthHeader = (httpHeaders.Authorization == null ? null : httpHeaders.Authorization[0]) assert outputAuthHeader == expectedAuthHeader where: 'the following configurations are used' scenario | authEnabled | ncmpAuthHeader || expectedAuthHeader @@ -179,4 +176,4 @@ class DmiRestClientSpec extends Specification { 'DMI basic auth disabled, with NCMP bearer token' | false | BEARER_AUTH_HEADER || BEARER_AUTH_HEADER 'DMI basic auth disabled, with NCMP basic auth' | false | BASIC_AUTH_HEADER || NO_AUTH_HEADER } -} +}
\ No newline at end of file diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/DmiWebClientConfigurationSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/DmiWebClientConfigurationSpec.groovy index ee7ab3f28..05ecaa11b 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/DmiWebClientConfigurationSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/DmiWebClientConfigurationSpec.groovy @@ -29,7 +29,7 @@ import spock.lang.Specification @SpringBootTest @ContextConfiguration(classes = [HttpClientConfiguration]) -@TestPropertySource(properties = ['ncmp.dmi.httpclient.connectionTimeoutInSeconds=1', 'ncmp.dmi.httpclient.maximumInMemorySizeInMegabytes=1']) +@TestPropertySource(properties = ['ncmp.dmi.httpclient.data-services.connectionTimeoutInSeconds=1', 'ncmp.dmi.httpclient.model-services.maximumInMemorySizeInMegabytes=1']) @EnableConfigurationProperties class DmiWebClientConfigurationSpec extends Specification { @@ -42,11 +42,27 @@ class DmiWebClientConfigurationSpec extends Specification { new DmiWebClientConfiguration(httpClientConfiguration) != null } - def 'Creating a WebClient instance.'() { - given: 'WebClient configuration invoked' - def webClientInstance = objectUnderTest.webClient() - expect: 'the system can create an instance' - assert webClientInstance != null - assert webClientInstance instanceof WebClient + def 'Creating a web client instance data service.'() { + given: 'Web client configuration is invoked' + def dataServicesWebClient = objectUnderTest.dataServicesWebClient() + expect: 'the system can create an instance for data service' + assert dataServicesWebClient != null + assert dataServicesWebClient instanceof WebClient + } + + def 'Creating a web client instance model service.'() { + given: 'Web client configuration invoked' + def modelServicesWebClient = objectUnderTest.modelServicesWebClient() + expect: 'the system can create an instance for model service' + assert modelServicesWebClient != null + assert modelServicesWebClient instanceof WebClient + } + + def 'Creating a web client instance health service.'() { + given: 'Web client configuration invoked' + def healthChecksWebClient = objectUnderTest.healthChecksWebClient() + expect: 'the system can create an instance for health service' + assert healthChecksWebClient != null + assert healthChecksWebClient instanceof WebClient } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/HttpClientConfigurationSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/HttpClientConfigurationSpec.groovy index 4ede360e6..b7ced2382 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/HttpClientConfigurationSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/HttpClientConfigurationSpec.groovy @@ -30,18 +30,42 @@ import spock.lang.Specification @SpringBootTest @ContextConfiguration(classes = [HttpClientConfiguration]) @EnableConfigurationProperties(HttpClientConfiguration.class) -@TestPropertySource(properties = ["ncmp.dmi.httpclient.readTimeoutInSeconds=123", "ncmp.dmi.httpclient.maximumConnectionsTotal=200"]) +@TestPropertySource(properties = ["ncmp.dmi.httpclient.data-services.readTimeoutInSeconds=789", "ncmp.dmi.httpclient.model-services.maximumConnectionsTotal=111"]) class HttpClientConfigurationSpec extends Specification { @Autowired private HttpClientConfiguration httpClientConfiguration - def 'Test HttpClientConfiguration properties with custom and default values'() { - expect: 'properties are populated correctly' - assert httpClientConfiguration.connectionTimeoutInSeconds == 123 - assert httpClientConfiguration.readTimeoutInSeconds == 123 - assert httpClientConfiguration.writeTimeoutInSeconds == 30 - assert httpClientConfiguration.maximumConnectionsTotal == 200 - assert httpClientConfiguration.maximumInMemorySizeInMegabytes == 16 + def 'Test http client configuration properties of data with custom and default values'() { + expect: 'properties are populated correctly for data' + with(httpClientConfiguration.dataServices) { + assert connectionTimeoutInSeconds == 123 + assert readTimeoutInSeconds == 789 + assert writeTimeoutInSeconds == 30 + assert maximumConnectionsTotal == 100 + assert maximumInMemorySizeInMegabytes == 7 + } + } + + def 'Test http client configuration properties of model with custom and default values'() { + expect: 'properties are populated correctly for model' + with(httpClientConfiguration.modelServices) { + assert connectionTimeoutInSeconds == 456 + assert readTimeoutInSeconds == 30 + assert writeTimeoutInSeconds == 30 + assert maximumConnectionsTotal == 111 + assert maximumInMemorySizeInMegabytes == 8 + } + } + + def 'Test http client configuration properties of health with default values'() { + expect: 'properties are populated correctly for health' + with(httpClientConfiguration.healthCheckServices) { + assert connectionTimeoutInSeconds == 30 + assert readTimeoutInSeconds == 30 + assert writeTimeoutInSeconds == 30 + assert maximumConnectionsTotal == 10 + assert maximumInMemorySizeInMegabytes == 1 + } } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiDataOperationsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiDataOperationsSpec.groovy index a84e1348e..ad3f85c84 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiDataOperationsSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiDataOperationsSpec.groovy @@ -21,15 +21,14 @@ package org.onap.cps.ncmp.api.impl.operations -import org.onap.cps.ncmp.api.impl.inventory.CmHandleState - -import static org.onap.cps.ncmp.api.NcmpResponseStatus.UNKNOWN_ERROR import static org.onap.cps.ncmp.api.impl.events.mapper.CloudEventMapper.toTargetEvent import static org.onap.cps.ncmp.api.impl.operations.DatastoreType.PASSTHROUGH_OPERATIONAL import static org.onap.cps.ncmp.api.impl.operations.DatastoreType.PASSTHROUGH_RUNNING import static org.onap.cps.ncmp.api.impl.operations.OperationType.CREATE import static org.onap.cps.ncmp.api.impl.operations.OperationType.READ import static org.onap.cps.ncmp.api.impl.operations.OperationType.UPDATE +import static org.onap.cps.ncmp.api.impl.operations.RequiredDmiService.DATA +import static org.onap.cps.ncmp.api.NcmpResponseStatus.UNKNOWN_ERROR import com.fasterxml.jackson.databind.ObjectMapper import org.onap.cps.events.EventsPublisher @@ -40,6 +39,7 @@ import org.onap.cps.ncmp.api.models.DataOperationRequest import org.onap.cps.ncmp.api.models.CmResourceAddress import org.onap.cps.ncmp.events.async1_0_0.DataOperationEvent import org.onap.cps.ncmp.utils.TestUtils +import org.onap.cps.ncmp.api.impl.inventory.CmHandleState import org.onap.cps.utils.JsonObjectMapper import org.spockframework.spring.SpringBean import org.springframework.beans.factory.annotation.Autowired @@ -77,7 +77,7 @@ class DmiDataOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity<Object>(HttpStatus.OK) def expectedUrl = "${dmiServiceBaseUrl}${expectedDatastoreInUrl}?resourceIdentifier=${resourceIdentifier}${expectedOptionsInUrl}" def expectedJson = '{"operation":"read","cmHandleProperties":' + expectedProperties + ',"moduleSetTag":""}' - mockDmiRestClient.postOperationWithJsonData(expectedUrl, expectedJson, READ, NO_AUTH_HEADER) >> responseFromDmi + mockDmiRestClient.postOperationWithJsonData(DATA, expectedUrl, expectedJson, READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get resource data is invoked' def cmResourceAddress = new CmResourceAddress(dataStore.datastoreName, cmHandleId, resourceIdentifier) def result = objectUnderTest.getResourceDataFromDmi(cmResourceAddress, options, NO_TOPIC, NO_REQUEST_ID, NO_AUTH_HEADER) @@ -103,11 +103,11 @@ class DmiDataOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity<Object>(HttpStatus.ACCEPTED) def expectedDmiBatchResourceDataUrl = "someServiceName/dmi/v1/data?requestId=requestId&topic=my-topic-name" def expectedBatchRequestAsJson = '{"operations":[{"operation":"read","operationId":"operational-14","datastore":"ncmp-datastore:passthrough-operational","options":"some option","resourceIdentifier":"some resource identifier","cmHandles":[{"id":"some-cm-handle","moduleSetTag":"","cmHandleProperties":{"prop1":"val1"}}]}]}' - mockDmiRestClient.postOperationWithJsonData(expectedDmiBatchResourceDataUrl, _, READ.operationName, NO_AUTH_HEADER) >> responseFromDmi + mockDmiRestClient.postOperationWithJsonData(DATA, expectedDmiBatchResourceDataUrl, _, READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get resource data for group of cm handles are invoked' objectUnderTest.requestResourceDataFromDmi('my-topic-name', dataOperationRequest, 'requestId', NO_AUTH_HEADER) then: 'the post operation was called and ncmp generated dmi request body json args' - 1 * mockDmiRestClient.postOperationWithJsonData(expectedDmiBatchResourceDataUrl, expectedBatchRequestAsJson, READ, NO_AUTH_HEADER) + 1 * mockDmiRestClient.postOperationWithJsonData(DATA, expectedDmiBatchResourceDataUrl, expectedBatchRequestAsJson, READ, NO_AUTH_HEADER) } def 'Execute (async) data operation from DMI service with Exception.'() { @@ -139,7 +139,7 @@ class DmiDataOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity<Object>(HttpStatus.OK) def expectedUrl = dmiServiceBaseUrl + "passthrough-operational?resourceIdentifier=/" def expectedJson = '{"operation":"read","cmHandleProperties":{"prop1":"val1"},"moduleSetTag":"my-module-set-tag"}' - mockDmiRestClient.postOperationWithJsonData(expectedUrl, expectedJson, READ, null) >> responseFromDmi + mockDmiRestClient.postOperationWithJsonData(DATA, expectedUrl, expectedJson, READ, null) >> responseFromDmi when: 'get resource data is invoked' def result = objectUnderTest.getResourceDataFromDmi( PASSTHROUGH_OPERATIONAL.datastoreName, cmHandleId, NO_REQUEST_ID) then: 'the result is the response from the DMI service' @@ -153,7 +153,7 @@ class DmiDataOperationsSpec extends DmiOperationsBaseSpec { def expectedUrl = "${dmiServiceBaseUrl}passthrough-running?resourceIdentifier=${resourceIdentifier}" def expectedJson = '{"operation":"' + expectedOperationInUrl + '","dataType":"some data type","data":"requestData","cmHandleProperties":{"prop1":"val1"},"moduleSetTag":""}' def responseFromDmi = new ResponseEntity<Object>(HttpStatus.OK) - mockDmiRestClient.postOperationWithJsonData(expectedUrl, expectedJson, operation, NO_AUTH_HEADER) >> responseFromDmi + mockDmiRestClient.postOperationWithJsonData(DATA, expectedUrl, expectedJson, operation, NO_AUTH_HEADER) >> responseFromDmi when: 'write resource method is invoked' def result = objectUnderTest.writeResourceDataPassThroughRunningFromDmi(cmHandleId, 'parent/child', operation, 'requestData', 'some data type', NO_AUTH_HEADER) then: 'the result is the response from the DMI service' diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiModelOperationsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiModelOperationsSpec.groovy index de5e15e50..db7f26f5f 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiModelOperationsSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/operations/DmiModelOperationsSpec.groovy @@ -21,6 +21,9 @@ package org.onap.cps.ncmp.api.impl.operations +import static org.onap.cps.ncmp.api.impl.operations.OperationType.READ +import static org.onap.cps.ncmp.api.impl.operations.RequiredDmiService.MODEL + import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.databind.ObjectMapper import org.onap.cps.ncmp.api.impl.config.DmiProperties @@ -34,8 +37,6 @@ import org.springframework.http.ResponseEntity import org.springframework.test.context.ContextConfiguration import spock.lang.Shared -import static org.onap.cps.ncmp.api.impl.operations.OperationType.READ - @SpringBootTest @ContextConfiguration(classes = [DmiProperties, DmiModelOperations]) class DmiModelOperationsSpec extends DmiOperationsBaseSpec { @@ -58,7 +59,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { def moduleReferencesAsLisOfMaps = [[moduleName: 'mod1', revision: 'A'], [moduleName: 'mod2', revision: 'X']] def expectedUrl = "${dmiServiceName}/dmi/v1/ch/${cmHandleId}/modules" def responseFromDmi = new ResponseEntity([schemas: moduleReferencesAsLisOfMaps], HttpStatus.OK) - mockDmiRestClient.postOperationWithJsonData(expectedUrl, '{"cmHandleProperties":{},"moduleSetTag":""}', READ, NO_AUTH_HEADER) >> responseFromDmi + mockDmiRestClient.postOperationWithJsonData(MODEL, expectedUrl, '{"cmHandleProperties":{},"moduleSetTag":""}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get module references is called' def result = objectUnderTest.getModuleReferences(yangModelCmHandle) then: 'the result consists of expected module references' @@ -89,7 +90,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { mockYangModelCmHandleRetrieval(dmiProperties) and: 'a positive response from DMI service when it is called with tha expected parameters' def responseFromDmi = new ResponseEntity<String>(HttpStatus.OK) - mockDmiRestClient.postOperationWithJsonData("${dmiServiceName}/dmi/v1/ch/${cmHandleId}/modules", + mockDmiRestClient.postOperationWithJsonData(MODEL, "${dmiServiceName}/dmi/v1/ch/${cmHandleId}/modules", '{"cmHandleProperties":' + expectedAdditionalPropertiesInRequest + ',"moduleSetTag":""}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'a get module references is called' def result = objectUnderTest.getModuleReferences(yangModelCmHandle) @@ -108,7 +109,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity([[moduleName: 'mod1', revision: 'A', yangSource: 'some yang source'], [moduleName: 'mod2', revision: 'C', yangSource: 'other yang source']], HttpStatus.OK) def expectedModuleReferencesInRequest = '{"name":"mod1","revision":"A"},{"name":"mod2","revision":"X"}' - mockDmiRestClient.postOperationWithJsonData("${dmiServiceName}/dmi/v1/ch/${cmHandleId}/moduleResources", + mockDmiRestClient.postOperationWithJsonData(MODEL, "${dmiServiceName}/dmi/v1/ch/${cmHandleId}/moduleResources", '{"data":{"modules":[' + expectedModuleReferencesInRequest + ']},"cmHandleProperties":{}}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get new yang resources from DMI service' def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) @@ -140,7 +141,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { mockYangModelCmHandleRetrieval(dmiProperties) and: 'a positive response from DMI service when it is called with the expected moduleSetTag, modules and properties' def responseFromDmi = new ResponseEntity<>([[moduleName: 'mod1', revision: 'A', yangSource: 'some yang source']], HttpStatus.OK) - mockDmiRestClient.postOperationWithJsonData("${dmiServiceName}/dmi/v1/ch/${cmHandleId}/moduleResources", + mockDmiRestClient.postOperationWithJsonData(MODEL, "${dmiServiceName}/dmi/v1/ch/${cmHandleId}/moduleResources", '{"data":{"modules":[{"name":"mod1","revision":"A"},{"name":"mod2","revision":"X"}]},"cmHandleProperties":' + expectedAdditionalPropertiesInRequest + '}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get new yang resources from DMI service' @@ -158,9 +159,9 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { mockYangModelCmHandleRetrieval([], moduleSetTag) and: 'a positive response from DMI service when it is called with the expected moduleSetTag' def responseFromDmi = new ResponseEntity<>([[moduleName: 'mod1', revision: 'A', yangSource: 'some yang source']], HttpStatus.OK) - mockDmiRestClient.postOperationWithJsonData("${dmiServiceName}/dmi/v1/ch/${cmHandleId}/moduleResources", - '{' + expectedModuleSetTagInRequest + '"data":{"modules":[{"name":"mod1","revision":"A"},{"name":"mod2","revision":"X"}]},"cmHandleProperties":{}}', - READ, NO_AUTH_HEADER) >> responseFromDmi + mockDmiRestClient.postOperationWithJsonData(MODEL, "${dmiServiceName}/dmi/v1/ch/${cmHandleId}/moduleResources", + '{' + expectedModuleSetTagInRequest + '"data":{"modules":[{"name":"mod1","revision":"A"},{"name":"mod2","revision":"X"}]},"cmHandleProperties":{}}', + READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get new yang resources from DMI service' def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) then: 'the result is the response from DMI service' diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/utils/context/CpsApplicationContextSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/utils/context/CpsApplicationContextSpec.groovy index b7fa44925..ee117160c 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/utils/context/CpsApplicationContextSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/utils/context/CpsApplicationContextSpec.groovy @@ -1,3 +1,23 @@ +/* + * ============LICENSE_START======================================================== + * Copyright (c) 2023-2024 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + package org.onap.cps.ncmp.api.impl.utils.context import com.fasterxml.jackson.databind.ObjectMapper diff --git a/cps-ncmp-service/src/test/resources/application.yml b/cps-ncmp-service/src/test/resources/application.yml index 2a93f4081..e35f47100 100644 --- a/cps-ncmp-service/src/test/resources/application.yml +++ b/cps-ncmp-service/src/test/resources/application.yml @@ -37,8 +37,12 @@ app: ncmp: dmi: httpclient: - connectionTimeoutInSeconds: 123 - maximumInMemorySizeInMegabytes: 16 + data-services: + connectionTimeoutInSeconds: 123 + maximumInMemorySizeInMegabytes: 7 + model-services: + connectionTimeoutInSeconds: 456 + maximumInMemorySizeInMegabytes: 8 auth: username: some-user password: some-password diff --git a/cps-rest/docs/openapi/cpsDataV2.yml b/cps-rest/docs/openapi/cpsDataV2.yml index cbb5ce410..a1433badf 100644 --- a/cps-rest/docs/openapi/cpsDataV2.yml +++ b/cps-rest/docs/openapi/cpsDataV2.yml @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2022-2023 TechMahindra Ltd. +# Copyright (c) 2022-2024 TechMahindra Ltd. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -75,4 +75,55 @@ deltaByDataspaceAndAnchors: $ref: 'components.yml#/components/responses/Forbidden' '500': $ref: 'components.yml#/components/responses/InternalServerError' - x-codegen-request-body-name: xpath
\ No newline at end of file + x-codegen-request-body-name: xpath + +deltaByDataspaceAnchorAndPayload: + post: + description: Get delta between an anchor in a dataspace and JSON payload + tags: + - cps-data + summary: Get delta between an anchor and JSON payload + operationId: getDeltaByDataspaceAnchorAndPayload + parameters: + - $ref: 'components.yml#/components/parameters/dataspaceNameInPath' + - $ref: 'components.yml#/components/parameters/anchorNameInPath' + - $ref: 'components.yml#/components/parameters/xpathInQuery' + requestBody: + content: + multipart/form-data: + schema: + type: object + properties: + json: + type: object + example: + test:bookstore: + bookstore-name: Chapters + categories: + - code: 01 + name: SciFi + - code: 02 + name: kids + file: + type: string + format: binary + required: + - json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + dataSample: + $ref: 'components.yml#/components/examples/deltaReportSample' + '400': + $ref: 'components.yml#/components/responses/BadRequest' + '401': + $ref: 'components.yml#/components/responses/Unauthorized' + '403': + $ref: 'components.yml#/components/responses/Forbidden' + '500': + $ref: 'components.yml#/components/responses/InternalServerError'
\ No newline at end of file diff --git a/cps-rest/docs/openapi/openapi.yml b/cps-rest/docs/openapi/openapi.yml index f29335a0a..b4e0b7040 100644 --- a/cps-rest/docs/openapi/openapi.yml +++ b/cps-rest/docs/openapi/openapi.yml @@ -2,7 +2,7 @@ # Copyright (C) 2021-2023 Nordix Foundation # Modifications Copyright (C) 2021 Pantheon.tech # Modifications Copyright (C) 2021 Bell Canada. -# Modifications Copyright (C) 2022-2023 TechMahindra Ltd. +# Modifications Copyright (C) 2022-2024 TechMahindra Ltd. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -104,9 +104,12 @@ paths: /{apiVersion}/dataspaces/{dataspace-name}/anchors/{anchor-name}/list-nodes: $ref: 'cpsData.yml#/listElementByDataspaceAndAnchor' - /v2/dataspaces/{dataspace-name}/anchors/{anchor-name}/delta: + /v2/dataspaces/{dataspace-name}/anchors/{anchor-name}/deltaAnchors: $ref: 'cpsDataV2.yml#/deltaByDataspaceAndAnchors' + /v2/dataspaces/{dataspace-name}/anchors/{anchor-name}/deltaPayload: + $ref: 'cpsDataV2.yml#/deltaByDataspaceAnchorAndPayload' + /v1/dataspaces/{dataspace-name}/anchors/{anchor-name}/nodes/query: $ref: 'cpsQueryV1Deprecated.yml#/nodesByDataspaceAndAnchorAndCpsPath' diff --git a/cps-rest/src/main/java/org/onap/cps/rest/controller/DataRestController.java b/cps-rest/src/main/java/org/onap/cps/rest/controller/DataRestController.java index 310171b30..f579c82d2 100755 --- a/cps-rest/src/main/java/org/onap/cps/rest/controller/DataRestController.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/controller/DataRestController.java @@ -24,12 +24,15 @@ package org.onap.cps.rest.controller; +import static org.onap.cps.rest.utils.MultipartFileUtil.extractYangResourcesMap; + import io.micrometer.core.annotation.Timed; import jakarta.validation.ValidationException; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import lombok.RequiredArgsConstructor; @@ -49,6 +52,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; @RestController @RequestMapping("${rest.api.cps-base-path}") @@ -172,6 +176,27 @@ public class DataRestController implements CpsDataApi { } @Override + public ResponseEntity<Object> getDeltaByDataspaceAnchorAndPayload(final String dataspaceName, + final String sourceAnchorName, + final Object jsonPayload, + final String xpath, + final MultipartFile multipartFile) { + final FetchDescendantsOption fetchDescendantsOption = FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS; + + final Map<String, String> yangResourceMap; + if (multipartFile == null) { + yangResourceMap = Collections.emptyMap(); + } else { + yangResourceMap = extractYangResourcesMap(multipartFile); + } + final Collection<DeltaReport> deltaReports = Collections.unmodifiableList( + cpsDataService.getDeltaByDataspaceAnchorAndPayload(dataspaceName, sourceAnchorName, + xpath, yangResourceMap, jsonPayload.toString(), fetchDescendantsOption)); + + return new ResponseEntity<>(jsonObjectMapper.asJsonString(deltaReports), HttpStatus.OK); + } + + @Override @Timed(value = "cps.data.controller.get.delta", description = "Time taken to get delta between anchors") public ResponseEntity<Object> getDeltaByDataspaceAndAnchors(final String dataspaceName, diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy index 3f5dcf263..317b9c5b7 100755 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy @@ -41,7 +41,9 @@ import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest import org.springframework.http.HttpStatus import org.springframework.http.MediaType +import org.springframework.mock.web.MockMultipartFile import org.springframework.test.web.servlet.MockMvc +import org.springframework.web.multipart.MultipartFile import spock.lang.Shared import spock.lang.Specification @@ -49,6 +51,7 @@ import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS import static org.onap.cps.spi.FetchDescendantsOption.OMIT_DESCENDANTS import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put @@ -101,6 +104,10 @@ class DataRestControllerSpec extends Specification { static DataNode dataNodeWithChild = new DataNodeBuilder().withXpath('/parent') .withChildDataNodes([new DataNodeBuilder().withXpath("/parent/child").build()]).build() + @Shared + static MultipartFile multipartYangFile = new MockMultipartFile("file", 'filename.yang', "text/plain", 'content'.getBytes()) + + def setup() { dataNodeBaseEndpointV1 = "$basePath/v1/dataspaces/$dataspaceName" dataNodeBaseEndpointV2 = "$basePath/v2/dataspaces/$dataspaceName" @@ -337,9 +344,9 @@ class DataRestControllerSpec extends Specification { def 'Get delta between two anchors'() { given: 'the service returns a list containing delta reports' - def deltaReports = new DeltaReportBuilder().actionAdd().withXpath('/bookstore').withSourceData('bookstore-name': 'Easons').withTargetData('bookstore-name': 'Easons').build() + def deltaReports = new DeltaReportBuilder().actionUpdate().withXpath('some xpath').withSourceData('some key': 'some value').withTargetData('some key': 'some value').build() def xpath = 'some xpath' - def endpoint = "$dataNodeBaseEndpointV2/anchors/sourceAnchor/delta" + def endpoint = "$dataNodeBaseEndpointV2/anchors/sourceAnchor/deltaAnchors" mockCpsDataService.getDeltaByDataspaceAndAnchors(dataspaceName, 'sourceAnchor', 'targetAnchor', xpath, OMIT_DESCENDANTS) >> [deltaReports] when: 'get delta request is performed using REST API' def response = @@ -350,7 +357,48 @@ class DataRestControllerSpec extends Specification { then: 'expected response code is returned' assert response.status == HttpStatus.OK.value() and: 'the response contains expected value' - assert response.contentAsString.contains("[{\"action\":\"add\",\"xpath\":\"/bookstore\",\"sourceData\":{\"bookstore-name\":\"Easons\"},\"targetData\":{\"bookstore-name\":\"Easons\"}}]") + assert response.contentAsString.contains("[{\"action\":\"update\",\"xpath\":\"some xpath\",\"sourceData\":{\"some key\":\"some value\"},\"targetData\":{\"some key\":\"some value\"}}]") + } + + def 'Get delta between anchor and JSON payload with multipart file'() { + given: 'sample delta report, xpath, yang model file and json payload' + def deltaReports = new DeltaReportBuilder().actionAdd().withXpath('some xpath').build() + def xpath = 'some xpath' + def endpoint = "$dataNodeBaseEndpointV2/anchors/$anchorName/deltaPayload" + and: 'the service layer returns a list containing delta reports' + mockCpsDataService.getDeltaByDataspaceAnchorAndPayload(dataspaceName, anchorName, xpath, ['filename.yang':'content'], expectedJsonData, INCLUDE_ALL_DESCENDANTS) >> [deltaReports] + when: 'get delta request is performed using REST API' + def response = + mvc.perform(multipart(endpoint) + .file(multipartYangFile) + .param("json", requestBodyJson) + .param('xpath', xpath) + .contentType(MediaType.MULTIPART_FORM_DATA)) + .andReturn().response + then: 'expected response code is returned' + assert response.status == HttpStatus.OK.value() + and: 'the response contains expected value' + assert response.contentAsString.contains("[{\"action\":\"add\",\"xpath\":\"some xpath\"}]") + } + + def 'Get delta between anchor and JSON payload without multipart file'() { + given: 'sample delta report, xpath, and json payload' + def deltaReports = new DeltaReportBuilder().actionRemove().withXpath('some xpath').build() + def xpath = 'some xpath' + def endpoint = "$dataNodeBaseEndpointV2/anchors/$anchorName/deltaPayload" + and: 'the service layer returns a list containing delta reports' + mockCpsDataService.getDeltaByDataspaceAnchorAndPayload(dataspaceName, anchorName, xpath, [:], expectedJsonData, INCLUDE_ALL_DESCENDANTS) >> [deltaReports] + when: 'get delta request is performed using REST API' + def response = + mvc.perform(multipart(endpoint) + .param("json", requestBodyJson) + .param('xpath', xpath) + .contentType(MediaType.MULTIPART_FORM_DATA)) + .andReturn().response + then: 'expected response code is returned' + assert response.status == HttpStatus.OK.value() + and: 'the response contains expected value' + assert response.contentAsString.contains("[{\"action\":\"remove\",\"xpath\":\"some xpath\"}]") } def 'Update data node leaves: #scenario.'() { @@ -507,4 +555,5 @@ class DataRestControllerSpec extends Specification { 'without observed timestamp' | null || 1 | HttpStatus.NO_CONTENT 'with invalid observed timestamp' | 'invalid' || 0 | HttpStatus.BAD_REQUEST } + } diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java b/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java index 71ed06103..f396b49e6 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java @@ -4,7 +4,7 @@ * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2021-2022 Bell Canada * Modifications Copyright (C) 2022 Deutsche Telekom AG - * Modifications Copyright (C) 2024 TechMahindra Ltd. + * Modifications Copyright (C) 2023-2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -303,4 +303,20 @@ public interface CpsDataService { List<DeltaReport> getDeltaByDataspaceAndAnchors(String dataspaceName, String sourceAnchorName, String targetAnchorName, String xpath, FetchDescendantsOption fetchDescendantsOption); + + /** + * Retrieves the delta between an anchor and JSON payload by xpath, using dataspace name and anchor name. + * + * @param dataspaceName source dataspace name + * @param sourceAnchorName source anchor name + * @param xpath xpath + * @param yangResourcesNameToContentMap YANG resources (files) map where key is a name and value is content + * @param targetData target data to be compared in JSON string format + * @param fetchDescendantsOption defines the scope of data to fetch: defaulted to INCLUDE_ALL_DESCENDANTS + * @return list containing {@link DeltaReport} objects + */ + List<DeltaReport> getDeltaByDataspaceAnchorAndPayload(String dataspaceName, String sourceAnchorName, String xpath, + Map<String, String> yangResourcesNameToContentMap, + String targetData, + FetchDescendantsOption fetchDescendantsOption); } diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java b/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java index 3496fc7c4..6386d38ff 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java @@ -30,6 +30,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -50,6 +51,9 @@ import org.onap.cps.spi.model.DataNodeBuilder; import org.onap.cps.spi.model.DeltaReport; import org.onap.cps.spi.utils.CpsValidator; import org.onap.cps.utils.ContentType; +import org.onap.cps.utils.DataMapUtils; +import org.onap.cps.utils.JsonObjectMapper; +import org.onap.cps.utils.PrefixResolver; import org.onap.cps.utils.YangParser; import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode; import org.springframework.stereotype.Service; @@ -69,6 +73,8 @@ public class CpsDataServiceImpl implements CpsDataService { private final CpsValidator cpsValidator; private final YangParser yangParser; private final CpsDeltaService cpsDeltaService; + private final JsonObjectMapper jsonObjectMapper; + private final PrefixResolver prefixResolver; @Override public void saveData(final String dataspaceName, final String anchorName, final String nodeData, @@ -83,7 +89,8 @@ public class CpsDataServiceImpl implements CpsDataService { final OffsetDateTime observedTimestamp, final ContentType contentType) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); final Anchor anchor = cpsAnchorService.getAnchor(dataspaceName, anchorName); - final Collection<DataNode> dataNodes = buildDataNodes(anchor, ROOT_NODE_XPATH, nodeData, contentType); + final Collection<DataNode> dataNodes = + buildDataNodesWithParentNodeXpath(anchor, ROOT_NODE_XPATH, nodeData, contentType); cpsDataPersistenceService.storeDataNodes(dataspaceName, anchorName, dataNodes); sendDataUpdatedEvent(anchor, ROOT_NODE_XPATH, Operation.CREATE, observedTimestamp); } @@ -102,7 +109,8 @@ public class CpsDataServiceImpl implements CpsDataService { final ContentType contentType) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); final Anchor anchor = cpsAnchorService.getAnchor(dataspaceName, anchorName); - final Collection<DataNode> dataNodes = buildDataNodes(anchor, parentNodeXpath, nodeData, contentType); + final Collection<DataNode> dataNodes = + buildDataNodesWithParentNodeXpath(anchor, parentNodeXpath, nodeData, contentType); cpsDataPersistenceService.addChildDataNodes(dataspaceName, anchorName, parentNodeXpath, dataNodes); sendDataUpdatedEvent(anchor, parentNodeXpath, Operation.CREATE, observedTimestamp); } @@ -115,7 +123,7 @@ public class CpsDataServiceImpl implements CpsDataService { cpsValidator.validateNameCharacters(dataspaceName, anchorName); final Anchor anchor = cpsAnchorService.getAnchor(dataspaceName, anchorName); final Collection<DataNode> listElementDataNodeCollection = - buildDataNodes(anchor, parentNodeXpath, jsonData, ContentType.JSON); + buildDataNodesWithParentNodeXpath(anchor, parentNodeXpath, jsonData, ContentType.JSON); if (isRootNodeXpath(parentNodeXpath)) { cpsDataPersistenceService.storeDataNodes(dataspaceName, anchorName, listElementDataNodeCollection); } else { @@ -153,8 +161,8 @@ public class CpsDataServiceImpl implements CpsDataService { final String nodeData, final OffsetDateTime observedTimestamp, final ContentType contentType) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); final Anchor anchor = cpsAnchorService.getAnchor(dataspaceName, anchorName); - final Collection<DataNode> dataNodesInPatch = buildDataNodes(anchor, parentNodeXpath, nodeData, - contentType); + final Collection<DataNode> dataNodesInPatch = + buildDataNodesWithParentNodeXpath(anchor, parentNodeXpath, nodeData, contentType); final Map<String, Map<String, Serializable>> xpathToUpdatedLeaves = dataNodesInPatch.stream() .collect(Collectors.toMap(DataNode::getXpath, DataNode::getLeaves)); cpsDataPersistenceService.batchUpdateDataLeaves(dataspaceName, anchorName, xpathToUpdatedLeaves); @@ -171,7 +179,7 @@ public class CpsDataServiceImpl implements CpsDataService { cpsValidator.validateNameCharacters(dataspaceName, anchorName); final Anchor anchor = cpsAnchorService.getAnchor(dataspaceName, anchorName); final Collection<DataNode> dataNodeUpdates = - buildDataNodes(anchor, parentNodeXpath, dataNodeUpdatesAsJson, ContentType.JSON); + buildDataNodesWithParentNodeXpath(anchor, parentNodeXpath, dataNodeUpdatesAsJson, ContentType.JSON); for (final DataNode dataNodeUpdate : dataNodeUpdates) { processDataNodeUpdate(anchor, dataNodeUpdate); } @@ -215,6 +223,29 @@ public class CpsDataServiceImpl implements CpsDataService { return cpsDeltaService.getDeltaReports(sourceDataNodes, targetDataNodes); } + @Timed(value = "cps.data.service.get.deltaBetweenAnchorAndPayload", + description = "Time taken to get delta between anchor and a payload") + @Override + public List<DeltaReport> getDeltaByDataspaceAnchorAndPayload(final String dataspaceName, + final String sourceAnchorName, final String xpath, + final Map<String, String> yangResourcesNameToContentMap, + final String targetData, + final FetchDescendantsOption fetchDescendantsOption) { + + final Anchor sourceAnchor = cpsAnchorService.getAnchor(dataspaceName, sourceAnchorName); + + final Collection<DataNode> sourceDataNodes = getDataNodes(dataspaceName, + sourceAnchorName, xpath, fetchDescendantsOption); + + final Collection<DataNode> sourceDataNodesRebuilt = + new ArrayList<>(rebuildSourceDataNodes(xpath, sourceAnchor, sourceDataNodes)); + + final Collection<DataNode> targetDataNodes = + new ArrayList<>(buildTargetDataNodes(sourceAnchor, xpath, yangResourcesNameToContentMap, targetData)); + + return cpsDeltaService.getDeltaReports(sourceDataNodesRebuilt, targetDataNodes); + } + @Override @Timed(value = "cps.data.service.datanode.descendants.update", description = "Time taken to update a data node and descendants") @@ -223,7 +254,8 @@ public class CpsDataServiceImpl implements CpsDataService { final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); final Anchor anchor = cpsAnchorService.getAnchor(dataspaceName, anchorName); - final Collection<DataNode> dataNodes = buildDataNodes(anchor, parentNodeXpath, jsonData, ContentType.JSON); + final Collection<DataNode> dataNodes = + buildDataNodesWithParentNodeXpath(anchor, parentNodeXpath, jsonData, ContentType.JSON); cpsDataPersistenceService.updateDataNodesAndDescendants(dataspaceName, anchorName, dataNodes); sendDataUpdatedEvent(anchor, parentNodeXpath, Operation.UPDATE, observedTimestamp); } @@ -236,7 +268,7 @@ public class CpsDataServiceImpl implements CpsDataService { final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); final Anchor anchor = cpsAnchorService.getAnchor(dataspaceName, anchorName); - final Collection<DataNode> dataNodes = buildDataNodes(anchor, nodesJsonData); + final Collection<DataNode> dataNodes = buildDataNodesWithParentNodeXpath(anchor, nodesJsonData); cpsDataPersistenceService.updateDataNodesAndDescendants(dataspaceName, anchorName, dataNodes); nodesJsonData.keySet().forEach(nodeXpath -> sendDataUpdatedEvent(anchor, nodeXpath, Operation.UPDATE, observedTimestamp)); @@ -250,7 +282,7 @@ public class CpsDataServiceImpl implements CpsDataService { cpsValidator.validateNameCharacters(dataspaceName, anchorName); final Anchor anchor = cpsAnchorService.getAnchor(dataspaceName, anchorName); final Collection<DataNode> newListElements = - buildDataNodes(anchor, parentNodeXpath, jsonData, ContentType.JSON); + buildDataNodesWithParentNodeXpath(anchor, parentNodeXpath, jsonData, ContentType.JSON); replaceListContent(dataspaceName, anchorName, parentNodeXpath, newListElements, observedTimestamp); } @@ -324,16 +356,68 @@ public class CpsDataServiceImpl implements CpsDataService { sendDataUpdatedEvent(anchor, listNodeXpath, Operation.DELETE, observedTimestamp); } - private Collection<DataNode> buildDataNodes(final Anchor anchor, final Map<String, String> nodesJsonData) { + + private Collection<DataNode> rebuildSourceDataNodes(final String xpath, final Anchor sourceAnchor, + final Collection<DataNode> sourceDataNodes) { + + final Collection<DataNode> sourceDataNodesRebuilt = new ArrayList<>(); + if (sourceDataNodes != null) { + final String sourceDataNodesAsJson = getDataNodesAsJson(sourceAnchor, sourceDataNodes); + sourceDataNodesRebuilt.addAll( + buildDataNodesWithAnchorAndXpath(sourceAnchor, xpath, sourceDataNodesAsJson, ContentType.JSON)); + } + return sourceDataNodesRebuilt; + } + + private Collection<DataNode> buildTargetDataNodes(final Anchor sourceAnchor, final String xpath, + final Map<String, String> yangResourcesNameToContentMap, + final String targetData) { + if (yangResourcesNameToContentMap.isEmpty()) { + return buildDataNodesWithAnchorAndXpath(sourceAnchor, xpath, targetData, ContentType.JSON); + } else { + return buildDataNodesWithYangResourceAndXpath(yangResourcesNameToContentMap, xpath, + targetData, ContentType.JSON); + } + } + + private String getDataNodesAsJson(final Anchor anchor, final Collection<DataNode> dataNodes) { + + final List<Map<String, Object>> prefixToDataNodes = prefixResolver(anchor, dataNodes); + final Map<String, Object> targetDataAsJsonObject = getNodeDataAsJsonString(prefixToDataNodes); + return jsonObjectMapper.asJsonString(targetDataAsJsonObject); + } + + private Map<String, Object> getNodeDataAsJsonString(final List<Map<String, Object>> prefixToDataNodes) { + final Map<String, Object> nodeDataAsJson = new HashMap<>(); + for (final Map<String, Object> prefixToDataNode : prefixToDataNodes) { + nodeDataAsJson.putAll(prefixToDataNode); + } + return nodeDataAsJson; + } + + private List<Map<String, Object>> prefixResolver(final Anchor anchor, final Collection<DataNode> dataNodes) { + final List<Map<String, Object>> prefixToDataNodes = new ArrayList<>(dataNodes.size()); + for (final DataNode dataNode: dataNodes) { + final String prefix = prefixResolver + .getPrefix(anchor.getDataspaceName(), anchor.getName(), dataNode.getXpath()); + final Map<String, Object> prefixToDataNode = DataMapUtils.toDataMapWithIdentifier(dataNode, prefix); + prefixToDataNodes.add(prefixToDataNode); + } + return prefixToDataNodes; + } + + private Collection<DataNode> buildDataNodesWithParentNodeXpath(final Anchor anchor, + final Map<String, String> nodesJsonData) { final Collection<DataNode> dataNodes = new ArrayList<>(); for (final Map.Entry<String, String> nodeJsonData : nodesJsonData.entrySet()) { - dataNodes.addAll(buildDataNodes(anchor, nodeJsonData.getKey(), nodeJsonData.getValue(), ContentType.JSON)); + dataNodes.addAll(buildDataNodesWithParentNodeXpath(anchor, nodeJsonData.getKey(), + nodeJsonData.getValue(), ContentType.JSON)); } return dataNodes; } - private Collection<DataNode> buildDataNodes(final Anchor anchor, final String parentNodeXpath, - final String nodeData, final ContentType contentType) { + private Collection<DataNode> buildDataNodesWithParentNodeXpath(final Anchor anchor, final String parentNodeXpath, + final String nodeData, final ContentType contentType) { if (ROOT_NODE_XPATH.equals(parentNodeXpath)) { final ContainerNode containerNode = yangParser.parseData(contentType, nodeData, anchor, ""); @@ -358,6 +442,63 @@ public class CpsDataServiceImpl implements CpsDataService { return dataNodes; } + private Collection<DataNode> buildDataNodesWithParentNodeXpath( + final Map<String, String> yangResourcesNameToContentMap, final String xpath, + final String nodeData, final ContentType contentType) { + + if (isRootNodeXpath(xpath)) { + final ContainerNode containerNode = yangParser.parseData(contentType, nodeData, + yangResourcesNameToContentMap, ""); + final Collection<DataNode> dataNodes = new DataNodeBuilder() + .withContainerNode(containerNode) + .buildCollection(); + if (dataNodes.isEmpty()) { + throw new DataValidationException("No data nodes.", + "Data nodes were not found under the xpath " + xpath); + } + return dataNodes; + } + final String normalizedParentNodeXpath = CpsPathUtil.getNormalizedXpath(xpath); + final ContainerNode containerNode = + yangParser.parseData(contentType, nodeData, yangResourcesNameToContentMap, normalizedParentNodeXpath); + final Collection<DataNode> dataNodes = new DataNodeBuilder() + .withParentNodeXpath(normalizedParentNodeXpath) + .withContainerNode(containerNode) + .buildCollection(); + if (dataNodes.isEmpty()) { + throw new DataValidationException("No data nodes.", "Data nodes were not found under the xpath " + xpath); + } + return dataNodes; + } + + private Collection<DataNode> buildDataNodesWithAnchorAndXpath(final Anchor anchor, final String xpath, + final String nodeData, + final ContentType contentType) { + + if (!isRootNodeXpath(xpath)) { + final String parentNodeXpath = CpsPathUtil.getNormalizedParentXpath(xpath); + if (parentNodeXpath.isEmpty()) { + return buildDataNodesWithParentNodeXpath(anchor, ROOT_NODE_XPATH, nodeData, contentType); + } + return buildDataNodesWithParentNodeXpath(anchor, parentNodeXpath, nodeData, contentType); + } + return buildDataNodesWithParentNodeXpath(anchor, xpath, nodeData, contentType); + } + + private Collection<DataNode> buildDataNodesWithYangResourceAndXpath( + final Map<String, String> yangResourcesNameToContentMap, final String xpath, + final String nodeData, final ContentType contentType) { + if (!isRootNodeXpath(xpath)) { + final String parentNodeXpath = CpsPathUtil.getNormalizedParentXpath(xpath); + if (parentNodeXpath.isEmpty()) { + return buildDataNodesWithParentNodeXpath(yangResourcesNameToContentMap, ROOT_NODE_XPATH, + nodeData, contentType); + } + return buildDataNodesWithParentNodeXpath(yangResourcesNameToContentMap, parentNodeXpath, + nodeData, contentType); + } + return buildDataNodesWithParentNodeXpath(yangResourcesNameToContentMap, xpath, nodeData, contentType); + } private static boolean isRootNodeXpath(final String xpath) { return ROOT_NODE_XPATH.equals(xpath); diff --git a/cps-service/src/main/java/org/onap/cps/spi/model/DeltaReport.java b/cps-service/src/main/java/org/onap/cps/spi/model/DeltaReport.java index fb9c1971b..34715e70b 100644 --- a/cps-service/src/main/java/org/onap/cps/spi/model/DeltaReport.java +++ b/cps-service/src/main/java/org/onap/cps/spi/model/DeltaReport.java @@ -20,6 +20,7 @@ package org.onap.cps.spi.model; +import com.fasterxml.jackson.annotation.JsonInclude; import java.io.Serializable; import java.util.Map; import lombok.AccessLevel; @@ -28,6 +29,7 @@ import lombok.Setter; @Setter(AccessLevel.PROTECTED) @Getter +@JsonInclude(JsonInclude.Include.NON_NULL) public class DeltaReport { public static final String ADD_ACTION = "add"; diff --git a/cps-service/src/main/java/org/onap/cps/utils/YangParser.java b/cps-service/src/main/java/org/onap/cps/utils/YangParser.java index 6299ef39f..dc23c6bc4 100644 --- a/cps-service/src/main/java/org/onap/cps/utils/YangParser.java +++ b/cps-service/src/main/java/org/onap/cps/utils/YangParser.java @@ -1,6 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2024 Nordix Foundation. + * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,20 +22,25 @@ package org.onap.cps.utils; import io.micrometer.core.annotation.Timed; +import java.util.Map; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.impl.YangTextSchemaSourceSetCache; import org.onap.cps.spi.exceptions.DataValidationException; import org.onap.cps.spi.model.Anchor; +import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder; import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode; import org.opendaylight.yangtools.yang.model.api.SchemaContext; import org.springframework.stereotype.Service; +@Slf4j @Service @RequiredArgsConstructor public class YangParser { private final YangParserHelper yangParserHelper; private final YangTextSchemaSourceSetCache yangTextSchemaSourceSetCache; + private final TimedYangTextSchemaSourceSetBuilder timedYangTextSchemaSourceSetBuilder; /** * Parses data into (normalized) ContainerNode according to schema context for the given anchor. @@ -58,11 +64,33 @@ public class YangParser { return yangParserHelper.parseData(contentType, nodeData, schemaContext, parentNodeXpath); } + /** + * Parses data into (normalized) ContainerNode according to schema context for the given yang resource. + * + * @param nodeData data string + * @param yangResourcesNameToContentMap yang resource to content map + * @return the NormalizedNode object + */ + @Timed(value = "cps.utils.yangparser.nodedata.with.parent.with.yangResourceMap.parse", + description = "Time taken to parse node data with a parent") + public ContainerNode parseData(final ContentType contentType, + final String nodeData, + final Map<String, String> yangResourcesNameToContentMap, + final String parentNodeXpath) { + final SchemaContext schemaContext = getSchemaContext(yangResourcesNameToContentMap); + return yangParserHelper.parseData(contentType, nodeData, schemaContext, parentNodeXpath); + } + private SchemaContext getSchemaContext(final Anchor anchor) { return yangTextSchemaSourceSetCache.get(anchor.getDataspaceName(), anchor.getSchemaSetName()).getSchemaContext(); } + private SchemaContext getSchemaContext(final Map<String, String> yangResourcesNameToContentMap) { + return timedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourcesNameToContentMap) + .getSchemaContext(); + } + private void invalidateCache(final Anchor anchor) { yangTextSchemaSourceSetCache.removeFromCache(anchor.getDataspaceName(), anchor.getSchemaSetName()); } diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy index 4542ecb67..edf25715b 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy @@ -23,6 +23,7 @@ package org.onap.cps.api.impl +import com.fasterxml.jackson.databind.ObjectMapper import ch.qos.logback.classic.Level import ch.qos.logback.classic.Logger import ch.qos.logback.core.read.ListAppender @@ -43,6 +44,9 @@ import org.onap.cps.spi.utils.CpsValidator import org.onap.cps.utils.ContentType import org.onap.cps.utils.YangParser import org.onap.cps.utils.YangParserHelper +import org.onap.cps.utils.JsonObjectMapper +import org.onap.cps.utils.PrefixResolver +import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder import org.onap.cps.yang.YangTextSchemaSourceSet import org.onap.cps.yang.YangTextSchemaSourceSetBuilder import org.slf4j.LoggerFactory @@ -56,11 +60,15 @@ class CpsDataServiceImplSpec extends Specification { def mockCpsAnchorService = Mock(CpsAnchorService) def mockYangTextSchemaSourceSetCache = Mock(YangTextSchemaSourceSetCache) def mockCpsValidator = Mock(CpsValidator) - def yangParser = new YangParser(new YangParserHelper(), mockYangTextSchemaSourceSetCache) + def mockTimedYangTextSchemaSourceSetBuilder = Mock(TimedYangTextSchemaSourceSetBuilder) + def yangParser = new YangParser(new YangParserHelper(), mockYangTextSchemaSourceSetCache, mockTimedYangTextSchemaSourceSetBuilder) def mockCpsDeltaService = Mock(CpsDeltaService); def mockDataUpdateEventsService = Mock(CpsDataUpdateEventsService) + def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper()) + def mockPrefixResolver = Mock(PrefixResolver) - def objectUnderTest = new CpsDataServiceImpl(mockCpsDataPersistenceService, mockDataUpdateEventsService, mockCpsAnchorService, mockCpsValidator, yangParser, mockCpsDeltaService) + def objectUnderTest = new CpsDataServiceImpl(mockCpsDataPersistenceService, mockDataUpdateEventsService, mockCpsAnchorService, + mockCpsValidator, yangParser, mockCpsDeltaService, jsonObjectMapper, mockPrefixResolver) def logger = (Logger) LoggerFactory.getLogger(objectUnderTest.class) def loggingListAppender @@ -230,6 +238,60 @@ class CpsDataServiceImplSpec extends Specification { 1 * mockCpsDeltaService.getDeltaReports(sourceDataNodes, targetDataNodes) } + def 'Get delta between anchor and payload with user provided schema #scenario'() { + given: 'user provided schema set ' + def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') + setupSchemaSetMocksForDelta(yangResourcesNameToContentMap) + when: 'attempt to get delta between an anchor and a JSON payload' + objectUnderTest.getDeltaByDataspaceAnchorAndPayload(dataspaceName, anchorName, xpath, yangResourcesNameToContentMap, jsonData, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) + then: 'dataspacename and anchor names are validated' + 1 * mockCpsValidator.validateNameCharacters(['some-dataspace', 'some-anchor']) + and: 'source data nodes are fetched using appropriate persistence layer method' + 1 * mockCpsDataPersistenceService.getDataNodes(dataspaceName, anchorName, xpath, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) >> sourceDataNodes + and: 'appropriate delta service method is invoked once with correct source and target data nodes' + 1 * mockCpsDeltaService.getDeltaReports({sourceDataNodesRebuilt -> sourceDataNodesRebuilt.xpath[0] == expectedNodeXpath}, {targetDataNodes -> targetDataNodes.xpath[0] == expectedNodeXpath}) + where: 'following data was used' + scenario | xpath | sourceDataNodes | jsonData || expectedNodeXpath + 'root node xpath' | '/' | [new DataNodeBuilder().withXpath('/bookstore').build()] | '{"bookstore":{"bookstore-name":"Easons"}}' || '/bookstore' + 'parent xpath' | '/bookstore' | [new DataNodeBuilder().withXpath('/bookstore').build()] | '{"bookstore":{"bookstore-name":"Easons"}}' || '/bookstore' + 'non-root xpath' | '/bookstore/categories[@code="02"]' | [new DataNodeBuilder().withXpath('/bookstore/categories[@code="02"]').withLeaves(["code":"02"]).build()] | '{"categories":[{"name":"kids","code":"02"}]}' || '/bookstore/categories[@code=\'02\']' + } + + def 'Get delta between anchor and payload by using schema from anchor #scenario'() { + given: 'schema set for a given dataspace and anchor' + setupSchemaSetMocks("bookstore.yang") + when: 'attempt to get delta between an anchor and a JSON payload' + objectUnderTest.getDeltaByDataspaceAnchorAndPayload(dataspaceName, anchorName, xpath, [:], jsonData, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) + then: 'dataspacename and anchor names are validated' + 1 * mockCpsValidator.validateNameCharacters(['some-dataspace', 'some-anchor']) + and: 'source data nodes are fetched using appropriate persistence layer method' + 1 * mockCpsDataPersistenceService.getDataNodes(dataspaceName, anchorName, xpath, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) >> sourceDataNodes + and: 'appropriate delta service method is invoked once with correct source and target data nodes' + 1 * mockCpsDeltaService.getDeltaReports({sourceDataNodesRebuilt -> sourceDataNodesRebuilt.xpath[0] == expectedNodeXpath}, {targetDataNodes -> targetDataNodes.xpath[0] == expectedNodeXpath}) + where: 'following data was used' + scenario | xpath | sourceDataNodes | jsonData || expectedNodeXpath + 'root node xpath' | '/' | [new DataNodeBuilder().withXpath('/bookstore').build()] | '{"bookstore":{"bookstore-name":"Easons"}}' || '/bookstore' + 'parent xpath' | '/bookstore' | [new DataNodeBuilder().withXpath('/bookstore').build()] | '{"bookstore":{"bookstore-name":"Easons"}}' || '/bookstore' + 'non-root xpath' | '/bookstore/categories[@code="02"]' | [new DataNodeBuilder().withXpath('/bookstore/categories[@code="02"]').withLeaves(["code":"02"]).build()] | '{"categories":[{"name":"kids","code":"02"}]}' || '/bookstore/categories[@code=\'02\']' + } + + def 'Delta between anchor and payload error scenario #scenario'() { + given: 'schema set for given anchor and dataspace references bookstore model' + def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') + setupSchemaSetMocksForDelta(yangResourcesNameToContentMap) + when: 'attempt to get delta between anchor and payload' + objectUnderTest.getDeltaByDataspaceAnchorAndPayload(dataspaceName, anchorName, xpath, yangResourcesNameToContentMap, jsonData, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) + then: 'expected exception is thrown' + thrown(DataValidationException) + where: 'following parameters were used' + scenario | xpath | jsonData + 'invalid json data with root node xpath' | '/' | '{"some-key": "some-value"' + 'empty json data with root node xpath' | '/' | '{}' + 'invalid json data with parent node xpath' | '/bookstore' | '{"some-key": "some-value"' + 'empty json data with parent node xpath' | '/bookstore' | '{}' + 'empty json data with xpath' | "/bookstore/categories[@code='02']" | '{}' + } + def 'Update data node leaves: #scenario.'() { given: 'schema set for given anchor and dataspace references test-tree model' setupSchemaSetMocks('test-tree.yang') @@ -503,4 +565,12 @@ class CpsDataServiceImplSpec extends Specification { mockYangTextSchemaSourceSet.getSchemaContext() >> schemaContext } + def setupSchemaSetMocksForDelta(Map<String, String> yangResourcesNameToContentMap) { + def mockYangTextSchemaSourceSet = Mock(YangTextSchemaSourceSet) + mockTimedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourcesNameToContentMap) >> mockYangTextSchemaSourceSet + mockYangTextSchemaSourceSetCache.get(_, _) >> mockYangTextSchemaSourceSet + def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext() + mockYangTextSchemaSourceSet.getSchemaContext() >> schemaContext + } + } diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy index 57f2f8ea7..9e55e8f10 100755 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy @@ -23,6 +23,7 @@ package org.onap.cps.api.impl
+import com.fasterxml.jackson.databind.ObjectMapper
import org.onap.cps.TestUtils
import org.onap.cps.api.CpsAnchorService
import org.onap.cps.api.CpsDeltaService
@@ -31,6 +32,8 @@ import org.onap.cps.spi.CpsDataPersistenceService import org.onap.cps.spi.CpsModulePersistenceService
import org.onap.cps.spi.model.Anchor
import org.onap.cps.spi.utils.CpsValidator
+import org.onap.cps.utils.JsonObjectMapper
+import org.onap.cps.utils.PrefixResolver
import org.onap.cps.utils.ContentType
import org.onap.cps.utils.YangParser
import org.onap.cps.utils.YangParserHelper
@@ -45,15 +48,17 @@ class E2ENetworkSliceSpec extends Specification { def mockYangTextSchemaSourceSetCache = Mock(YangTextSchemaSourceSetCache)
def mockCpsValidator = Mock(CpsValidator)
def timedYangTextSchemaSourceSetBuilder = new TimedYangTextSchemaSourceSetBuilder()
- def yangParser = new YangParser(new YangParserHelper(), mockYangTextSchemaSourceSetCache)
+ def yangParser = new YangParser(new YangParserHelper(), mockYangTextSchemaSourceSetCache, timedYangTextSchemaSourceSetBuilder)
def mockCpsDeltaService = Mock(CpsDeltaService)
+ def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper())
+ def mockPrefixResolver = Mock(PrefixResolver)
def cpsModuleServiceImpl = new CpsModuleServiceImpl(mockModuleStoreService,
mockYangTextSchemaSourceSetCache, mockCpsAnchorService, mockCpsValidator,timedYangTextSchemaSourceSetBuilder)
def mockDataUpdateEventsService = Mock(CpsDataUpdateEventsService)
- def cpsDataServiceImpl = new CpsDataServiceImpl(mockDataStoreService, mockDataUpdateEventsService, mockCpsAnchorService, mockCpsValidator, yangParser, mockCpsDeltaService)
-
+ def cpsDataServiceImpl = new CpsDataServiceImpl(mockDataStoreService, mockDataUpdateEventsService, mockCpsAnchorService, mockCpsValidator,
+ yangParser, mockCpsDeltaService, jsonObjectMapper, mockPrefixResolver)
def dataspaceName = 'someDataspace'
def anchorName = 'someAnchor'
def schemaSetName = 'someSchemaSet'
diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy index 99070fe72..18d0502e3 100644 --- a/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy @@ -1,6 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2024 Nordix Foundation + * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,9 +21,12 @@ package org.onap.cps.utils +import org.onap.cps.TestUtils import org.onap.cps.spi.exceptions.DataValidationException import org.onap.cps.spi.model.Anchor +import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder import org.onap.cps.yang.YangTextSchemaSourceSet +import org.onap.cps.yang.YangTextSchemaSourceSetBuilder import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode import org.opendaylight.yangtools.yang.model.api.SchemaContext import spock.lang.Specification @@ -32,10 +36,12 @@ class YangParserSpec extends Specification { def mockYangParserHelper = Mock(YangParserHelper) def mockYangTextSchemaSourceSetCache = Mock(YangTextSchemaSourceSetCache) + def mockTimedYangTextSchemaSourceSetBuilder = Mock(TimedYangTextSchemaSourceSetBuilder) - def objectUnderTest = new YangParser(mockYangParserHelper, mockYangTextSchemaSourceSetCache) + def objectUnderTest = new YangParser(mockYangParserHelper, mockYangTextSchemaSourceSetCache, mockTimedYangTextSchemaSourceSetBuilder) def anchor = new Anchor(dataspaceName: 'my dataspace', schemaSetName: 'my schema') + def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') def mockYangTextSchemaSourceSet = Mock(YangTextSchemaSourceSet) def mockSchemaContext = Mock(SchemaContext) def containerNodeFromYangUtils = Mock(ContainerNode) @@ -82,4 +88,15 @@ class YangParserSpec extends Specification { 1 * mockYangTextSchemaSourceSetCache.removeFromCache('my dataspace', 'my schema') } + def 'Parsing data with yang resource to context map.'() { + given: 'the schema source set for the yang resource map is returned' + mockTimedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourcesNameToContentMap) >> mockYangTextSchemaSourceSet + when: 'parsing some json data' + def result = objectUnderTest.parseData(ContentType.JSON, 'some json', yangResourcesNameToContentMap, noParent) + then: 'the yang parser helper always returns a container node' + 1 * mockYangParserHelper.parseData(ContentType.JSON, 'some json', mockSchemaContext, noParent) >> containerNodeFromYangUtils + and: 'the result is the same container node as return from yang utils' + assert result == containerNodeFromYangUtils + } + } diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 82a890d79..b9df799c9 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -70,6 +70,8 @@ Bug Fixes Features -------- +3.4.9 + - `CPS-1836 <https://jira.onap.org/browse/CPS-1836>`_ Delta between anchor and JSON payload. Version: 3.4.8 ============== diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/CpsDataServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/CpsDataServiceIntegrationSpec.groovy index 897d4aeb5..779c0b84c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/CpsDataServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/CpsDataServiceIntegrationSpec.groovy @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2023-2024 Nordix Foundation - * Modifications Copyright (C) 2024 TechMahindra Ltd. + * Modifications Copyright (C) 2023-2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -456,7 +456,7 @@ class CpsDataServiceIntegrationSpec extends FunctionalSpecBase { restoreBookstoreDataAnchor(2) } - def 'Get delta between 2 anchors for when #scenario'() { + def 'Get delta between 2 anchors'() { when: 'attempt to get delta report between anchors' def result = objectUnderTest.getDeltaByDataspaceAndAnchors(FUNCTIONAL_TEST_DATASPACE_3, BOOKSTORE_ANCHOR_3, BOOKSTORE_ANCHOR_5, '/', OMIT_DESCENDANTS) then: 'delta report contains expected number of changes' @@ -585,6 +585,46 @@ class CpsDataServiceIntegrationSpec extends FunctionalSpecBase { assert deltaReportEntities.get('targetPayload').containsAll(expectedTargetDataInChildNode) } + def 'Get delta between anchor and JSON payload'() { + when: 'attempt to get delta report between anchor and JSON payload' + def jsonPayload = "{\"book-store:bookstore\":{\"bookstore-name\":\"Crossword Bookstores\"},\"book-store:bookstore-address\":{\"address\":\"Bangalore, India\",\"postal-code\":\"560062\",\"bookstore-name\":\"Crossword Bookstores\"}}" + def result = objectUnderTest.getDeltaByDataspaceAnchorAndPayload(FUNCTIONAL_TEST_DATASPACE_3, BOOKSTORE_ANCHOR_3, '/', [:], jsonPayload, OMIT_DESCENDANTS) + then: 'delta report contains expected number of changes' + result.size() == 3 + and: 'delta report contains UPDATE action with expected xpath' + assert result[0].getAction() == 'update' + assert result[0].getXpath() == '/bookstore' + and: 'delta report contains REMOVE action with expected xpath' + assert result[1].getAction() == 'remove' + assert result[1].getXpath() == "/bookstore-address[@bookstore-name='Easons-1']" + and: 'delta report contains ADD action with expected xpath' + assert result[2].getAction() == 'add' + assert result[2].getXpath() == "/bookstore-address[@bookstore-name='Crossword Bookstores']" + } + + def 'Get delta between anchor and payload returns empty response when JSON payload is identical to anchor data'() { + when: 'attempt to get delta report between anchor and JSON payload (replacing the string Easons with Easons-1 because the data in JSON file is modified, to append anchor number, during the setup process of the integration tests)' + def jsonPayload = readResourceDataFile('bookstore/bookstoreData.json').replace('Easons', 'Easons-1') + def result = objectUnderTest.getDeltaByDataspaceAnchorAndPayload(FUNCTIONAL_TEST_DATASPACE_3, BOOKSTORE_ANCHOR_3, '/', [:], jsonPayload, INCLUDE_ALL_DESCENDANTS) + then: 'delta report is empty' + assert result.isEmpty() + } + + def 'Get delta between anchor and payload error scenario: #scenario'() { + when: 'attempt to get delta between anchor and json payload' + objectUnderTest.getDeltaByDataspaceAnchorAndPayload(dataspaceName, sourceAnchor, xpath, [:], jsonPayload, INCLUDE_ALL_DESCENDANTS) + then: 'expected exception is thrown' + thrown(expectedException) + where: 'following data was used' + scenario | dataspaceName | sourceAnchor | xpath | jsonPayload || expectedException + 'invalid dataspace name' | 'Invalid dataspace' | 'not-relevant' | '/' | '{some-json}' || DataValidationException + 'invalid anchor name' | FUNCTIONAL_TEST_DATASPACE_3 | 'invalid anchor' | '/' | '{some-json}' || DataValidationException + 'non-existing dataspace' | 'non-existing' | 'not-relevant' | '/' | '{some-json}' || DataspaceNotFoundException + 'non-existing anchor' | FUNCTIONAL_TEST_DATASPACE_3 | 'non-existing-anchor' | '/' | '{some-json}' || AnchorNotFoundException + 'empty json payload with root node xpath' | FUNCTIONAL_TEST_DATASPACE_3 | BOOKSTORE_ANCHOR_3 | '/' | '' || DataValidationException + 'empty json payload with non-root node xpath' | FUNCTIONAL_TEST_DATASPACE_3 | BOOKSTORE_ANCHOR_3 | '/bookstore' | '' || DataValidationException + } + def getDeltaReportEntities(List<DeltaReport> deltaReport) { def xpaths = [] def action = [] |