diff options
12 files changed, 276 insertions, 21 deletions
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/datajobs/DataJobResultService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/datajobs/DataJobResultService.java new file mode 100644 index 0000000000..c6b7a8bd04 --- /dev/null +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/datajobs/DataJobResultService.java @@ -0,0 +1,45 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.api.datajobs; + +/** + * Service interface to retrieve the result of a data job. + * The operations interact with a DMI Plugin to retrieve data job results. + */ +public interface DataJobResultService { + + /** + * Retrieves the result of a specific data job. + * + * @param authorization The authorization header from the REST request. + * @param dmiServiceName The name of the DMI Service relevant to the data job. + * @param dataProducerId The ID of the producer registered by DMI, used for operations related to this request. + * This could include alternate IDs or specific identifiers. + * @param dataProducerJobId The identifier of the data producer job within the DMI system. + * @param destination The destination of the results: Kafka topic name or S3 bucket name. + * @return The result of the data job. + */ + String getDataJobResult(final String authorization, + final String dmiServiceName, + final String dataProducerId, + final String dataProducerJobId, + final String destination); +} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/datajobs/DataJobResultServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/datajobs/DataJobResultServiceImpl.java new file mode 100644 index 0000000000..031cedc78b --- /dev/null +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/datajobs/DataJobResultServiceImpl.java @@ -0,0 +1,55 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.impl.datajobs; + +import lombok.RequiredArgsConstructor; +import org.onap.cps.ncmp.api.datajobs.DataJobResultService; +import org.onap.cps.ncmp.impl.dmi.DmiProperties; +import org.onap.cps.ncmp.impl.dmi.DmiRestClient; +import org.onap.cps.ncmp.impl.dmi.DmiServiceUrlTemplateBuilder; +import org.onap.cps.ncmp.impl.dmi.UrlTemplateParameters; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class DataJobResultServiceImpl implements DataJobResultService { + + private final DmiRestClient dmiRestClient; + private final DmiProperties dmiProperties; + + @Override + public String getDataJobResult(final String authorization, + final String dmiServiceName, + final String dataProducerId, + final String dataProducerJobId, + final String destination) { + final UrlTemplateParameters urlTemplateParameters = DmiServiceUrlTemplateBuilder.newInstance() + .fixedPathSegment("cmwriteJob") + .fixedPathSegment("dataProducer") + .variablePathSegment("dataProducerId", dataProducerId) + .fixedPathSegment("dataProducerJob") + .variablePathSegment("dataProducerJobId", dataProducerJobId) + .fixedPathSegment("result") + .queryParameter("destination", destination) + .createUrlTemplateParameters(dmiServiceName, dmiProperties.getDmiBasePath()); + return dmiRestClient.getDataJobResult(urlTemplateParameters, authorization).block(); + } +} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/dmi/DmiRestClient.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/dmi/DmiRestClient.java index 177b4b0bf2..c10132060d 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/dmi/DmiRestClient.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/dmi/DmiRestClient.java @@ -164,6 +164,24 @@ public class DmiRestClient { .onErrorMap(throwable -> handleDmiClientException(throwable, OperationType.READ.getOperationName())); } + /** + * Retrieves the result of a data job from the DMI service. + * + * @param urlTemplateParameters The URL template parameters for the DMI data job status endpoint. + * @param authorization The authorization token to be added to the request headers. + * @return A Mono emitting the result of the data job as a String. + * @throws DmiClientRequestException If there is an error during the DMI request. + */ + public Mono<String> getDataJobResult(final UrlTemplateParameters urlTemplateParameters, + final String authorization) { + return dataServicesWebClient.get() + .uri(urlTemplateParameters.urlTemplate(), urlTemplateParameters.urlVariables()) + .headers(httpHeaders -> configureHttpHeaders(httpHeaders, authorization)) + .retrieve().bodyToMono(String.class) + .onErrorMap(throwable -> handleDmiClientException(throwable, + OperationType.READ.getOperationName())); + } + private WebClient getWebClient(final RequiredDmiService requiredDmiService) { return requiredDmiService.equals(RequiredDmiService.DATA) ? dataServicesWebClient : modelServicesWebClient; } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/datajobs/DataJobResultServiceImplSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/datajobs/DataJobResultServiceImplSpec.groovy new file mode 100644 index 0000000000..3af474040e --- /dev/null +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/datajobs/DataJobResultServiceImplSpec.groovy @@ -0,0 +1,55 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.impl.datajobs + +import org.onap.cps.ncmp.impl.dmi.DmiProperties +import org.onap.cps.ncmp.impl.dmi.DmiRestClient +import org.onap.cps.ncmp.impl.dmi.UrlTemplateParameters +import reactor.core.publisher.Mono +import spock.lang.Specification + +class DataJobResultServiceImplSpec extends Specification { + + def mockDmiRestClient = Mock(DmiRestClient) + def mockDmiProperties = Mock(DmiProperties) + def objectUnderTest = new DataJobResultServiceImpl(mockDmiRestClient, mockDmiProperties) + + def setup() { + mockDmiProperties.dmiBasePath >> 'dmi' + } + + def 'Retrieve data job result.'() { + given: 'the required parameters for querying' + def dmiServiceName = 'some-dmi-service' + def dataProducerJobId = 'some-data-producer-job-id' + def dataProducerId = 'some-data-producer-id' + def authorization = 'my authorization header' + def destination = 'some-destination' + def urlParams = new UrlTemplateParameters('some-dmi-service/dmi/v1/cmwriteJob/dataProducer/{dataProducerId}/dataProducerJob/{dataProducerJobId}/result?destination={destination}', ['dataProducerJobId':'some-data-producer-job-id', 'dataProducerId':'some-data-producer-id', 'destination': 'some-destination']) + and: 'the rest client returns the result for the given parameters' + mockDmiRestClient.getDataJobResult(urlParams, authorization) >> Mono.just('some result') + when: 'the job status is queried' + def result = objectUnderTest.getDataJobResult(authorization, dmiServiceName,dataProducerId, dataProducerJobId, destination) + then: 'the result from the rest client is returned' + assert result != null + assert result == 'some result' + } +} diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/dmi/DmiRestClientSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/dmi/DmiRestClientSpec.groovy index 3444d7b86a..d92e69a136 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/dmi/DmiRestClientSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/dmi/DmiRestClientSpec.groovy @@ -175,4 +175,16 @@ class DmiRestClientSpec extends Specification { then: 'the response equals to the expected value' assert response == 'some status' } + + def 'Get data job result from DMI.'() { + given: 'the Data web client returns a valid response entity for the expected parameters' + mockDataServicesWebClient.get() >> mockRequestBody + def result = 'some result' + mockResponse.bodyToMono(String.class) >> Mono.just(result) + when: 'GET operation is invoked for Data Service' + def response = objectUnderTest.getDataJobResult(urlTemplateParameters, NO_AUTH_HEADER).block() + then: 'the response has some value' + assert response != null + assert result == 'some result' + } } diff --git a/csit/prepare-csit.sh b/csit/prepare-csit.sh index fbd5dc5f0d..1b8578e0ce 100755 --- a/csit/prepare-csit.sh +++ b/csit/prepare-csit.sh @@ -71,7 +71,7 @@ echo "Versioning information:" python3 --version echo "Installing confluent kafka library for robot framework:" -pip install robotframework-confluentkafkalibrary +pip install robotframework-confluentkafkalibrary==2.4.0-2 pip freeze python3 -m robot.run --version || :
\ No newline at end of file diff --git a/csit/pylibs.txt b/csit/pylibs.txt index 32bfa6faca..3eeb1ab9ec 100644 --- a/csit/pylibs.txt +++ b/csit/pylibs.txt @@ -9,7 +9,7 @@ robotframework-requests==0.9.3 robotframework-selenium2library==3.0.0 robotframework-extendedselenium2library robotframework-sshlibrary -robotframework-confluentkafkalibrary +robotframework-confluentkafkalibrary==2.4.0-2 scapy # Module jsonpath is needed by current AAA idmlite suite. jsonpath-rw diff --git a/csit/tests/cps-data-operations/cps-data-operations.robot b/csit/tests/cps-data-operations/cps-data-operations.robot index 85e8a2a857..96212ff632 100644 --- a/csit/tests/cps-data-operations/cps-data-operations.robot +++ b/csit/tests/cps-data-operations/cps-data-operations.robot @@ -26,6 +26,7 @@ Library OperatingSystem Library RequestsLibrary Library BuiltIn Library ConfluentKafkaLibrary +Library String Suite Setup Create Session CPS_URL http://${CPS_CORE_HOST}:${CPS_CORE_PORT} @@ -63,8 +64,7 @@ Consume cloud event from client topic Compare Header Values ${header_key_value_pair[0]} ${header_key_value_pair[1]} "ce_specversion" "1.0" Compare Header Values ${header_key_value_pair[0]} ${header_key_value_pair[1]} "ce_type" "org.onap.cps.ncmp.events.async1_0_0.DataOperationEvent" Compare Header Values ${header_key_value_pair[0]} ${header_key_value_pair[1]} "ce_correlationid" "${expectedRequestId}" - # Need to check the root cause of this failure. To be investigated separately as part of CPS-2363 - # Compare Header Values ${header_key_value_pair[0]} ${header_key_value_pair[1]} "ce_source" "DMI" + Compare Header Values ${header_key_value_pair[0]} ${header_key_value_pair[1]} "ce_source" "DMI" END [Teardown] Basic Teardown ${group_id} @@ -79,12 +79,20 @@ Is CM Handle READY [Arguments] ${uri} ${headers} ${cmHandle} ${response}= GET On Session CPS_URL ${uri} headers=${headers} Should Be Equal As Strings ${response.status_code} 200 + ${number_of_items}= Count Items In JSON Response ${response} + Should Be True ${number_of_items} > 0 FOR ${item} IN ${response.json()} IF "${item['cmHandle']}" == "${cmHandle}" Should Be Equal As Strings ${item['state']['cmHandleState']} READY END END +Count Items In JSON Response + [Arguments] ${response} + ${json_data}= Evaluate json.loads('${response.content.decode("utf-8")}') json + ${number_of_items}= Get Length ${json_data} + RETURN ${number_of_items} + Basic Teardown [Arguments] ${group_id} Unsubscribe ${group_id} diff --git a/csit/tests/cps-data-sync/cps-data-sync.robot b/csit/tests/cps-data-sync/cps-data-sync.robot index 6fc1876421..b8ba479e7c 100644 --- a/csit/tests/cps-data-sync/cps-data-sync.robot +++ b/csit/tests/cps-data-sync/cps-data-sync.robot @@ -35,11 +35,6 @@ ${ncmpBasePath} /ncmp *** Test Cases *** -Check if ietfYang-PNFDemo is READY - ${uri}= Set Variable ${ncmpBasePath}/v1/ch/ietfYang-PNFDemo - ${headers}= Create Dictionary Authorization=${auth} - Wait Until Keyword Succeeds 20sec 200ms Is CM Handle READY ${uri} ${headers} ietfYang-PNFDemo - Operational state goes to UNSYNCHRONIZED when data sync (flag) is enabled ${uri}= Set Variable ${ncmpBasePath}/v1/ch/ietfYang-PNFDemo/data-sync ${params}= Create Dictionary dataSyncEnabled=true @@ -57,15 +52,6 @@ Operational state goes to SYNCHRONIZED after sometime when data sync (flag) is e Wait Until Keyword Succeeds 40sec 100ms Is CM Handle State SYNCHRONIZED ${uri} ${headers} *** Keywords *** -Is CM Handle READY - [Arguments] ${uri} ${headers} ${cmHandle} - ${response}= GET On Session CPS_URL ${uri} headers=${headers} - Should Be Equal As Strings ${response.status_code} 200 - FOR ${item} IN ${response.json()} - IF "${item['cmHandle']}" == "${cmHandle}" - Should Be Equal As Strings ${item['state']['cmHandleState']} READY - END - END Is CM Handle State SYNCHRONIZED [Arguments] ${uri} ${headers} diff --git a/csit/tests/cps-model-sync/cps-model-sync.robot b/csit/tests/cps-model-sync/cps-model-sync.robot index bb881f6a67..514076f085 100644 --- a/csit/tests/cps-model-sync/cps-model-sync.robot +++ b/csit/tests/cps-model-sync/cps-model-sync.robot @@ -25,6 +25,7 @@ Library Collections Library OperatingSystem Library RequestsLibrary Library BuiltIn +Library String Suite Setup Create Session CPS_URL http://${CPS_CORE_HOST}:${CPS_CORE_PORT} @@ -88,13 +89,40 @@ Get cm handle details and confirm it has been deleted ${headers}= Create Dictionary Authorization=${auth} ${response}= GET On Session CPS_URL ${uri} headers=${headers} expected_status=404 +Check if ietfYang-PNFDemo is READY + ${uri}= Set Variable ${ncmpBasePath}/v1/ch/ietfYang-PNFDemo + ${headers}= Create Dictionary Authorization=${auth} + Wait Until Keyword Succeeds 20sec 200ms Is CM Handle READY ${uri} ${headers} ietfYang-PNFDemo + Get modules for registered data node ${uri}= Set Variable ${ncmpBasePath}/v1/ch/ietfYang-PNFDemo/modules ${headers}= Create Dictionary Authorization=${auth} ${response}= GET On Session CPS_URL ${uri} headers=${headers} Should Be Equal As Strings ${response.status_code} 200 + ${number_of_items}= Count Items In JSON Response ${response} + Should Be True ${number_of_items} > 0 FOR ${item} IN @{response.json()} IF "${item['moduleName']}" == "stores" Should Be Equal As Strings "${item['revision']}" "2020-09-15" END - END
\ No newline at end of file + END + +*** Keywords *** + +Is CM Handle READY + [Arguments] ${uri} ${headers} ${cmHandle} + ${response}= GET On Session CPS_URL ${uri} headers=${headers} + Should Be Equal As Strings ${response.status_code} 200 + ${number_of_items}= Count Items In JSON Response ${response} + Should Be True ${number_of_items} > 0 + FOR ${item} IN ${response.json()} + IF "${item['cmHandle']}" == "${cmHandle}" + Should Be Equal As Strings ${item['state']['cmHandleState']} READY + END + END + +Count Items In JSON Response + [Arguments] ${response} + ${json_data}= Evaluate json.loads('${response.content.decode("utf-8")}') json + ${number_of_items}= Get Length ${json_data} + RETURN ${number_of_items}
\ No newline at end of file diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy index 56d8f19e64..fcc23db782 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy @@ -20,8 +20,6 @@ package org.onap.cps.integration.base -import org.onap.cps.ncmp.api.datajobs.models.SubJobWriteRequest - import static org.onap.cps.integration.base.CpsIntegrationSpecBase.readResourceDataFile import groovy.json.JsonSlurper @@ -29,6 +27,7 @@ import java.util.regex.Matcher import okhttp3.mockwebserver.Dispatcher import okhttp3.mockwebserver.MockResponse import okhttp3.mockwebserver.RecordedRequest +import org.onap.cps.ncmp.api.datajobs.models.SubJobWriteRequest import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus import org.springframework.http.MediaType @@ -97,6 +96,10 @@ class DmiDispatcher extends Dispatcher { case ~'^/dmi/v1/cmwriteJob/dataProducer/(.*)/dataProducerJob/(.*)/status$': return mockResponseWithBody(HttpStatus.OK, '{"status":"status details from mock service"}') + // get data job result + case ~'^/dmi/v1/cmwriteJob/dataProducer/(.*)/dataProducerJob/(.*)/result(.*)$': + return mockResponseWithBody(HttpStatus.OK, '{ "result": "some result"}') + // get write sub job response case ~'^/dmi/v1/cmwriteJob(.*)$': return mockWriteJobResponse(request) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/DataJobResultServiceSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/DataJobResultServiceSpec.groovy new file mode 100644 index 0000000000..241d31a642 --- /dev/null +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/DataJobResultServiceSpec.groovy @@ -0,0 +1,45 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.integration.functional.ncmp + +import org.onap.cps.integration.base.CpsIntegrationSpecBase +import org.onap.cps.ncmp.api.datajobs.DataJobResultService +import org.springframework.beans.factory.annotation.Autowired + +class DataJobResultServiceSpec extends CpsIntegrationSpecBase { + + @Autowired + DataJobResultService dataJobResultService; + + def 'Get the status of a data job from DMI.'() { + given: 'the required data about the data job' + def authorization = 'my authorization header' + def dmiServiceName = DMI1_URL + def dataProducerId = 'some-data-producer-id' + def dataProducerJobId = 'some-data-producer-job-id' + def destination = 'some-destination' + when: 'the data job status checked' + def result = dataJobResultService.getDataJobResult(authorization, dmiServiceName, dataProducerId, dataProducerJobId, destination) + then: 'the status is that defined in the mock service.' + assert result != null + assert result == '{ "result": "some result"}' + } +} |