diff options
14 files changed, 533 insertions, 65 deletions
diff --git a/Changelog.md b/Changelog.md index d678e9bc..72a8b534 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.5.4] - 23/02/2021 +### Added +- JWT token support for HTTP/HTTPS +### Changed +- FileData / FileServerData prepared to store uri elements + ## [1.5.3] - 11/02/2021 ### Added - HTTPS support for DFC diff --git a/datafile-app-server/pom.xml b/datafile-app-server/pom.xml index 5e0330b0..64b6f656 100644 --- a/datafile-app-server/pom.xml +++ b/datafile-app-server/pom.xml @@ -26,7 +26,7 @@ <parent> <groupId>org.onap.dcaegen2.collectors</groupId> <artifactId>datafile</artifactId> - <version>1.5.3-SNAPSHOT</version> + <version>1.5.4-SNAPSHOT</version> </parent> <groupId>org.onap.dcaegen2.collectors.datafile</groupId> @@ -116,6 +116,10 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-autoconfigure</artifactId> </dependency> + <dependency> + <groupId>org.apache.httpcomponents.core5</groupId> + <artifactId>httpcore5</artifactId> + </dependency> <!-- Actuator dependencies --> <dependency> diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java index 32241fdb..2c44abb5 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START====================================================================== * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Modifications copyright (C) 2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -16,12 +17,15 @@ package org.onap.dcaegen2.collectors.datafile.commons; +import java.util.List; import java.util.Optional; +import org.apache.hc.core5.http.NameValuePair; import org.immutables.value.Value; /** * Data about the file server to collect a file from. + * In case of http protocol it also contains data required to recreate target uri * * @author <a href="mailto:henrik.b.andersson@est.tech">Henrik Andersson</a> * @@ -37,4 +41,9 @@ public interface FileServerData { public String password(); public Optional<Integer> port(); + + @Value.Redacted + public Optional<List<NameValuePair>> queryParameters(); + + public Optional<String> uriRawFragment(); } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java index 3ccc9fb2..4564a44f 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.NotNull; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; import org.onap.dcaegen2.collectors.datafile.service.HttpUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,17 +62,22 @@ public class DfcHttpClient implements FileCollectClient { @Override public void open() throws DatafileTaskException { logger.trace("Setting httpClient for file download."); - basicAuthDataPresentOrThrow(); + String authorizationContent = getAuthorizationContent(); this.client = HttpClient.create(pool).keepAlive(true).headers( - h -> h.add("Authorization", HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password()))); + h -> h.add("Authorization", authorizationContent)); logger.trace("httpClient, auth header was set."); } - private void basicAuthDataPresentOrThrow() throws DatafileTaskException { - if ((this.fileServerData.userId().isEmpty()) || (this.fileServerData.password().isEmpty())) { + protected String getAuthorizationContent() throws DatafileTaskException { + String jwtToken = HttpUtils.getJWTToken(fileServerData); + if (!jwtToken.isEmpty()) { + return HttpUtils.jwtAuthContent(jwtToken); + } + if (!HttpUtils.isBasicAuthDataFilled(fileServerData)) { throw new DatafileTaskException("Not sufficient basic auth data for file."); } + return HttpUtils.basicAuthContent(this.fileServerData.userId(), this.fileServerData.password()); } @Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException { @@ -92,7 +98,10 @@ public class DfcHttpClient implements FileCollectClient { } if (isDownloadFailed(errorMessage)) { - throw new DatafileTaskException("Error occured during datafile download: ", errorMessage.get()); + if (errorMessage.get() instanceof NonRetryableDatafileTaskException) { + throw (NonRetryableDatafileTaskException) errorMessage.get(); + } + throw (DatafileTaskException) errorMessage.get(); } logger.trace("HTTP collectFile OK"); @@ -104,7 +113,11 @@ public class DfcHttpClient implements FileCollectClient { @NotNull protected Consumer<Throwable> processFailedConnectionWithServer(CountDownLatch latch, AtomicReference<Exception> errorMessages) { return (Throwable response) -> { - errorMessages.set(new Exception("Error in connection has occurred during file download", response)); + Exception e = new Exception("Error in connection has occurred during file download", response); + errorMessages.set(new DatafileTaskException(response.getMessage(), e)); + if (response instanceof NonRetryableDatafileTaskException) { + errorMessages.set(new NonRetryableDatafileTaskException(response.getMessage(), e)); + } latch.countDown(); }; } @@ -119,7 +132,7 @@ public class DfcHttpClient implements FileCollectClient { logger.trace("CollectFile fetched: {}", localFile); response.close(); } catch (IOException e) { - errorMessages.set(new Exception("Error fetching file with", e)); + errorMessages.set(new DatafileTaskException("Error fetching file with", e)); } finally { latch.countDown(); } @@ -128,24 +141,31 @@ public class DfcHttpClient implements FileCollectClient { protected Flux<InputStream> getServerResponse(String remoteFile) { return client.get() - .uri(prepareUri(remoteFile)) + .uri(HttpUtils.prepareHttpUri(fileServerData, remoteFile)) .response((responseReceiver, byteBufFlux) -> { logger.trace("HTTP response status - {}", responseReceiver.status()); if(isResponseOk(responseReceiver)){ return byteBufFlux.aggregate().asInputStream(); } - return Mono.error(new Throwable("Unexpected server response code - " - + responseReceiver.status().toString())); + if (isErrorInConnection(responseReceiver)) { + return Mono.error(new NonRetryableDatafileTaskException( + HttpUtils.nonRetryableResponse(getResponseCode(responseReceiver)))); + } + return Mono.error(new DatafileTaskException( + HttpUtils.retryableResponse(getResponseCode(responseReceiver)))); }); } protected boolean isResponseOk(HttpClientResponse httpClientResponse) { - return httpClientResponse.status().code() == 200; + return getResponseCode(httpClientResponse) == 200; + } + + private int getResponseCode(HttpClientResponse responseReceiver) { + return responseReceiver.status().code(); } - @NotNull protected String prepareUri(String remoteFile) { - int port = fileServerData.port().orElse(HttpUtils.HTTP_DEFAULT_PORT); - return "http://" + fileServerData.serverAddress() + ":" + port + remoteFile; + protected boolean isErrorInConnection(HttpClientResponse httpClientResponse) { + return getResponseCode(httpClientResponse) >= 400; } @Override public void close() { diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java index 3090815a..c2d72f67 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java @@ -27,7 +27,6 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; -import org.jetbrains.annotations.NotNull; import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; @@ -85,10 +84,11 @@ public class DfcHttpsClient implements FileCollectClient { @Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException { logger.trace("Prepare to collectFile {}", localFile); - HttpGet httpGet = new HttpGet(prepareUri(remoteFile)); - if (basicAuthValidNotPresentOrThrow()) { - httpGet.addHeader("Authorization", - HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password())); + HttpGet httpGet = new HttpGet(HttpUtils.prepareHttpsUri(fileServerData, remoteFile)); + + String authorizationContent = getAuthorizationContent(); + if (!authorizationContent.isEmpty()) { + httpGet.addHeader("Authorization", authorizationContent); } try { HttpResponse httpResponse = makeCall(httpGet); @@ -99,11 +99,23 @@ public class DfcHttpsClient implements FileCollectClient { logger.trace("HTTPS collectFile OK"); } + private String getAuthorizationContent() throws DatafileTaskException { + String jwtToken = HttpUtils.getJWTToken(fileServerData); + if (shouldUseBasicAuth(jwtToken)) { + return HttpUtils.basicAuthContent(this.fileServerData.userId(), this.fileServerData.password()); + } + return HttpUtils.jwtAuthContent(jwtToken); + } + + private boolean shouldUseBasicAuth(String jwtToken) throws DatafileTaskException { + return basicAuthValidNotPresentOrThrow() && jwtToken.isEmpty(); + } + protected boolean basicAuthValidNotPresentOrThrow() throws DatafileTaskException { if (isAuthDataEmpty()) { return false; } - if (isAuthDataFilled()) { + if (HttpUtils.isBasicAuthDataFilled(fileServerData)) { return true; } throw new DatafileTaskException("Not sufficient basic auth data for file."); @@ -113,15 +125,6 @@ public class DfcHttpsClient implements FileCollectClient { return this.fileServerData.userId().isEmpty() && this.fileServerData.password().isEmpty(); } - private boolean isAuthDataFilled() { - return !this.fileServerData.userId().isEmpty() && !this.fileServerData.password().isEmpty(); - } - - @NotNull protected String prepareUri(String remoteFile) { - int port = fileServerData.port().orElse(HttpUtils.HTTPS_DEFAULT_PORT); - return "https://" + fileServerData.serverAddress() + ":" + port + remoteFile; - } - protected HttpResponse makeCall(HttpGet httpGet) throws IOException, DatafileTaskException { try { @@ -131,10 +134,10 @@ public class DfcHttpsClient implements FileCollectClient { } EntityUtils.consume(httpResponse.getEntity()); - throw new NonRetryableDatafileTaskException( - "Unexpected response code - " + httpResponse.getStatusLine().getStatusCode() - + ". No retry attempts will be done."); - + if (isErrorInConnection(httpResponse)) { + throw new NonRetryableDatafileTaskException(HttpUtils.retryableResponse(getResponseCode(httpResponse))); + } + throw new DatafileTaskException(HttpUtils.nonRetryableResponse(getResponseCode(httpResponse))); } catch (ConnectTimeoutException | UnknownHostException | HttpHostConnectException | SSLHandshakeException e) { throw new NonRetryableDatafileTaskException( "Unable to get file from xNF. No retry attempts will be done.", e); @@ -146,8 +149,16 @@ public class DfcHttpsClient implements FileCollectClient { return httpsClient.execute(httpGet); } - protected boolean isResponseOk(HttpResponse response) { - return response.getStatusLine().getStatusCode() == 200; + protected boolean isResponseOk(HttpResponse httpResponse) { + return getResponseCode(httpResponse) == 200; + } + + private int getResponseCode(HttpResponse httpResponse) { + return httpResponse.getStatusLine().getStatusCode(); + } + + protected boolean isErrorInConnection(HttpResponse httpResponse) { + return getResponseCode(httpResponse) >= 400; } protected void processResponse(HttpResponse response, Path localFile) throws IOException { diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java index 1c8f57da..e7d2e15c 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START======================================================= * Copyright (C) 2019 Nordix Foundation. + * Copyright (C) 2021 Nokia. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +24,11 @@ package org.onap.dcaegen2.collectors.datafile.model; import java.net.URI; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import java.util.Optional; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.net.URIBuilder; import org.immutables.gson.Gson; import org.immutables.value.Value; import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; @@ -101,6 +105,7 @@ public abstract class FileData { /** * Get the data about the file server where the file should be collected from. + * Query data included as it can contain JWT token * * @return the data about the file server where the file should be collected from. */ @@ -114,6 +119,15 @@ public abstract class FileData { if (uri.getPort() > 0) { builder.port(uri.getPort()); } + URIBuilder uriBuilder = new URIBuilder(uri); + List<NameValuePair> query = uriBuilder.getQueryParams(); + if (query != null && !query.isEmpty()) { + builder.queryParameters(query); + } + String fragment = uri.getRawFragment(); + if (fragment != null && fragment.length() > 0) { + builder.uriRawFragment(fragment); + } return builder.build(); } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java index e2c1e2ff..69bfcf47 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START====================================================================== * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. - * Modifications Copyright (C) 2020 Nokia. All rights reserved + * Modifications Copyright (C) 2020-2021 Nokia. All rights reserved * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,23 +19,208 @@ package org.onap.dcaegen2.collectors.datafile.service; +import org.apache.hc.core5.http.NameValuePair; import org.apache.http.HttpStatus; +import org.jetbrains.annotations.NotNull; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import java.util.ArrayList; import java.util.Base64; +import java.util.List; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class HttpUtils implements HttpStatus { + private static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); public static final int HTTP_DEFAULT_PORT = 80; public static final int HTTPS_DEFAULT_PORT = 443; + public static final String JWT_TOKEN_NAME = "access_token"; + public static final String AUTH_JWT_WARN = "Both JWT token and Basic auth data present. Omitting basic auth info."; + public static final String AUTH_JWT_ERROR = "More than one JWT token present in the queryParameters. Omitting JWT token."; private HttpUtils() { } - public static boolean isSuccessfulResponseCode(Integer statusCode) { + @NotNull + public static String nonRetryableResponse(int responseCode) { + return "Unexpected response code - " + responseCode; + } + + @NotNull + public static String retryableResponse(int responseCode) { + return "Unexpected response code - " + responseCode + ". No retry attempts will be done."; + } + + public static boolean isSuccessfulResponseCodeWithDataRouter(Integer statusCode) { return statusCode >= 200 && statusCode < 300; } - public static String basicAuth(String username, String password) { + public static boolean isBasicAuthDataFilled(final FileServerData fileServerData) { + return !fileServerData.userId().isEmpty() && !fileServerData.password().isEmpty(); + } + + public static String basicAuthContent(String username, String password) { return "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes()); } + + public static String jwtAuthContent(String token) { + return "Bearer " + token; + } + + /** + * Prepare uri to retrieve file from xNF using HTTP connection. If JWT token was included + * in the queryParameters, it is removed. Other entries are rewritten. + * + * @param fileServerData fileServerData including - server address, port, queryParameters and uriRawFragment + * @param remoteFile file which has to be downloaded + * @return uri String representing the xNF HTTP location + */ + @NotNull public static String prepareHttpUri(FileServerData fileServerData, String remoteFile){ + return prepareUri("http", fileServerData, remoteFile, HTTP_DEFAULT_PORT); + } + + /** + * Prepare uri to retrieve file from xNF using HTTPS connection. If JWT token was included + * in the queryParameters, it is removed. Other entries are rewritten. + * + * @param fileServerData fileServerData including - server address, port, queryParameters and uriRawFragment + * @param remoteFile file which has to be downloaded + * @return uri String representing the xNF HTTPS location + */ + @NotNull public static String prepareHttpsUri(FileServerData fileServerData, String remoteFile){ + return prepareUri("https", fileServerData, remoteFile, HTTPS_DEFAULT_PORT); + } + + /** + * Prepare uri to retrieve file from xNF. If JWT token was included + * in the queryParameters, it is removed. Other entries are rewritten. + * + * @param scheme scheme which is used during the connection + * @param fileServerData fileServerData including - server address, port, query and fragment + * @param remoteFile file which has to be downloaded + * @param defaultPort default port which exchange empty entry for given connection type + * @return uri String representing the xNF location + */ + @NotNull public static String prepareUri(String scheme, FileServerData fileServerData, String remoteFile, int defaultPort) { + int port = fileServerData.port().orElse(defaultPort); + String query = rewriteQueryWithoutToken(fileServerData.queryParameters().orElse(new ArrayList<>())); + String fragment = fileServerData.uriRawFragment().orElse(""); + if (!query.isEmpty()) { + query = "?" + query; + } + if (!fragment.isEmpty()) { + fragment = "#" + fragment; + } + return scheme + "://" + fileServerData.serverAddress() + ":" + port + remoteFile + query + fragment; + } + + /** + * Returns JWT token string (if single exist) from the queryParameters. + * + * @param fileServerData file server data which contain queryParameters where JWT token may exist + * @return JWT token value if single token entry exist or empty string elsewhere. + * If JWT token key has no value, empty string will be returned. + */ + public static String getJWTToken(FileServerData fileServerData) { + Optional<List<NameValuePair>> query = fileServerData.queryParameters(); + if (!query.isPresent()) { + return ""; + } + List<NameValuePair> queryParameters = query.get(); + if (queryParameters.isEmpty()) { + return ""; + } + boolean jwtTokenKeyPresent = HttpUtils.isQueryWithSingleJWT(queryParameters); + if (!jwtTokenKeyPresent) { + return ""; + } + String token = HttpUtils.getJWTToken(query.get()); + if (HttpUtils.isBasicAuthDataFilled(fileServerData)) { + logger.warn(HttpUtils.AUTH_JWT_WARN); + } + return token; + } + + /** + * Checks if the queryParameters contains single JWT token entry. Valid queryParameters + * contains only one token entry. + * + * @param query queryParameters + * @return true if queryParameters contains single token + */ + public static boolean isQueryWithSingleJWT(List<NameValuePair> query) { + if (query == null) { + return false; + } + int i = getJWTTokenCount(query); + if (i == 0) { + return false; + } + if (i > 1) { + logger.error(AUTH_JWT_ERROR); + return false; + } + return true; + } + + /** + * Returns the number of JWT token entries. Valid queryParameters contains only one token entry. + * + * @param queryElements elements of the queryParameters + * @return true if queryParameters contains single JWT token entry + */ + public static int getJWTTokenCount(List<NameValuePair> queryElements) { + int i = 0; + for (NameValuePair element : queryElements) { + if (element.getName().equals(JWT_TOKEN_NAME)) { + i++; + } + } + return i; + } + + private static String getJWTToken(List<NameValuePair> query) { + for (NameValuePair element : query) { + if (!element.getName().equals(JWT_TOKEN_NAME)) { + continue; + } + if (element.getValue() != null) { + return element.getValue(); + } + return ""; + } + return ""; + } + + /** + * Rewrites HTTP queryParameters without JWT token + * + * @param query list of NameValuePair of elements sent in the queryParameters + * @return String representation of queryParameters elements which were provided in the input + * Empty string is possible when queryParameters is empty or contains only access_token key. + */ + public static String rewriteQueryWithoutToken(List<NameValuePair> query) { + if (query.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (NameValuePair nvp : query) { + if (nvp.getName().equals(JWT_TOKEN_NAME)) { + continue; + } + sb.append(nvp.getName()); + if (nvp.getValue() != null) { + sb.append("="); + sb.append(nvp.getValue()); + } + sb.append("&"); + } + if ((sb.length() > 0) && (sb.charAt(sb.length() - 1 ) == '&')) { + sb.deleteCharAt(sb.length() - 1 ); + } + return sb.toString(); + } } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java index c24c6c96..ef2341fb 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START======================================================= * Copyright (C) 2019 Nordix Foundation. - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -133,7 +133,7 @@ public class DataRouterPublisher { private Mono<FilePublishInformation> handleHttpResponse(HttpStatus response, FilePublishInformation publishInfo) { MDC.setContextMap(publishInfo.getContext()); - if (HttpUtils.isSuccessfulResponseCode(response.value())) { + if (HttpUtils.isSuccessfulResponseCodeWithDataRouter(response.value())) { counters.incTotalPublishedFiles(); logger.trace("Publishing file {} to DR successful!", publishInfo.getName()); return Mono.just(publishInfo); diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java index cfc77549..70380437 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START====================================================================== * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -111,7 +111,7 @@ public class FileCollector { counters.incNoOfCollectedFiles(); return Mono.just(Optional.of(getFilePublishInformation(fileData, localFile, context))); } catch (NonRetryableDatafileTaskException nre) { - logger.warn("Failed to download file: {} {}, reason: {}", fileData.sourceName(), fileData.name(), nre); + logger.warn("Failed to download file: {} {}, reason: ", fileData.sourceName(), fileData.name(), nre); incFailedAttemptsCounter(fileData); return Mono.just(Optional.empty()); // Give up } catch (DatafileTaskException e) { diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java index 2013950a..98804a0c 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java @@ -15,6 +15,7 @@ */ package org.onap.dcaegen2.collectors.datafile.http; +import org.apache.hc.core5.net.URIBuilder; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -29,6 +30,7 @@ import reactor.netty.http.client.HttpClientConfig; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.net.URISyntaxException; import java.nio.file.Path; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -49,6 +51,8 @@ class DfcHttpClientTest { private static final String PASSWORD = "123"; private static final String XNF_ADDRESS = "127.0.0.1"; private static final int PORT = 80; + private static final String JWT_PASSWORD = "thisIsThePassword"; + private static String ACCESS_TOKEN = "access_token"; @Mock private Path pathMock; @@ -61,10 +65,10 @@ class DfcHttpClientTest { } @Test - void openConnection_successAuthSetup() throws DatafileTaskException { + void openConnection_successBasicAuthSetup() throws DatafileTaskException { dfcHttpClientSpy.open(); HttpClientConfig config = dfcHttpClientSpy.client.configuration(); - assertEquals(HttpUtils.basicAuth(USERNAME, PASSWORD), config.headers().get("Authorization")); + assertEquals(HttpUtils.basicAuthContent(USERNAME, PASSWORD), config.headers().get("Authorization")); } @Test @@ -81,25 +85,34 @@ class DfcHttpClientTest { .hasMessageContaining("Not sufficient basic auth data for file."); } + @Test - void prepareUri_UriWithoutPort() { - ImmutableFileServerData serverData = ImmutableFileServerData.builder() - .serverAddress(XNF_ADDRESS) - .userId(USERNAME).password(PASSWORD) - .build(); - DfcHttpClient clientNoPortSpy = spy(new DfcHttpClient(serverData)); + void collectFile_AllOk() throws Exception { String REMOTE_FILE = "any"; + Flux<InputStream> fis = Flux.just(new ByteArrayInputStream("ReturnedString".getBytes())); + + dfcHttpClientSpy.open(); + + when(dfcHttpClientSpy.getServerResponse(any())).thenReturn(fis); + doReturn(false).when(dfcHttpClientSpy).isDownloadFailed(any()); + + dfcHttpClientSpy.collectFile(REMOTE_FILE, pathMock); + dfcHttpClientSpy.close(); - String retrievedUri = clientNoPortSpy.prepareUri(REMOTE_FILE); - assertTrue(retrievedUri.startsWith("http://" + XNF_ADDRESS + ":80")); + verify(dfcHttpClientSpy, times(1)).getServerResponse(ArgumentMatchers.eq(REMOTE_FILE)); + verify(dfcHttpClientSpy, times(1)).processDataFromServer(any(), any(), any()); + verify(dfcHttpClientSpy, times(1)).isDownloadFailed(any()); } @Test - void collectFile_AllOk() throws Exception { + void collectFile_AllOkWithJWTToken() throws Exception { + dfcHttpClientSpy = spy(new DfcHttpClient(fileServerDataWithJWTToken())); String REMOTE_FILE = "any"; Flux<InputStream> fis = Flux.just(new ByteArrayInputStream("ReturnedString".getBytes())); dfcHttpClientSpy.open(); + HttpClientConfig config = dfcHttpClientSpy.client.configuration(); + assertEquals(HttpUtils.jwtAuthContent(JWT_PASSWORD), config.headers().get("Authorization")); when(dfcHttpClientSpy.getServerResponse(any())).thenReturn(fis); doReturn(false).when(dfcHttpClientSpy).isDownloadFailed(any()); @@ -123,7 +136,7 @@ class DfcHttpClientTest { doReturn(fis).when(dfcHttpClientSpy).getServerResponse(any()); assertThatThrownBy(() -> dfcHttpClientSpy.collectFile(REMOTE_FILE, pathMock)) - .hasMessageContaining("Error occured during datafile download: "); + .hasMessageContaining(ERROR_RESPONSE); verify(dfcHttpClientSpy, times(1)).getServerResponse(REMOTE_FILE); verify(dfcHttpClientSpy, times(1)).processFailedConnectionWithServer(any(), any()); dfcHttpClientSpy.close(); @@ -142,4 +155,16 @@ class DfcHttpClientTest { .port(PORT) .build(); } + + private ImmutableFileServerData fileServerDataWithJWTToken() throws URISyntaxException { + String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD; + + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId("") + .password("") + .port(PORT) + .queryParameters(new URIBuilder(query).getQueryParams()) + .build(); + } } diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java index 8df91d3a..168bb08c 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java @@ -16,6 +16,7 @@ package org.onap.dcaegen2.collectors.datafile.http; +import org.apache.hc.core5.net.URIBuilder; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.conn.ConnectTimeoutException; @@ -25,15 +26,18 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; import java.io.IOException; import java.io.InputStream; +import java.net.URISyntaxException; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -50,6 +54,8 @@ class DfcHttpsClientTest { private static final String PASSWORD = "123"; private static final String XNF_ADDRESS = "127.0.0.1"; private static final int PORT = 443; + private static final String JWT_PASSWORD = "thisIsThePassword"; + private static String ACCESS_TOKEN = "access_token"; private static String remoteFile = "remoteFile"; @Mock @@ -105,6 +111,29 @@ class DfcHttpsClientTest { } @Test + void dfcHttpsClient_flow_successfulCallWithJWTAndResponseProcessing() throws Exception { + FileServerData serverData = jWTTokenInFileServerData(); + dfcHttpsClientSpy = spy(new DfcHttpsClient(serverData, connectionManager)); + + doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy) + .executeHttpClient(any(HttpGet.class)); + doReturn((long)3).when(dfcHttpsClientSpy).writeFile(eq(localFile), any(InputStream.class)); + + dfcHttpsClientSpy.open(); + dfcHttpsClientSpy.collectFile(remoteFile, localFile); + dfcHttpsClientSpy.close(); + + verify(dfcHttpsClientSpy, times(1)).makeCall(any(HttpGet.class)); + verify(dfcHttpsClientSpy, times(1)) + .executeHttpClient(any(HttpGet.class)); + verify(dfcHttpsClientSpy, times(1)) + .processResponse(HttpClientResponseHelper.APACHE_RESPONSE_OK, localFile); + verify(dfcHttpsClientSpy, times(1)) + .writeFile(eq(localFile), any(InputStream.class)); + assertFalse(serverData.toString().contains(JWT_PASSWORD)); + } + + @Test void dfcHttpsClient_flow_failedCallUnexpectedResponseCode() throws Exception { doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy) .executeHttpClient(any(HttpGet.class)); @@ -112,7 +141,7 @@ class DfcHttpsClientTest { dfcHttpsClientSpy.open(); - assertThrows(NonRetryableDatafileTaskException.class, + assertThrows(DatafileTaskException.class, () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile)); } @@ -157,8 +186,19 @@ class DfcHttpsClientTest { private ImmutableFileServerData invalidUserInFileServerData() { return ImmutableFileServerData.builder() .serverAddress(XNF_ADDRESS) - .userId("demo").password("") + .userId(USERNAME).password("") + .port(PORT) + .build(); + } + + private ImmutableFileServerData jWTTokenInFileServerData() throws URISyntaxException { + String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD; + + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId("").password("") .port(PORT) + .queryParameters(new URIBuilder(query).getQueryParams()) .build(); } } diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java index a95bfe2b..953f3226 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java @@ -1,6 +1,7 @@ /* * ============LICENSE_START====================================================================== * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Modifications Copyright (C) 2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -16,20 +17,167 @@ package org.onap.dcaegen2.collectors.datafile.service; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.net.URIBuilder; +import org.junit.jupiter.api.Test; +import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData; + +import java.net.URISyntaxException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.Test; - class HttpUtilsTest { + private static final String XNF_ADDRESS = "127.0.0.1"; + private static final int PORT = 443; + private static final String JWT_PASSWORD = "thisIsThePassword"; + private static final String ACCESS_TOKEN = "access_token"; + private static final String ANOTHER_TOKEN = "another_token"; + private static final String ANOTHER_DATA = "another_data"; + private static final String FRAGMENT = "thisIsTheFragment"; + private static final String USERNAME = "bob"; + private static final String PASSWORD = "123"; + + @Test + void shouldReturnSuccessfulResponse() { + assertTrue(HttpUtils.isSuccessfulResponseCodeWithDataRouter(200)); + } + + @Test + void shouldReturnBadResponse() { + assertFalse(HttpUtils.isSuccessfulResponseCodeWithDataRouter(404)); + } + @Test - public void shouldReturnSuccessfulResponse() { - assertTrue(HttpUtils.isSuccessfulResponseCode(200)); + void isSingleQueryWithJWT_validToken() throws URISyntaxException { + assertTrue(HttpUtils.isQueryWithSingleJWT(validTokenSingleQueryData())); + assertTrue(HttpUtils.isQueryWithSingleJWT(validTokenDoubleQueryData())); } @Test - public void shouldReturnBadResponse() { - assertFalse(HttpUtils.isSuccessfulResponseCode(404)); + void isSingleQueryWithJWT_invalidToken() throws URISyntaxException { + assertFalse(HttpUtils.isQueryWithSingleJWT(validQueryNoToken())); + assertFalse(HttpUtils.isQueryWithSingleJWT(queryDataDoubleToken())); + assertFalse(HttpUtils.isQueryWithSingleJWT(null)); + } + + @Test + void getJWTToken_jWTTokenPresent() throws URISyntaxException { + assertEquals(JWT_PASSWORD, HttpUtils.getJWTToken(fileServerDataWithJWTToken())); + assertEquals(JWT_PASSWORD, HttpUtils.getJWTToken(fileServerDataWithJWTTokenLongQueryAndFragment())); + } + + @Test + void getJWTToken_JWTTokenNotPresent() throws URISyntaxException { + assertEquals("", HttpUtils.getJWTToken(fileServerDataQueryWithoutToken())); + } + + @Test + void prepareUri_UriWithoutPort() { + ImmutableFileServerData serverData = ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId(USERNAME).password(PASSWORD) + .build(); + String REMOTE_FILE = "any"; + + String retrievedUri = HttpUtils.prepareUri("http", serverData, REMOTE_FILE, 80); + assertTrue(retrievedUri.startsWith("http://" + XNF_ADDRESS + ":80")); + } + + @Test + void prepareUri_verifyUriWithTokenAndFragment() throws URISyntaxException { + String file = "/file"; + String expected = "http://" + XNF_ADDRESS + ":" + PORT + file + "?" + + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&" + + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "#" + FRAGMENT; + assertEquals(expected, HttpUtils.prepareUri("http", fileServerDataWithJWTTokenLongQueryAndFragment(), file, 443)); + } + + @Test + void prepareUri_verifyUriWithoutTokenAndWithoutFragment() throws URISyntaxException { + String file = "/file"; + String expected = "http://" + XNF_ADDRESS + ":" + PORT + file; + assertEquals(expected, HttpUtils.prepareUri("http", fileServerDataNoTokenNoFragment(), file, 443)); + } + + private List<NameValuePair> validTokenSingleQueryData() throws URISyntaxException { + String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD; + return new URIBuilder(query).getQueryParams(); + } + + private List<NameValuePair> validTokenDoubleQueryData() throws URISyntaxException { + StringBuilder doubleQuery = new StringBuilder(); + doubleQuery.append("?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&"); + doubleQuery.append(ACCESS_TOKEN + "=" + JWT_PASSWORD); + return new URIBuilder(doubleQuery.toString()).getQueryParams(); + } + + private List<NameValuePair> validQueryNoToken() throws URISyntaxException { + String query = "?" + ANOTHER_TOKEN + "=" + JWT_PASSWORD; + return new URIBuilder(query).getQueryParams(); + } + + private List<NameValuePair> queryDataDoubleToken() throws URISyntaxException { + StringBuilder doubleToken = new StringBuilder(); + doubleToken.append("?" + ACCESS_TOKEN + "=" + JWT_PASSWORD + "&"); + doubleToken.append(ACCESS_TOKEN + "=" + JWT_PASSWORD + "&"); + doubleToken.append(ANOTHER_TOKEN + "=" + ANOTHER_DATA); + return new URIBuilder(doubleToken.toString()).getQueryParams(); + } + + private ImmutableFileServerData fileServerDataWithJWTToken() throws URISyntaxException { + String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD; + + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId("") + .password("") + .port(PORT) + .queryParameters(new URIBuilder(query).getQueryParams()) + .build(); + } + + private ImmutableFileServerData fileServerDataWithJWTTokenLongQueryAndFragment() throws URISyntaxException { + StringBuilder query = new StringBuilder(); + query.append("?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&"); + query.append(ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&"); + query.append(ACCESS_TOKEN + "=" + JWT_PASSWORD + "&"); + query.append(ANOTHER_TOKEN + "=" + ANOTHER_DATA); + + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId("") + .password("") + .port(PORT) + .queryParameters(new URIBuilder(query.toString()).getQueryParams()) + .uriRawFragment(FRAGMENT) + .build(); + } + + private ImmutableFileServerData fileServerDataQueryWithoutToken() throws URISyntaxException { + StringBuilder query = new StringBuilder(); + query.append("?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA); + + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId("") + .password("") + .port(PORT) + .queryParameters(new URIBuilder(query.toString()).getQueryParams()) + .build(); + } + + private ImmutableFileServerData fileServerDataNoTokenNoFragment() throws URISyntaxException { + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId("") + .password("") + .port(PORT) + .queryParameters(new URIBuilder("").getQueryParams()) + .uriRawFragment("") + .build(); } } @@ -31,7 +31,7 @@ <groupId>org.onap.dcaegen2.collectors</groupId> <artifactId>datafile</artifactId> - <version>1.5.3-SNAPSHOT</version> + <version>1.5.4-SNAPSHOT</version> <name>dcaegen2-collectors.datafile</name> <description>datafile collector</description> @@ -59,6 +59,7 @@ <commons-io.version>2.8.0</commons-io.version> <commons-net.version>3.3</commons-net.version> <projectreactor.version>2020.0.2</projectreactor.version> + <httpcomponents.core5.version>5.0.3</httpcomponents.core5.version> <!-- LOGGING SETTINGS --> <slf4j.version>1.7.25</slf4j.version> @@ -189,6 +190,11 @@ <version>${slf4j.version}</version> </dependency> <dependency> + <groupId>org.apache.httpcomponents.core5</groupId> + <artifactId>httpcore5</artifactId> + <version>${httpcomponents.core5.version}</version> + </dependency> + <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> <version>${junit-jupiter.version}</version> diff --git a/version.properties b/version.properties index 8e262309..80869687 100644 --- a/version.properties +++ b/version.properties @@ -1,6 +1,6 @@ major=1
minor=5
-patch=3
+patch=4
base_version=${major}.${minor}.${patch}
release_version=${base_version}
snapshot_version=${base_version}-SNAPSHOT
|