diff options
Diffstat (limited to 'datafile-app-server/src/main/java/org/onap/dcaegen2')
7 files changed, 281 insertions, 42 deletions
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java index 32241fdb..2c44abb5 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START====================================================================== * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Modifications copyright (C) 2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -16,12 +17,15 @@ package org.onap.dcaegen2.collectors.datafile.commons; +import java.util.List; import java.util.Optional; +import org.apache.hc.core5.http.NameValuePair; import org.immutables.value.Value; /** * Data about the file server to collect a file from. + * In case of http protocol it also contains data required to recreate target uri * * @author <a href="mailto:henrik.b.andersson@est.tech">Henrik Andersson</a> * @@ -37,4 +41,9 @@ public interface FileServerData { public String password(); public Optional<Integer> port(); + + @Value.Redacted + public Optional<List<NameValuePair>> queryParameters(); + + public Optional<String> uriRawFragment(); } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java index 3ccc9fb2..4564a44f 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.NotNull; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; import org.onap.dcaegen2.collectors.datafile.service.HttpUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,17 +62,22 @@ public class DfcHttpClient implements FileCollectClient { @Override public void open() throws DatafileTaskException { logger.trace("Setting httpClient for file download."); - basicAuthDataPresentOrThrow(); + String authorizationContent = getAuthorizationContent(); this.client = HttpClient.create(pool).keepAlive(true).headers( - h -> h.add("Authorization", HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password()))); + h -> h.add("Authorization", authorizationContent)); logger.trace("httpClient, auth header was set."); } - private void basicAuthDataPresentOrThrow() throws DatafileTaskException { - if ((this.fileServerData.userId().isEmpty()) || (this.fileServerData.password().isEmpty())) { + protected String getAuthorizationContent() throws DatafileTaskException { + String jwtToken = HttpUtils.getJWTToken(fileServerData); + if (!jwtToken.isEmpty()) { + return HttpUtils.jwtAuthContent(jwtToken); + } + if (!HttpUtils.isBasicAuthDataFilled(fileServerData)) { throw new DatafileTaskException("Not sufficient basic auth data for file."); } + return HttpUtils.basicAuthContent(this.fileServerData.userId(), this.fileServerData.password()); } @Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException { @@ -92,7 +98,10 @@ public class DfcHttpClient implements FileCollectClient { } if (isDownloadFailed(errorMessage)) { - throw new DatafileTaskException("Error occured during datafile download: ", errorMessage.get()); + if (errorMessage.get() instanceof NonRetryableDatafileTaskException) { + throw (NonRetryableDatafileTaskException) errorMessage.get(); + } + throw (DatafileTaskException) errorMessage.get(); } logger.trace("HTTP collectFile OK"); @@ -104,7 +113,11 @@ public class DfcHttpClient implements FileCollectClient { @NotNull protected Consumer<Throwable> processFailedConnectionWithServer(CountDownLatch latch, AtomicReference<Exception> errorMessages) { return (Throwable response) -> { - errorMessages.set(new Exception("Error in connection has occurred during file download", response)); + Exception e = new Exception("Error in connection has occurred during file download", response); + errorMessages.set(new DatafileTaskException(response.getMessage(), e)); + if (response instanceof NonRetryableDatafileTaskException) { + errorMessages.set(new NonRetryableDatafileTaskException(response.getMessage(), e)); + } latch.countDown(); }; } @@ -119,7 +132,7 @@ public class DfcHttpClient implements FileCollectClient { logger.trace("CollectFile fetched: {}", localFile); response.close(); } catch (IOException e) { - errorMessages.set(new Exception("Error fetching file with", e)); + errorMessages.set(new DatafileTaskException("Error fetching file with", e)); } finally { latch.countDown(); } @@ -128,24 +141,31 @@ public class DfcHttpClient implements FileCollectClient { protected Flux<InputStream> getServerResponse(String remoteFile) { return client.get() - .uri(prepareUri(remoteFile)) + .uri(HttpUtils.prepareHttpUri(fileServerData, remoteFile)) .response((responseReceiver, byteBufFlux) -> { logger.trace("HTTP response status - {}", responseReceiver.status()); if(isResponseOk(responseReceiver)){ return byteBufFlux.aggregate().asInputStream(); } - return Mono.error(new Throwable("Unexpected server response code - " - + responseReceiver.status().toString())); + if (isErrorInConnection(responseReceiver)) { + return Mono.error(new NonRetryableDatafileTaskException( + HttpUtils.nonRetryableResponse(getResponseCode(responseReceiver)))); + } + return Mono.error(new DatafileTaskException( + HttpUtils.retryableResponse(getResponseCode(responseReceiver)))); }); } protected boolean isResponseOk(HttpClientResponse httpClientResponse) { - return httpClientResponse.status().code() == 200; + return getResponseCode(httpClientResponse) == 200; + } + + private int getResponseCode(HttpClientResponse responseReceiver) { + return responseReceiver.status().code(); } - @NotNull protected String prepareUri(String remoteFile) { - int port = fileServerData.port().orElse(HttpUtils.HTTP_DEFAULT_PORT); - return "http://" + fileServerData.serverAddress() + ":" + port + remoteFile; + protected boolean isErrorInConnection(HttpClientResponse httpClientResponse) { + return getResponseCode(httpClientResponse) >= 400; } @Override public void close() { diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java index 3090815a..c2d72f67 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java @@ -27,7 +27,6 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; -import org.jetbrains.annotations.NotNull; import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; @@ -85,10 +84,11 @@ public class DfcHttpsClient implements FileCollectClient { @Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException { logger.trace("Prepare to collectFile {}", localFile); - HttpGet httpGet = new HttpGet(prepareUri(remoteFile)); - if (basicAuthValidNotPresentOrThrow()) { - httpGet.addHeader("Authorization", - HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password())); + HttpGet httpGet = new HttpGet(HttpUtils.prepareHttpsUri(fileServerData, remoteFile)); + + String authorizationContent = getAuthorizationContent(); + if (!authorizationContent.isEmpty()) { + httpGet.addHeader("Authorization", authorizationContent); } try { HttpResponse httpResponse = makeCall(httpGet); @@ -99,11 +99,23 @@ public class DfcHttpsClient implements FileCollectClient { logger.trace("HTTPS collectFile OK"); } + private String getAuthorizationContent() throws DatafileTaskException { + String jwtToken = HttpUtils.getJWTToken(fileServerData); + if (shouldUseBasicAuth(jwtToken)) { + return HttpUtils.basicAuthContent(this.fileServerData.userId(), this.fileServerData.password()); + } + return HttpUtils.jwtAuthContent(jwtToken); + } + + private boolean shouldUseBasicAuth(String jwtToken) throws DatafileTaskException { + return basicAuthValidNotPresentOrThrow() && jwtToken.isEmpty(); + } + protected boolean basicAuthValidNotPresentOrThrow() throws DatafileTaskException { if (isAuthDataEmpty()) { return false; } - if (isAuthDataFilled()) { + if (HttpUtils.isBasicAuthDataFilled(fileServerData)) { return true; } throw new DatafileTaskException("Not sufficient basic auth data for file."); @@ -113,15 +125,6 @@ public class DfcHttpsClient implements FileCollectClient { return this.fileServerData.userId().isEmpty() && this.fileServerData.password().isEmpty(); } - private boolean isAuthDataFilled() { - return !this.fileServerData.userId().isEmpty() && !this.fileServerData.password().isEmpty(); - } - - @NotNull protected String prepareUri(String remoteFile) { - int port = fileServerData.port().orElse(HttpUtils.HTTPS_DEFAULT_PORT); - return "https://" + fileServerData.serverAddress() + ":" + port + remoteFile; - } - protected HttpResponse makeCall(HttpGet httpGet) throws IOException, DatafileTaskException { try { @@ -131,10 +134,10 @@ public class DfcHttpsClient implements FileCollectClient { } EntityUtils.consume(httpResponse.getEntity()); - throw new NonRetryableDatafileTaskException( - "Unexpected response code - " + httpResponse.getStatusLine().getStatusCode() - + ". No retry attempts will be done."); - + if (isErrorInConnection(httpResponse)) { + throw new NonRetryableDatafileTaskException(HttpUtils.retryableResponse(getResponseCode(httpResponse))); + } + throw new DatafileTaskException(HttpUtils.nonRetryableResponse(getResponseCode(httpResponse))); } catch (ConnectTimeoutException | UnknownHostException | HttpHostConnectException | SSLHandshakeException e) { throw new NonRetryableDatafileTaskException( "Unable to get file from xNF. No retry attempts will be done.", e); @@ -146,8 +149,16 @@ public class DfcHttpsClient implements FileCollectClient { return httpsClient.execute(httpGet); } - protected boolean isResponseOk(HttpResponse response) { - return response.getStatusLine().getStatusCode() == 200; + protected boolean isResponseOk(HttpResponse httpResponse) { + return getResponseCode(httpResponse) == 200; + } + + private int getResponseCode(HttpResponse httpResponse) { + return httpResponse.getStatusLine().getStatusCode(); + } + + protected boolean isErrorInConnection(HttpResponse httpResponse) { + return getResponseCode(httpResponse) >= 400; } protected void processResponse(HttpResponse response, Path localFile) throws IOException { diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java index 1c8f57da..e7d2e15c 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START======================================================= * Copyright (C) 2019 Nordix Foundation. + * Copyright (C) 2021 Nokia. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +24,11 @@ package org.onap.dcaegen2.collectors.datafile.model; import java.net.URI; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import java.util.Optional; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.net.URIBuilder; import org.immutables.gson.Gson; import org.immutables.value.Value; import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; @@ -101,6 +105,7 @@ public abstract class FileData { /** * Get the data about the file server where the file should be collected from. + * Query data included as it can contain JWT token * * @return the data about the file server where the file should be collected from. */ @@ -114,6 +119,15 @@ public abstract class FileData { if (uri.getPort() > 0) { builder.port(uri.getPort()); } + URIBuilder uriBuilder = new URIBuilder(uri); + List<NameValuePair> query = uriBuilder.getQueryParams(); + if (query != null && !query.isEmpty()) { + builder.queryParameters(query); + } + String fragment = uri.getRawFragment(); + if (fragment != null && fragment.length() > 0) { + builder.uriRawFragment(fragment); + } return builder.build(); } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java index e2c1e2ff..69bfcf47 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START====================================================================== * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. - * Modifications Copyright (C) 2020 Nokia. All rights reserved + * Modifications Copyright (C) 2020-2021 Nokia. All rights reserved * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,23 +19,208 @@ package org.onap.dcaegen2.collectors.datafile.service; +import org.apache.hc.core5.http.NameValuePair; import org.apache.http.HttpStatus; +import org.jetbrains.annotations.NotNull; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import java.util.ArrayList; import java.util.Base64; +import java.util.List; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class HttpUtils implements HttpStatus { + private static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); public static final int HTTP_DEFAULT_PORT = 80; public static final int HTTPS_DEFAULT_PORT = 443; + public static final String JWT_TOKEN_NAME = "access_token"; + public static final String AUTH_JWT_WARN = "Both JWT token and Basic auth data present. Omitting basic auth info."; + public static final String AUTH_JWT_ERROR = "More than one JWT token present in the queryParameters. Omitting JWT token."; private HttpUtils() { } - public static boolean isSuccessfulResponseCode(Integer statusCode) { + @NotNull + public static String nonRetryableResponse(int responseCode) { + return "Unexpected response code - " + responseCode; + } + + @NotNull + public static String retryableResponse(int responseCode) { + return "Unexpected response code - " + responseCode + ". No retry attempts will be done."; + } + + public static boolean isSuccessfulResponseCodeWithDataRouter(Integer statusCode) { return statusCode >= 200 && statusCode < 300; } - public static String basicAuth(String username, String password) { + public static boolean isBasicAuthDataFilled(final FileServerData fileServerData) { + return !fileServerData.userId().isEmpty() && !fileServerData.password().isEmpty(); + } + + public static String basicAuthContent(String username, String password) { return "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes()); } + + public static String jwtAuthContent(String token) { + return "Bearer " + token; + } + + /** + * Prepare uri to retrieve file from xNF using HTTP connection. If JWT token was included + * in the queryParameters, it is removed. Other entries are rewritten. + * + * @param fileServerData fileServerData including - server address, port, queryParameters and uriRawFragment + * @param remoteFile file which has to be downloaded + * @return uri String representing the xNF HTTP location + */ + @NotNull public static String prepareHttpUri(FileServerData fileServerData, String remoteFile){ + return prepareUri("http", fileServerData, remoteFile, HTTP_DEFAULT_PORT); + } + + /** + * Prepare uri to retrieve file from xNF using HTTPS connection. If JWT token was included + * in the queryParameters, it is removed. Other entries are rewritten. + * + * @param fileServerData fileServerData including - server address, port, queryParameters and uriRawFragment + * @param remoteFile file which has to be downloaded + * @return uri String representing the xNF HTTPS location + */ + @NotNull public static String prepareHttpsUri(FileServerData fileServerData, String remoteFile){ + return prepareUri("https", fileServerData, remoteFile, HTTPS_DEFAULT_PORT); + } + + /** + * Prepare uri to retrieve file from xNF. If JWT token was included + * in the queryParameters, it is removed. Other entries are rewritten. + * + * @param scheme scheme which is used during the connection + * @param fileServerData fileServerData including - server address, port, query and fragment + * @param remoteFile file which has to be downloaded + * @param defaultPort default port which exchange empty entry for given connection type + * @return uri String representing the xNF location + */ + @NotNull public static String prepareUri(String scheme, FileServerData fileServerData, String remoteFile, int defaultPort) { + int port = fileServerData.port().orElse(defaultPort); + String query = rewriteQueryWithoutToken(fileServerData.queryParameters().orElse(new ArrayList<>())); + String fragment = fileServerData.uriRawFragment().orElse(""); + if (!query.isEmpty()) { + query = "?" + query; + } + if (!fragment.isEmpty()) { + fragment = "#" + fragment; + } + return scheme + "://" + fileServerData.serverAddress() + ":" + port + remoteFile + query + fragment; + } + + /** + * Returns JWT token string (if single exist) from the queryParameters. + * + * @param fileServerData file server data which contain queryParameters where JWT token may exist + * @return JWT token value if single token entry exist or empty string elsewhere. + * If JWT token key has no value, empty string will be returned. + */ + public static String getJWTToken(FileServerData fileServerData) { + Optional<List<NameValuePair>> query = fileServerData.queryParameters(); + if (!query.isPresent()) { + return ""; + } + List<NameValuePair> queryParameters = query.get(); + if (queryParameters.isEmpty()) { + return ""; + } + boolean jwtTokenKeyPresent = HttpUtils.isQueryWithSingleJWT(queryParameters); + if (!jwtTokenKeyPresent) { + return ""; + } + String token = HttpUtils.getJWTToken(query.get()); + if (HttpUtils.isBasicAuthDataFilled(fileServerData)) { + logger.warn(HttpUtils.AUTH_JWT_WARN); + } + return token; + } + + /** + * Checks if the queryParameters contains single JWT token entry. Valid queryParameters + * contains only one token entry. + * + * @param query queryParameters + * @return true if queryParameters contains single token + */ + public static boolean isQueryWithSingleJWT(List<NameValuePair> query) { + if (query == null) { + return false; + } + int i = getJWTTokenCount(query); + if (i == 0) { + return false; + } + if (i > 1) { + logger.error(AUTH_JWT_ERROR); + return false; + } + return true; + } + + /** + * Returns the number of JWT token entries. Valid queryParameters contains only one token entry. + * + * @param queryElements elements of the queryParameters + * @return true if queryParameters contains single JWT token entry + */ + public static int getJWTTokenCount(List<NameValuePair> queryElements) { + int i = 0; + for (NameValuePair element : queryElements) { + if (element.getName().equals(JWT_TOKEN_NAME)) { + i++; + } + } + return i; + } + + private static String getJWTToken(List<NameValuePair> query) { + for (NameValuePair element : query) { + if (!element.getName().equals(JWT_TOKEN_NAME)) { + continue; + } + if (element.getValue() != null) { + return element.getValue(); + } + return ""; + } + return ""; + } + + /** + * Rewrites HTTP queryParameters without JWT token + * + * @param query list of NameValuePair of elements sent in the queryParameters + * @return String representation of queryParameters elements which were provided in the input + * Empty string is possible when queryParameters is empty or contains only access_token key. + */ + public static String rewriteQueryWithoutToken(List<NameValuePair> query) { + if (query.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (NameValuePair nvp : query) { + if (nvp.getName().equals(JWT_TOKEN_NAME)) { + continue; + } + sb.append(nvp.getName()); + if (nvp.getValue() != null) { + sb.append("="); + sb.append(nvp.getValue()); + } + sb.append("&"); + } + if ((sb.length() > 0) && (sb.charAt(sb.length() - 1 ) == '&')) { + sb.deleteCharAt(sb.length() - 1 ); + } + return sb.toString(); + } } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java index c24c6c96..ef2341fb 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START======================================================= * Copyright (C) 2019 Nordix Foundation. - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -133,7 +133,7 @@ public class DataRouterPublisher { private Mono<FilePublishInformation> handleHttpResponse(HttpStatus response, FilePublishInformation publishInfo) { MDC.setContextMap(publishInfo.getContext()); - if (HttpUtils.isSuccessfulResponseCode(response.value())) { + if (HttpUtils.isSuccessfulResponseCodeWithDataRouter(response.value())) { counters.incTotalPublishedFiles(); logger.trace("Publishing file {} to DR successful!", publishInfo.getName()); return Mono.just(publishInfo); diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java index cfc77549..70380437 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START====================================================================== * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -111,7 +111,7 @@ public class FileCollector { counters.incNoOfCollectedFiles(); return Mono.just(Optional.of(getFilePublishInformation(fileData, localFile, context))); } catch (NonRetryableDatafileTaskException nre) { - logger.warn("Failed to download file: {} {}, reason: {}", fileData.sourceName(), fileData.name(), nre); + logger.warn("Failed to download file: {} {}, reason: ", fileData.sourceName(), fileData.name(), nre); incFailedAttemptsCounter(fileData); return Mono.just(Optional.empty()); // Give up } catch (DatafileTaskException e) { |