From 3eced2927621fc1a4260e802b06164fafcda57cf Mon Sep 17 00:00:00 2001 From: Krzysztof Gajewski Date: Wed, 13 Jan 2021 12:47:27 +0100 Subject: Add HTTPS to collect files from xNFs - plus small refactoring related to above - update to version 1.5.3 Issue-ID: DCAEGEN2-2528 Signed-off-by: Krzysztof Gajewski Change-Id: I2531c85967964f1359bafd5b694afbf662edf54e --- .../collectors/datafile/commons/Scheme.java | 18 ++- .../collectors/datafile/commons/SecurityUtil.java | 50 ++++++ .../datafile/configuration/AppConfig.java | 22 +-- .../datafile/configuration/CertificateConfig.java | 50 ++++++ .../datafile/configuration/CloudConfigParser.java | 15 +- .../datafile/configuration/FtpesConfig.java | 50 ------ .../collectors/datafile/ftp/FtpesClient.java | 22 +-- .../collectors/datafile/http/DfcHttpClient.java | 11 +- .../collectors/datafile/http/DfcHttpsClient.java | 170 +++++++++++++++++++++ .../http/HttpsClientConnectionManagerUtil.java | 132 ++++++++++++++++ .../collectors/datafile/model/Counters.java | 20 +++ .../collectors/datafile/service/HttpUtils.java | 1 + .../collectors/datafile/tasks/FileCollector.java | 25 ++- .../collectors/datafile/tasks/ScheduledTasks.java | 7 +- 14 files changed, 498 insertions(+), 95 deletions(-) create mode 100644 datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java create mode 100644 datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java delete mode 100644 datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java create mode 100644 datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java create mode 100644 datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java (limited to 'datafile-app-server/src/main') diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java index afa3aaea..b9aa6449 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START======================================================= * Copyright (C) 2019 Nordix Foundation. All rights reserved. - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,10 +28,10 @@ import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; * */ public enum Scheme { - FTPES, SFTP, HTTP; + FTPES, SFTP, HTTP, HTTPS; public static final String DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG = "DFC does not support protocol "; - public static final String SUPPORTED_PROTOCOLS_ERROR_MESSAGE = ". Supported protocols are FTPeS, sFTP and HTTP"; + public static final String SUPPORTED_PROTOCOLS_ERROR_MESSAGE = ". Supported protocols are FTPeS, sFTP, HTTP and HTTPS"; /** * Get a Scheme from a string. @@ -48,10 +48,22 @@ public enum Scheme { result = Scheme.SFTP; } else if ("HTTP".equalsIgnoreCase(schemeString)) { result = Scheme.HTTP; + } else if ("HTTPS".equalsIgnoreCase(schemeString)) { + result = Scheme.HTTPS; } else { throw new DatafileTaskException( DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG + schemeString + SUPPORTED_PROTOCOLS_ERROR_MESSAGE); } return result; } + + /** + * Check if Scheme is FTP type or HTTP type. + * + * @param scheme the Scheme which has to be checked. + * @return true if Scheme is FTP type or false if it is HTTP type + */ + public static boolean isFtpScheme(Scheme scheme) { + return scheme == SFTP || scheme == FTPES; + } } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java new file mode 100644 index 00000000..79db32d3 --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java @@ -0,0 +1,50 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.commons; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +/** + * Utility class containing functions used for certificates configuration + * + * @author Krzysztof Gajewski + */ +public final class SecurityUtil { + private SecurityUtil() {} + private static final Logger logger = LoggerFactory.getLogger(SecurityUtil.class); + + public static String getKeystorePasswordFromFile(String passwordPath) { + return getPasswordFromFile(passwordPath, "Keystore"); + } + + public static String getTruststorePasswordFromFile(String passwordPath) { + return getPasswordFromFile(passwordPath, "Truststore"); + } + + public static String getPasswordFromFile(String passwordPath, String element) { + try { + return new String(Files.readAllBytes(Paths.get(passwordPath))); + } catch (IOException e) { + logger.error("{} password file at path: {} cannot be opened ", element, passwordPath); + } + return ""; + } +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java index d933e337..b381c021 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START====================================================================== - * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. + * All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -37,6 +38,7 @@ import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil; import org.onap.dcaegen2.collectors.datafile.model.logging.MappedDiagnosticContext; import org.onap.dcaegen2.services.sdk.rest.services.cbs.client.api.CbsClient; import org.onap.dcaegen2.services.sdk.rest.services.cbs.client.api.CbsClientFactory; @@ -76,7 +78,7 @@ public class AppConfig { Properties systemEnvironment; private ConsumerConfiguration dmaapConsumerConfiguration; private Map publishingConfigurations; - private FtpesConfig ftpesConfiguration; + private CertificateConfig certificateConfiguration; private SftpConfig sftpConfiguration; private Disposable refreshConfigTask = null; @@ -163,8 +165,8 @@ public class AppConfig { return cfg; } - public synchronized FtpesConfig getFtpesConfiguration() { - return ftpesConfiguration; + public synchronized CertificateConfig getCertificateConfiguration() { + return certificateConfiguration; } public synchronized SftpConfig getSftpConfiguration() { @@ -193,7 +195,7 @@ public class AppConfig { CloudConfigParser parser = new CloudConfigParser(configurationObject, systemEnvironment); setConfiguration(parser.getConsumerConfiguration(), - parser.getDmaapPublisherConfigurations(), parser.getFtpesConfig(), + parser.getDmaapPublisherConfigurations(), parser.getCertificateConfig(), parser.getSftpConfig()); logConfig(); } catch (DatafileTaskException e) { @@ -204,7 +206,7 @@ public class AppConfig { private void logConfig() { logger.debug("Read and parsed sFTP configuration: [{}]", sftpConfiguration); - logger.debug("Read and parsed FTPes configuration: [{}]", ftpesConfiguration); + logger.debug("Read and parsed FTPes / HTTPS configuration: [{}]", certificateConfiguration); logger.debug("Read and parsed DMaaP configuration: [{}]", dmaapConsumerConfiguration); logger.debug("Read and parsed Publish configuration: [{}]", publishingConfigurations); } @@ -226,12 +228,14 @@ public class AppConfig { } private synchronized void setConfiguration(@NotNull ConsumerConfiguration consumerConfiguration, - @NotNull Map publisherConfiguration, @NotNull FtpesConfig ftpesConfig, - @NotNull SftpConfig sftpConfig) { + @NotNull Map publisherConfiguration, @NotNull CertificateConfig certificateConfig, + @NotNull SftpConfig sftpConfig) throws DatafileTaskException { this.dmaapConsumerConfiguration = consumerConfiguration; this.publishingConfigurations = publisherConfiguration; - this.ftpesConfiguration = ftpesConfig; + this.certificateConfiguration = certificateConfig; this.sftpConfiguration = sftpConfig; + HttpsClientConnectionManagerUtil.setupOrUpdate(certificateConfig.keyCert(), certificateConfig.keyPasswordPath(), + certificateConfig.trustedCa(), certificateConfig.trustedCaPasswordPath()); } JsonElement getJsonElement(InputStream inputStream) { diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java new file mode 100644 index 00000000..1d8b6143 --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java @@ -0,0 +1,50 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2018 NOKIA Intellectual Property, 2019 Nordix Foundation. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.configuration; + +import java.io.Serializable; + +import org.immutables.gson.Gson; +import org.immutables.value.Value; +import org.springframework.stereotype.Component; + +@Component +@Value.Immutable +@Value.Style(builder = "new", redactedMask = "####") +@Gson.TypeAdapters +public abstract class CertificateConfig implements Serializable { + + private static final long serialVersionUID = 1L; + + @Value.Parameter + public abstract String keyCert(); + + @Value.Parameter + @Value.Redacted + public abstract String keyPasswordPath(); + + @Value.Parameter + public abstract String trustedCa(); + + @Value.Parameter + @Value.Redacted + public abstract String trustedCaPasswordPath(); +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java index 6ace4aae..d6b86433 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START======================================================= - * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. + * All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -187,12 +188,12 @@ public class CloudConfigParser { * @return the xNF communication security configuration. * @throws DatafileTaskException if a member of the configuration is missing. */ - public @NotNull FtpesConfig getFtpesConfig() throws DatafileTaskException { - return new ImmutableFtpesConfig.Builder() // - .keyCert(getAsString(jsonObject, "dmaap.ftpesConfig.keyCert")) - .keyPasswordPath(getAsString(jsonObject, "dmaap.ftpesConfig.keyPasswordPath")) - .trustedCa(getAsString(jsonObject, "dmaap.ftpesConfig.trustedCa")) - .trustedCaPasswordPath(getAsString(jsonObject, "dmaap.ftpesConfig.trustedCaPasswordPath")) // + public @NotNull CertificateConfig getCertificateConfig() throws DatafileTaskException { + return new ImmutableCertificateConfig.Builder() // + .keyCert(getAsString(jsonObject, "dmaap.certificateConfig.keyCert")) + .keyPasswordPath(getAsString(jsonObject, "dmaap.certificateConfig.keyPasswordPath")) + .trustedCa(getAsString(jsonObject, "dmaap.certificateConfig.trustedCa")) + .trustedCaPasswordPath(getAsString(jsonObject, "dmaap.certificateConfig.trustedCaPasswordPath")) // .build(); } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java deleted file mode 100644 index 9e9da7db..00000000 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java +++ /dev/null @@ -1,50 +0,0 @@ -/*- - * ============LICENSE_START======================================================= - * Copyright (C) 2018 NOKIA Intellectual Property, 2019 Nordix Foundation. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.dcaegen2.collectors.datafile.configuration; - -import java.io.Serializable; - -import org.immutables.gson.Gson; -import org.immutables.value.Value; -import org.springframework.stereotype.Component; - -@Component -@Value.Immutable -@Value.Style(builder = "new", redactedMask = "####") -@Gson.TypeAdapters -public abstract class FtpesConfig implements Serializable { - - private static final long serialVersionUID = 1L; - - @Value.Parameter - public abstract String keyCert(); - - @Value.Parameter - @Value.Redacted - public abstract String keyPasswordPath(); - - @Value.Parameter - public abstract String trustedCa(); - - @Value.Parameter - @Value.Redacted - public abstract String trustedCaPasswordPath(); -} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java index 9bacec88..a82b5478 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START====================================================================== * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -22,9 +22,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.KeyStoreException; @@ -43,6 +41,7 @@ import org.apache.commons.net.ftp.FTPReply; import org.apache.commons.net.ftp.FTPSClient; import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; import org.slf4j.Logger; @@ -194,13 +193,7 @@ public class FtpesClient implements FileCollectClient { protected TrustManager getTrustManager(Path trustedCaPath, String trustedCaPasswordPath) throws KeyStoreException, NoSuchAlgorithmException, IOException, CertificateException { - String trustedCaPassword = ""; - try { - trustedCaPassword = new String(Files.readAllBytes(Paths.get(trustedCaPasswordPath))); - } catch (IOException e) { - logger.error("Truststore password file at path: {} cannot be opened ", trustedCaPasswordPath); - e.printStackTrace(); - } + String trustedCaPassword = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath); synchronized (FtpesClient.class) { if (theTrustManager == null) { theTrustManager = createTrustManager(trustedCaPath, trustedCaPassword); @@ -211,14 +204,7 @@ public class FtpesClient implements FileCollectClient { protected KeyManager getKeyManager(Path keyCertPath, String keyCertPasswordPath) throws IOException, GeneralSecurityException { - String keyCertPassword = ""; - try { - keyCertPassword = new String(Files.readAllBytes(Paths.get(keyCertPasswordPath))); - } catch (IOException e) { - logger.error("Keystore password file at path: {} cannot be opened ", keyCertPasswordPath); - e.printStackTrace(); - } - + String keyCertPassword = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath); synchronized (FtpesClient.class) { if (theKeyManager == null) { theKeyManager = createKeyManager(keyCertPath, keyCertPassword); diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java index 86bfc210..3ccc9fb2 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START====================================================================== - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -37,6 +37,11 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +/** + * Gets file from PNF with HTTP protocol. + * + * @author Krzysztof Gajewski + */ public class DfcHttpClient implements FileCollectClient { //Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS @@ -111,7 +116,7 @@ public class DfcHttpClient implements FileCollectClient { try { long numBytes = Files.copy(response, localFile); logger.trace("Transmission was successful - {} bytes downloaded.", numBytes); - logger.trace("CollectFile fetched: {}", localFile.toString()); + logger.trace("CollectFile fetched: {}", localFile); response.close(); } catch (IOException e) { errorMessages.set(new Exception("Error fetching file with", e)); @@ -139,7 +144,7 @@ public class DfcHttpClient implements FileCollectClient { } @NotNull protected String prepareUri(String remoteFile) { - int port = fileServerData.port().isPresent() ? fileServerData.port().get() : HttpUtils.HTTP_DEFAULT_PORT; + int port = fileServerData.port().orElse(HttpUtils.HTTP_DEFAULT_PORT); return "http://" + fileServerData.serverAddress() + ":" + port + remoteFile; } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java new file mode 100644 index 00000000..3090815a --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java @@ -0,0 +1,170 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.conn.HttpHostConnectException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.util.EntityUtils; +import org.jetbrains.annotations.NotNull; +import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.service.HttpUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLHandshakeException; +import java.io.IOException; +import java.io.InputStream; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +/** + * Gets file from PNF with HTTPS protocol. + * + * @author Krzysztof Gajewski + */ +public class DfcHttpsClient implements FileCollectClient { + + protected CloseableHttpClient httpsClient; + + private static final Logger logger = LoggerFactory.getLogger(DfcHttpsClient.class); + private static final int FIFTEEN_SECONDS = 15 * 1000; + + private final FileServerData fileServerData; + private final PoolingHttpClientConnectionManager connectionManager; + + public DfcHttpsClient(FileServerData fileServerData, PoolingHttpClientConnectionManager connectionManager) { + this.fileServerData = fileServerData; + this.connectionManager = connectionManager; + } + + @Override public void open() { + logger.trace("Setting httpsClient for file download."); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(true) + .build(); + + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(FIFTEEN_SECONDS) + .build(); + + httpsClient = HttpClients.custom() + .setConnectionManager(connectionManager) + .setDefaultSocketConfig(socketConfig) + .setDefaultRequestConfig(requestConfig) + .build(); + + logger.trace("httpsClient prepared for connection."); + } + + @Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException { + logger.trace("Prepare to collectFile {}", localFile); + HttpGet httpGet = new HttpGet(prepareUri(remoteFile)); + if (basicAuthValidNotPresentOrThrow()) { + httpGet.addHeader("Authorization", + HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password())); + } + try { + HttpResponse httpResponse = makeCall(httpGet); + processResponse(httpResponse, localFile); + } catch (IOException e) { + throw new DatafileTaskException("Error downloading file from server. ", e); + } + logger.trace("HTTPS collectFile OK"); + } + + protected boolean basicAuthValidNotPresentOrThrow() throws DatafileTaskException { + if (isAuthDataEmpty()) { + return false; + } + if (isAuthDataFilled()) { + return true; + } + throw new DatafileTaskException("Not sufficient basic auth data for file."); + } + + private boolean isAuthDataEmpty() { + return this.fileServerData.userId().isEmpty() && this.fileServerData.password().isEmpty(); + } + + private boolean isAuthDataFilled() { + return !this.fileServerData.userId().isEmpty() && !this.fileServerData.password().isEmpty(); + } + + @NotNull protected String prepareUri(String remoteFile) { + int port = fileServerData.port().orElse(HttpUtils.HTTPS_DEFAULT_PORT); + return "https://" + fileServerData.serverAddress() + ":" + port + remoteFile; + } + + protected HttpResponse makeCall(HttpGet httpGet) + throws IOException, DatafileTaskException { + try { + HttpResponse httpResponse = executeHttpClient(httpGet); + if (isResponseOk(httpResponse)) { + return httpResponse; + } + + EntityUtils.consume(httpResponse.getEntity()); + throw new NonRetryableDatafileTaskException( + "Unexpected response code - " + httpResponse.getStatusLine().getStatusCode() + + ". No retry attempts will be done."); + + } catch (ConnectTimeoutException | UnknownHostException | HttpHostConnectException | SSLHandshakeException e) { + throw new NonRetryableDatafileTaskException( + "Unable to get file from xNF. No retry attempts will be done.", e); + } + } + + protected CloseableHttpResponse executeHttpClient(HttpGet httpGet) + throws IOException { + return httpsClient.execute(httpGet); + } + + protected boolean isResponseOk(HttpResponse response) { + return response.getStatusLine().getStatusCode() == 200; + } + + protected void processResponse(HttpResponse response, Path localFile) throws IOException { + logger.trace("Starting to process response."); + HttpEntity entity = response.getEntity(); + InputStream stream = entity.getContent(); + long numBytes = writeFile(localFile, stream); + stream.close(); + EntityUtils.consume(entity); + logger.trace("Transmission was successful - {} bytes downloaded.", numBytes); + } + + protected long writeFile(Path localFile, InputStream stream) throws IOException { + return Files.copy(stream, localFile, StandardCopyOption.REPLACE_EXISTING); + } + + @Override public void close() { + logger.trace("Https client has ended downloading process."); + } +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java new file mode 100644 index 00000000..e60ec0f4 --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java @@ -0,0 +1,132 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; +import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.FileSystemResource; + +import javax.net.ssl.SSLContext; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; + +/** + * Utility class supplying connection manager for HTTPS protocol. + * + * @author Krzysztof Gajewski + */ +public class HttpsClientConnectionManagerUtil { + + private HttpsClientConnectionManagerUtil() { + } + + private static final Logger logger = LoggerFactory.getLogger(HttpsClientConnectionManagerUtil.class); + //Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS + private static final int MAX_NUMBER_OF_CONNECTIONS = 200; + private static PoolingHttpClientConnectionManager connectionManager; + + public static PoolingHttpClientConnectionManager instance() throws DatafileTaskException { + if (connectionManager == null) { + throw new DatafileTaskException("ConnectionManager has to be set or update first"); + } + return connectionManager; + } + + public static void setupOrUpdate(String keyCertPath, String keyCertPasswordPath, String trustedCaPath, + String trustedCaPasswordPath) throws DatafileTaskException { + synchronized (HttpsClientConnectionManagerUtil.class) { + if (connectionManager != null) { + connectionManager.close(); + connectionManager = null; + } + setup(keyCertPath, keyCertPasswordPath, trustedCaPath, trustedCaPasswordPath); + } + logger.trace("HttpsConnectionManager setup or updated"); + } + + private static void setup(String keyCertPath, String keyCertPasswordPath, String trustedCaPath, + String trustedCaPasswordPath) throws DatafileTaskException { + try { + SSLContextBuilder sslBuilder = SSLContexts.custom(); + sslBuilder = supplyKeyInfo(keyCertPath, keyCertPasswordPath, sslBuilder); + sslBuilder = supplyTrustInfo(trustedCaPath, trustedCaPasswordPath, sslBuilder); + + SSLContext sslContext = sslBuilder.build(); + + SSLConnectionSocketFactory sslConnectionSocketFactory = + new SSLConnectionSocketFactory(sslContext, new String[] {"TLSv1.2"}, null, + (hostname, session) -> true); + + Registry socketFactoryRegistry = + RegistryBuilder.create().register("https", sslConnectionSocketFactory) + .build(); + + connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connectionManager.setMaxTotal(MAX_NUMBER_OF_CONNECTIONS); + + } catch (Exception e) { + throw new DatafileTaskException("Unable to prepare HttpsConnectionManager : ", e); + } + } + + private static SSLContextBuilder supplyKeyInfo(String keyCertPath, String keyCertPasswordPath, + SSLContextBuilder sslBuilder) + throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException, + UnrecoverableKeyException { + String keyPass = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath); + KeyStore keyFile = createKeyStore(keyCertPath, keyPass); + return sslBuilder.loadKeyMaterial(keyFile, keyPass.toCharArray()); + } + + private static KeyStore createKeyStore(String trustedCaPath, String trustedCaPassword) + throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException { + logger.trace("Creating trust manager from file: {}", trustedCaPath); + try (InputStream fis = createInputStream(trustedCaPath)) { + KeyStore keyStore = KeyStore.getInstance("PKCS12"); + keyStore.load(fis, trustedCaPassword.toCharArray()); + return keyStore; + } + } + + private static InputStream createInputStream(String localFileName) throws IOException { + FileSystemResource realResource = new FileSystemResource(Paths.get(localFileName)); + return realResource.getInputStream(); + } + + private static SSLContextBuilder supplyTrustInfo(String trustedCaPath, String trustedCaPasswordPath, + SSLContextBuilder sslBuilder) + throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException { + String trustPass = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath); + File trustStoreFile = new File(trustedCaPath); + return sslBuilder.loadTrustMaterial(trustStoreFile, trustPass.toCharArray()); + } +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java index 8e8d847c..d5587e97 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java @@ -33,7 +33,9 @@ public class Counters { private final AtomicInteger numberOfSubscriptions = new AtomicInteger(); private int noOfCollectedFiles = 0; private int noOfFailedFtpAttempts = 0; + private int noOfFailedHttpAttempts = 0; private int noOfFailedFtp = 0; + private int noOfFailedHttp = 0; private int noOfFailedPublishAttempts = 0; private int totalPublishedFiles = 0; private int noOfFailedPublish = 0; @@ -62,10 +64,18 @@ public class Counters { noOfFailedFtpAttempts++; } + public synchronized void incNoOfFailedHttpAttempts() { + noOfFailedHttpAttempts++; + } + public synchronized void incNoOfFailedFtp() { noOfFailedFtp++; } + public synchronized void incNoOfFailedHttp() { + noOfFailedHttp++; + } + public synchronized void incNoOfFailedPublishAttempts() { noOfFailedPublishAttempts++; } @@ -89,7 +99,9 @@ public class Counters { str.append("\n"); str.append(format("collectedFiles", noOfCollectedFiles)); str.append(format("failedFtpAttempts", noOfFailedFtpAttempts)); + str.append(format("failedHttpAttempts", noOfFailedHttpAttempts)); str.append(format("failedFtp", noOfFailedFtp)); + str.append(format("failedHttp", noOfFailedHttp)); str.append("\n"); str.append(format("totalPublishedFiles", totalPublishedFiles)); str.append(format("lastPublishedTime", lastPublishedTime)); @@ -113,10 +125,18 @@ public class Counters { return noOfFailedFtpAttempts; } + public int getNoOfFailedHttpAttempts() { + return noOfFailedHttpAttempts; + } + public int getNoOfFailedFtp() { return noOfFailedFtp; } + public int getNoOfFailedHttp() { + return noOfFailedHttp; + } + public int getNoOfFailedPublishAttempts() { return noOfFailedPublishAttempts; } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java index 1dca0058..e2c1e2ff 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java @@ -26,6 +26,7 @@ import java.util.Base64; public final class HttpUtils implements HttpStatus { public static final int HTTP_DEFAULT_PORT = 80; + public static final int HTTPS_DEFAULT_PORT = 443; private HttpUtils() { } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java index e76d4156..cfc77549 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java @@ -23,8 +23,9 @@ import java.time.Duration; import java.util.Map; import java.util.Optional; +import org.onap.dcaegen2.collectors.datafile.commons.Scheme; import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; -import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig; +import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; @@ -32,6 +33,8 @@ import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient; import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient; import org.onap.dcaegen2.collectors.datafile.ftp.SftpClientSettings; import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient; +import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient; +import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil; import org.onap.dcaegen2.collectors.datafile.model.Counters; import org.onap.dcaegen2.collectors.datafile.model.FileData; import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation; @@ -109,11 +112,11 @@ public class FileCollector { return Mono.just(Optional.of(getFilePublishInformation(fileData, localFile, context))); } catch (NonRetryableDatafileTaskException nre) { logger.warn("Failed to download file: {} {}, reason: {}", fileData.sourceName(), fileData.name(), nre); - counters.incNoOfFailedFtpAttempts(); + incFailedAttemptsCounter(fileData); return Mono.just(Optional.empty()); // Give up } catch (DatafileTaskException e) { logger.warn("Failed to download file: {} {}, reason: ", fileData.sourceName(), fileData.name(), e); - counters.incNoOfFailedFtpAttempts(); + incFailedAttemptsCounter(fileData); return Mono.error(e); } catch (Exception throwable) { logger.warn("Failed to close client: {} {}, reason: {}", fileData.sourceName(), fileData.name(), @@ -122,6 +125,14 @@ public class FileCollector { } } + private void incFailedAttemptsCounter(FileData fileData) { + if (Scheme.isFtpScheme(fileData.scheme())) { + counters.incNoOfFailedFtpAttempts(); + } else { + counters.incNoOfFailedHttpAttempts(); + } + } + private FileCollectClient createClient(FileData fileData) throws DatafileTaskException { switch (fileData.scheme()) { case SFTP: @@ -130,6 +141,8 @@ public class FileCollector { return createFtpesClient(fileData); case HTTP: return createHttpClient(fileData); + case HTTPS: + return createHttpsClient(fileData); default: throw new DatafileTaskException("Unhandled protocol: " + fileData.scheme()); } @@ -163,7 +176,7 @@ public class FileCollector { } protected FtpesClient createFtpesClient(FileData fileData) { - FtpesConfig config = datafileAppConfig.getFtpesConfiguration(); + CertificateConfig config = datafileAppConfig.getCertificateConfiguration(); return new FtpesClient(fileData.fileServerData(), Paths.get(config.keyCert()), config.keyPasswordPath(), Paths.get(config.trustedCa()), config.trustedCaPasswordPath()); } @@ -171,4 +184,8 @@ public class FileCollector { protected FileCollectClient createHttpClient(FileData fileData) { return new DfcHttpClient(fileData.fileServerData()); } + + protected FileCollectClient createHttpsClient(FileData fileData) throws DatafileTaskException { + return new DfcHttpsClient(fileData.fileServerData(), HttpsClientConnectionManagerUtil.instance()); + } } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java index eba0a6cb..fa1757e2 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java @@ -23,6 +23,7 @@ import java.time.Instant; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import org.onap.dcaegen2.collectors.datafile.commons.Scheme; import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.model.Counters; @@ -257,7 +258,11 @@ public class ScheduledTasks { deleteFile(localFilePath, fileData.context); publishedFilesCache.remove(localFilePath); currentNumberOfTasks.decrementAndGet(); - counters.incNoOfFailedFtp(); + if (Scheme.isFtpScheme(fileData.fileData.scheme())) { + counters.incNoOfFailedFtp(); + } else { + counters.incNoOfFailedHttp(); + } return Mono.empty(); } -- cgit 1.2.3-korg