summaryrefslogtreecommitdiffstats
path: root/datafile-app-server/src
diff options
context:
space:
mode:
authorKrzysztof Gajewski <krzysztof.gajewski@nokia.com>2021-01-13 12:47:27 +0100
committerKrzysztof Gajewski <krzysztof.gajewski@nokia.com>2021-02-11 12:56:20 +0100
commit3eced2927621fc1a4260e802b06164fafcda57cf (patch)
treea4c86c521aea092d863eedd2d46c2520c8285783 /datafile-app-server/src
parent9e2a5fdd1c227412aa6eaa8bdfd414095a448deb (diff)
Add HTTPS to collect files from xNFs1.5.3
- plus small refactoring related to above - update to version 1.5.3 Issue-ID: DCAEGEN2-2528 Signed-off-by: Krzysztof Gajewski <krzysztof.gajewski@nokia.com> Change-Id: I2531c85967964f1359bafd5b694afbf662edf54e
Diffstat (limited to 'datafile-app-server/src')
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java18
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java50
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java22
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java (renamed from datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java)2
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java15
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java22
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java11
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java170
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java132
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java20
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java1
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java25
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java7
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java23
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java34
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java164
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java176
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java52
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java1
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java102
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java1
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java120
-rw-r--r--datafile-app-server/src/test/resources/datafile_endpoints_test.json8
-rw-r--r--datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json8
-rw-r--r--datafile-app-server/src/test/resources/keystore.p12bin0 -> 2857 bytes
-rw-r--r--datafile-app-server/src/test/resources/keystore.pass1
26 files changed, 1093 insertions, 92 deletions
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java
index afa3aaea..b9aa6449 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java
@@ -1,7 +1,7 @@
/*-
* ============LICENSE_START=======================================================
* Copyright (C) 2019 Nordix Foundation. All rights reserved.
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,10 +28,10 @@ import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
*
*/
public enum Scheme {
- FTPES, SFTP, HTTP;
+ FTPES, SFTP, HTTP, HTTPS;
public static final String DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG = "DFC does not support protocol ";
- public static final String SUPPORTED_PROTOCOLS_ERROR_MESSAGE = ". Supported protocols are FTPeS, sFTP and HTTP";
+ public static final String SUPPORTED_PROTOCOLS_ERROR_MESSAGE = ". Supported protocols are FTPeS, sFTP, HTTP and HTTPS";
/**
* Get a <code>Scheme</code> from a string.
@@ -48,10 +48,22 @@ public enum Scheme {
result = Scheme.SFTP;
} else if ("HTTP".equalsIgnoreCase(schemeString)) {
result = Scheme.HTTP;
+ } else if ("HTTPS".equalsIgnoreCase(schemeString)) {
+ result = Scheme.HTTPS;
} else {
throw new DatafileTaskException(
DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG + schemeString + SUPPORTED_PROTOCOLS_ERROR_MESSAGE);
}
return result;
}
+
+ /**
+ * Check if <code>Scheme</code> is FTP type or HTTP type.
+ *
+ * @param scheme the <code>Scheme</code> which has to be checked.
+ * @return true if <code>Scheme</code> is FTP type or false if it is HTTP type
+ */
+ public static boolean isFtpScheme(Scheme scheme) {
+ return scheme == SFTP || scheme == FTPES;
+ }
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java
new file mode 100644
index 00000000..79db32d3
--- /dev/null
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java
@@ -0,0 +1,50 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.commons;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/**
+ * Utility class containing functions used for certificates configuration
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
+public final class SecurityUtil {
+ private SecurityUtil() {}
+ private static final Logger logger = LoggerFactory.getLogger(SecurityUtil.class);
+
+ public static String getKeystorePasswordFromFile(String passwordPath) {
+ return getPasswordFromFile(passwordPath, "Keystore");
+ }
+
+ public static String getTruststorePasswordFromFile(String passwordPath) {
+ return getPasswordFromFile(passwordPath, "Truststore");
+ }
+
+ public static String getPasswordFromFile(String passwordPath, String element) {
+ try {
+ return new String(Files.readAllBytes(Paths.get(passwordPath)));
+ } catch (IOException e) {
+ logger.error("{} password file at path: {} cannot be opened ", element, passwordPath);
+ }
+ return "";
+ }
+}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java
index d933e337..b381c021 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java
@@ -1,6 +1,7 @@
/*-
* ============LICENSE_START======================================================================
- * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved.
+ * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation.
+ * All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -37,6 +38,7 @@ import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil;
import org.onap.dcaegen2.collectors.datafile.model.logging.MappedDiagnosticContext;
import org.onap.dcaegen2.services.sdk.rest.services.cbs.client.api.CbsClient;
import org.onap.dcaegen2.services.sdk.rest.services.cbs.client.api.CbsClientFactory;
@@ -76,7 +78,7 @@ public class AppConfig {
Properties systemEnvironment;
private ConsumerConfiguration dmaapConsumerConfiguration;
private Map<String, PublisherConfiguration> publishingConfigurations;
- private FtpesConfig ftpesConfiguration;
+ private CertificateConfig certificateConfiguration;
private SftpConfig sftpConfiguration;
private Disposable refreshConfigTask = null;
@@ -163,8 +165,8 @@ public class AppConfig {
return cfg;
}
- public synchronized FtpesConfig getFtpesConfiguration() {
- return ftpesConfiguration;
+ public synchronized CertificateConfig getCertificateConfiguration() {
+ return certificateConfiguration;
}
public synchronized SftpConfig getSftpConfiguration() {
@@ -193,7 +195,7 @@ public class AppConfig {
CloudConfigParser parser =
new CloudConfigParser(configurationObject, systemEnvironment);
setConfiguration(parser.getConsumerConfiguration(),
- parser.getDmaapPublisherConfigurations(), parser.getFtpesConfig(),
+ parser.getDmaapPublisherConfigurations(), parser.getCertificateConfig(),
parser.getSftpConfig());
logConfig();
} catch (DatafileTaskException e) {
@@ -204,7 +206,7 @@ public class AppConfig {
private void logConfig() {
logger.debug("Read and parsed sFTP configuration: [{}]", sftpConfiguration);
- logger.debug("Read and parsed FTPes configuration: [{}]", ftpesConfiguration);
+ logger.debug("Read and parsed FTPes / HTTPS configuration: [{}]", certificateConfiguration);
logger.debug("Read and parsed DMaaP configuration: [{}]", dmaapConsumerConfiguration);
logger.debug("Read and parsed Publish configuration: [{}]", publishingConfigurations);
}
@@ -226,12 +228,14 @@ public class AppConfig {
}
private synchronized void setConfiguration(@NotNull ConsumerConfiguration consumerConfiguration,
- @NotNull Map<String, PublisherConfiguration> publisherConfiguration, @NotNull FtpesConfig ftpesConfig,
- @NotNull SftpConfig sftpConfig) {
+ @NotNull Map<String, PublisherConfiguration> publisherConfiguration, @NotNull CertificateConfig certificateConfig,
+ @NotNull SftpConfig sftpConfig) throws DatafileTaskException {
this.dmaapConsumerConfiguration = consumerConfiguration;
this.publishingConfigurations = publisherConfiguration;
- this.ftpesConfiguration = ftpesConfig;
+ this.certificateConfiguration = certificateConfig;
this.sftpConfiguration = sftpConfig;
+ HttpsClientConnectionManagerUtil.setupOrUpdate(certificateConfig.keyCert(), certificateConfig.keyPasswordPath(),
+ certificateConfig.trustedCa(), certificateConfig.trustedCaPasswordPath());
}
JsonElement getJsonElement(InputStream inputStream) {
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java
index 9e9da7db..1d8b6143 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java
@@ -30,7 +30,7 @@ import org.springframework.stereotype.Component;
@Value.Immutable
@Value.Style(builder = "new", redactedMask = "####")
@Gson.TypeAdapters
-public abstract class FtpesConfig implements Serializable {
+public abstract class CertificateConfig implements Serializable {
private static final long serialVersionUID = 1L;
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java
index 6ace4aae..d6b86433 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java
@@ -1,6 +1,7 @@
/*-
* ============LICENSE_START=======================================================
- * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved.
+ * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation.
+ * All rights reserved.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -187,12 +188,12 @@ public class CloudConfigParser {
* @return the xNF communication security configuration.
* @throws DatafileTaskException if a member of the configuration is missing.
*/
- public @NotNull FtpesConfig getFtpesConfig() throws DatafileTaskException {
- return new ImmutableFtpesConfig.Builder() //
- .keyCert(getAsString(jsonObject, "dmaap.ftpesConfig.keyCert"))
- .keyPasswordPath(getAsString(jsonObject, "dmaap.ftpesConfig.keyPasswordPath"))
- .trustedCa(getAsString(jsonObject, "dmaap.ftpesConfig.trustedCa"))
- .trustedCaPasswordPath(getAsString(jsonObject, "dmaap.ftpesConfig.trustedCaPasswordPath")) //
+ public @NotNull CertificateConfig getCertificateConfig() throws DatafileTaskException {
+ return new ImmutableCertificateConfig.Builder() //
+ .keyCert(getAsString(jsonObject, "dmaap.certificateConfig.keyCert"))
+ .keyPasswordPath(getAsString(jsonObject, "dmaap.certificateConfig.keyPasswordPath"))
+ .trustedCa(getAsString(jsonObject, "dmaap.certificateConfig.trustedCa"))
+ .trustedCaPasswordPath(getAsString(jsonObject, "dmaap.certificateConfig.trustedCaPasswordPath")) //
.build();
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java
index 9bacec88..a82b5478 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java
@@ -1,7 +1,7 @@
/*-
* ============LICENSE_START======================================================================
* Copyright (C) 2018-2019 Nordix Foundation. All rights reserved.
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -22,9 +22,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.KeyStoreException;
@@ -43,6 +41,7 @@ import org.apache.commons.net.ftp.FTPReply;
import org.apache.commons.net.ftp.FTPSClient;
import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient;
import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
+import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
import org.slf4j.Logger;
@@ -194,13 +193,7 @@ public class FtpesClient implements FileCollectClient {
protected TrustManager getTrustManager(Path trustedCaPath, String trustedCaPasswordPath)
throws KeyStoreException, NoSuchAlgorithmException, IOException, CertificateException {
- String trustedCaPassword = "";
- try {
- trustedCaPassword = new String(Files.readAllBytes(Paths.get(trustedCaPasswordPath)));
- } catch (IOException e) {
- logger.error("Truststore password file at path: {} cannot be opened ", trustedCaPasswordPath);
- e.printStackTrace();
- }
+ String trustedCaPassword = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath);
synchronized (FtpesClient.class) {
if (theTrustManager == null) {
theTrustManager = createTrustManager(trustedCaPath, trustedCaPassword);
@@ -211,14 +204,7 @@ public class FtpesClient implements FileCollectClient {
protected KeyManager getKeyManager(Path keyCertPath, String keyCertPasswordPath)
throws IOException, GeneralSecurityException {
- String keyCertPassword = "";
- try {
- keyCertPassword = new String(Files.readAllBytes(Paths.get(keyCertPasswordPath)));
- } catch (IOException e) {
- logger.error("Keystore password file at path: {} cannot be opened ", keyCertPasswordPath);
- e.printStackTrace();
- }
-
+ String keyCertPassword = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath);
synchronized (FtpesClient.class) {
if (theKeyManager == null) {
theKeyManager = createKeyManager(keyCertPath, keyCertPassword);
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java
index 86bfc210..3ccc9fb2 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java
@@ -1,6 +1,6 @@
/*-
* ============LICENSE_START======================================================================
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -37,6 +37,11 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
+/**
+ * Gets file from PNF with HTTP protocol.
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
public class DfcHttpClient implements FileCollectClient {
//Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS
@@ -111,7 +116,7 @@ public class DfcHttpClient implements FileCollectClient {
try {
long numBytes = Files.copy(response, localFile);
logger.trace("Transmission was successful - {} bytes downloaded.", numBytes);
- logger.trace("CollectFile fetched: {}", localFile.toString());
+ logger.trace("CollectFile fetched: {}", localFile);
response.close();
} catch (IOException e) {
errorMessages.set(new Exception("Error fetching file with", e));
@@ -139,7 +144,7 @@ public class DfcHttpClient implements FileCollectClient {
}
@NotNull protected String prepareUri(String remoteFile) {
- int port = fileServerData.port().isPresent() ? fileServerData.port().get() : HttpUtils.HTTP_DEFAULT_PORT;
+ int port = fileServerData.port().orElse(HttpUtils.HTTP_DEFAULT_PORT);
return "http://" + fileServerData.serverAddress() + ":" + port + remoteFile;
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java
new file mode 100644
index 00000000..3090815a
--- /dev/null
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java
@@ -0,0 +1,170 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.conn.HttpHostConnectException;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.util.EntityUtils;
+import org.jetbrains.annotations.NotNull;
+import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient;
+import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
+import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.service.HttpUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.SSLHandshakeException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+/**
+ * Gets file from PNF with HTTPS protocol.
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
+public class DfcHttpsClient implements FileCollectClient {
+
+ protected CloseableHttpClient httpsClient;
+
+ private static final Logger logger = LoggerFactory.getLogger(DfcHttpsClient.class);
+ private static final int FIFTEEN_SECONDS = 15 * 1000;
+
+ private final FileServerData fileServerData;
+ private final PoolingHttpClientConnectionManager connectionManager;
+
+ public DfcHttpsClient(FileServerData fileServerData, PoolingHttpClientConnectionManager connectionManager) {
+ this.fileServerData = fileServerData;
+ this.connectionManager = connectionManager;
+ }
+
+ @Override public void open() {
+ logger.trace("Setting httpsClient for file download.");
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(true)
+ .build();
+
+ RequestConfig requestConfig = RequestConfig.custom()
+ .setConnectTimeout(FIFTEEN_SECONDS)
+ .build();
+
+ httpsClient = HttpClients.custom()
+ .setConnectionManager(connectionManager)
+ .setDefaultSocketConfig(socketConfig)
+ .setDefaultRequestConfig(requestConfig)
+ .build();
+
+ logger.trace("httpsClient prepared for connection.");
+ }
+
+ @Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException {
+ logger.trace("Prepare to collectFile {}", localFile);
+ HttpGet httpGet = new HttpGet(prepareUri(remoteFile));
+ if (basicAuthValidNotPresentOrThrow()) {
+ httpGet.addHeader("Authorization",
+ HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password()));
+ }
+ try {
+ HttpResponse httpResponse = makeCall(httpGet);
+ processResponse(httpResponse, localFile);
+ } catch (IOException e) {
+ throw new DatafileTaskException("Error downloading file from server. ", e);
+ }
+ logger.trace("HTTPS collectFile OK");
+ }
+
+ protected boolean basicAuthValidNotPresentOrThrow() throws DatafileTaskException {
+ if (isAuthDataEmpty()) {
+ return false;
+ }
+ if (isAuthDataFilled()) {
+ return true;
+ }
+ throw new DatafileTaskException("Not sufficient basic auth data for file.");
+ }
+
+ private boolean isAuthDataEmpty() {
+ return this.fileServerData.userId().isEmpty() && this.fileServerData.password().isEmpty();
+ }
+
+ private boolean isAuthDataFilled() {
+ return !this.fileServerData.userId().isEmpty() && !this.fileServerData.password().isEmpty();
+ }
+
+ @NotNull protected String prepareUri(String remoteFile) {
+ int port = fileServerData.port().orElse(HttpUtils.HTTPS_DEFAULT_PORT);
+ return "https://" + fileServerData.serverAddress() + ":" + port + remoteFile;
+ }
+
+ protected HttpResponse makeCall(HttpGet httpGet)
+ throws IOException, DatafileTaskException {
+ try {
+ HttpResponse httpResponse = executeHttpClient(httpGet);
+ if (isResponseOk(httpResponse)) {
+ return httpResponse;
+ }
+
+ EntityUtils.consume(httpResponse.getEntity());
+ throw new NonRetryableDatafileTaskException(
+ "Unexpected response code - " + httpResponse.getStatusLine().getStatusCode()
+ + ". No retry attempts will be done.");
+
+ } catch (ConnectTimeoutException | UnknownHostException | HttpHostConnectException | SSLHandshakeException e) {
+ throw new NonRetryableDatafileTaskException(
+ "Unable to get file from xNF. No retry attempts will be done.", e);
+ }
+ }
+
+ protected CloseableHttpResponse executeHttpClient(HttpGet httpGet)
+ throws IOException {
+ return httpsClient.execute(httpGet);
+ }
+
+ protected boolean isResponseOk(HttpResponse response) {
+ return response.getStatusLine().getStatusCode() == 200;
+ }
+
+ protected void processResponse(HttpResponse response, Path localFile) throws IOException {
+ logger.trace("Starting to process response.");
+ HttpEntity entity = response.getEntity();
+ InputStream stream = entity.getContent();
+ long numBytes = writeFile(localFile, stream);
+ stream.close();
+ EntityUtils.consume(entity);
+ logger.trace("Transmission was successful - {} bytes downloaded.", numBytes);
+ }
+
+ protected long writeFile(Path localFile, InputStream stream) throws IOException {
+ return Files.copy(stream, localFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ @Override public void close() {
+ logger.trace("Https client has ended downloading process.");
+ }
+}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java
new file mode 100644
index 00000000..e60ec0f4
--- /dev/null
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java
@@ -0,0 +1,132 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
+import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil;
+import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.io.FileSystemResource;
+
+import javax.net.ssl.SSLContext;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+
+/**
+ * Utility class supplying connection manager for HTTPS protocol.
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
+public class HttpsClientConnectionManagerUtil {
+
+ private HttpsClientConnectionManagerUtil() {
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(HttpsClientConnectionManagerUtil.class);
+ //Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS
+ private static final int MAX_NUMBER_OF_CONNECTIONS = 200;
+ private static PoolingHttpClientConnectionManager connectionManager;
+
+ public static PoolingHttpClientConnectionManager instance() throws DatafileTaskException {
+ if (connectionManager == null) {
+ throw new DatafileTaskException("ConnectionManager has to be set or update first");
+ }
+ return connectionManager;
+ }
+
+ public static void setupOrUpdate(String keyCertPath, String keyCertPasswordPath, String trustedCaPath,
+ String trustedCaPasswordPath) throws DatafileTaskException {
+ synchronized (HttpsClientConnectionManagerUtil.class) {
+ if (connectionManager != null) {
+ connectionManager.close();
+ connectionManager = null;
+ }
+ setup(keyCertPath, keyCertPasswordPath, trustedCaPath, trustedCaPasswordPath);
+ }
+ logger.trace("HttpsConnectionManager setup or updated");
+ }
+
+ private static void setup(String keyCertPath, String keyCertPasswordPath, String trustedCaPath,
+ String trustedCaPasswordPath) throws DatafileTaskException {
+ try {
+ SSLContextBuilder sslBuilder = SSLContexts.custom();
+ sslBuilder = supplyKeyInfo(keyCertPath, keyCertPasswordPath, sslBuilder);
+ sslBuilder = supplyTrustInfo(trustedCaPath, trustedCaPasswordPath, sslBuilder);
+
+ SSLContext sslContext = sslBuilder.build();
+
+ SSLConnectionSocketFactory sslConnectionSocketFactory =
+ new SSLConnectionSocketFactory(sslContext, new String[] {"TLSv1.2"}, null,
+ (hostname, session) -> true);
+
+ Registry<ConnectionSocketFactory> socketFactoryRegistry =
+ RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslConnectionSocketFactory)
+ .build();
+
+ connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connectionManager.setMaxTotal(MAX_NUMBER_OF_CONNECTIONS);
+
+ } catch (Exception e) {
+ throw new DatafileTaskException("Unable to prepare HttpsConnectionManager : ", e);
+ }
+ }
+
+ private static SSLContextBuilder supplyKeyInfo(String keyCertPath, String keyCertPasswordPath,
+ SSLContextBuilder sslBuilder)
+ throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException,
+ UnrecoverableKeyException {
+ String keyPass = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath);
+ KeyStore keyFile = createKeyStore(keyCertPath, keyPass);
+ return sslBuilder.loadKeyMaterial(keyFile, keyPass.toCharArray());
+ }
+
+ private static KeyStore createKeyStore(String trustedCaPath, String trustedCaPassword)
+ throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException {
+ logger.trace("Creating trust manager from file: {}", trustedCaPath);
+ try (InputStream fis = createInputStream(trustedCaPath)) {
+ KeyStore keyStore = KeyStore.getInstance("PKCS12");
+ keyStore.load(fis, trustedCaPassword.toCharArray());
+ return keyStore;
+ }
+ }
+
+ private static InputStream createInputStream(String localFileName) throws IOException {
+ FileSystemResource realResource = new FileSystemResource(Paths.get(localFileName));
+ return realResource.getInputStream();
+ }
+
+ private static SSLContextBuilder supplyTrustInfo(String trustedCaPath, String trustedCaPasswordPath,
+ SSLContextBuilder sslBuilder)
+ throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException {
+ String trustPass = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath);
+ File trustStoreFile = new File(trustedCaPath);
+ return sslBuilder.loadTrustMaterial(trustStoreFile, trustPass.toCharArray());
+ }
+}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java
index 8e8d847c..d5587e97 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java
@@ -33,7 +33,9 @@ public class Counters {
private final AtomicInteger numberOfSubscriptions = new AtomicInteger();
private int noOfCollectedFiles = 0;
private int noOfFailedFtpAttempts = 0;
+ private int noOfFailedHttpAttempts = 0;
private int noOfFailedFtp = 0;
+ private int noOfFailedHttp = 0;
private int noOfFailedPublishAttempts = 0;
private int totalPublishedFiles = 0;
private int noOfFailedPublish = 0;
@@ -62,10 +64,18 @@ public class Counters {
noOfFailedFtpAttempts++;
}
+ public synchronized void incNoOfFailedHttpAttempts() {
+ noOfFailedHttpAttempts++;
+ }
+
public synchronized void incNoOfFailedFtp() {
noOfFailedFtp++;
}
+ public synchronized void incNoOfFailedHttp() {
+ noOfFailedHttp++;
+ }
+
public synchronized void incNoOfFailedPublishAttempts() {
noOfFailedPublishAttempts++;
}
@@ -89,7 +99,9 @@ public class Counters {
str.append("\n");
str.append(format("collectedFiles", noOfCollectedFiles));
str.append(format("failedFtpAttempts", noOfFailedFtpAttempts));
+ str.append(format("failedHttpAttempts", noOfFailedHttpAttempts));
str.append(format("failedFtp", noOfFailedFtp));
+ str.append(format("failedHttp", noOfFailedHttp));
str.append("\n");
str.append(format("totalPublishedFiles", totalPublishedFiles));
str.append(format("lastPublishedTime", lastPublishedTime));
@@ -113,10 +125,18 @@ public class Counters {
return noOfFailedFtpAttempts;
}
+ public int getNoOfFailedHttpAttempts() {
+ return noOfFailedHttpAttempts;
+ }
+
public int getNoOfFailedFtp() {
return noOfFailedFtp;
}
+ public int getNoOfFailedHttp() {
+ return noOfFailedHttp;
+ }
+
public int getNoOfFailedPublishAttempts() {
return noOfFailedPublishAttempts;
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java
index 1dca0058..e2c1e2ff 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java
@@ -26,6 +26,7 @@ import java.util.Base64;
public final class HttpUtils implements HttpStatus {
public static final int HTTP_DEFAULT_PORT = 80;
+ public static final int HTTPS_DEFAULT_PORT = 443;
private HttpUtils() {
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java
index e76d4156..cfc77549 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java
@@ -23,8 +23,9 @@ import java.time.Duration;
import java.util.Map;
import java.util.Optional;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
-import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
+import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient;
@@ -32,6 +33,8 @@ import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient;
import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient;
import org.onap.dcaegen2.collectors.datafile.ftp.SftpClientSettings;
import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient;
+import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient;
+import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil;
import org.onap.dcaegen2.collectors.datafile.model.Counters;
import org.onap.dcaegen2.collectors.datafile.model.FileData;
import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
@@ -109,11 +112,11 @@ public class FileCollector {
return Mono.just(Optional.of(getFilePublishInformation(fileData, localFile, context)));
} catch (NonRetryableDatafileTaskException nre) {
logger.warn("Failed to download file: {} {}, reason: {}", fileData.sourceName(), fileData.name(), nre);
- counters.incNoOfFailedFtpAttempts();
+ incFailedAttemptsCounter(fileData);
return Mono.just(Optional.empty()); // Give up
} catch (DatafileTaskException e) {
logger.warn("Failed to download file: {} {}, reason: ", fileData.sourceName(), fileData.name(), e);
- counters.incNoOfFailedFtpAttempts();
+ incFailedAttemptsCounter(fileData);
return Mono.error(e);
} catch (Exception throwable) {
logger.warn("Failed to close client: {} {}, reason: {}", fileData.sourceName(), fileData.name(),
@@ -122,6 +125,14 @@ public class FileCollector {
}
}
+ private void incFailedAttemptsCounter(FileData fileData) {
+ if (Scheme.isFtpScheme(fileData.scheme())) {
+ counters.incNoOfFailedFtpAttempts();
+ } else {
+ counters.incNoOfFailedHttpAttempts();
+ }
+ }
+
private FileCollectClient createClient(FileData fileData) throws DatafileTaskException {
switch (fileData.scheme()) {
case SFTP:
@@ -130,6 +141,8 @@ public class FileCollector {
return createFtpesClient(fileData);
case HTTP:
return createHttpClient(fileData);
+ case HTTPS:
+ return createHttpsClient(fileData);
default:
throw new DatafileTaskException("Unhandled protocol: " + fileData.scheme());
}
@@ -163,7 +176,7 @@ public class FileCollector {
}
protected FtpesClient createFtpesClient(FileData fileData) {
- FtpesConfig config = datafileAppConfig.getFtpesConfiguration();
+ CertificateConfig config = datafileAppConfig.getCertificateConfiguration();
return new FtpesClient(fileData.fileServerData(), Paths.get(config.keyCert()), config.keyPasswordPath(),
Paths.get(config.trustedCa()), config.trustedCaPasswordPath());
}
@@ -171,4 +184,8 @@ public class FileCollector {
protected FileCollectClient createHttpClient(FileData fileData) {
return new DfcHttpClient(fileData.fileServerData());
}
+
+ protected FileCollectClient createHttpsClient(FileData fileData) throws DatafileTaskException {
+ return new DfcHttpsClient(fileData.fileServerData(), HttpsClientConnectionManagerUtil.instance());
+ }
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java
index eba0a6cb..fa1757e2 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java
@@ -23,6 +23,7 @@ import java.time.Instant;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.model.Counters;
@@ -257,7 +258,11 @@ public class ScheduledTasks {
deleteFile(localFilePath, fileData.context);
publishedFilesCache.remove(localFilePath);
currentNumberOfTasks.decrementAndGet();
- counters.incNoOfFailedFtp();
+ if (Scheme.isFtpScheme(fileData.fileData.scheme())) {
+ counters.incNoOfFailedFtp();
+ } else {
+ counters.incNoOfFailedHttp();
+ }
return Mono.empty();
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java
index 7f0c6422..bcbe7f82 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java
@@ -1,6 +1,7 @@
/*-
* ============LICENSE_START======================================================================
- * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved.
+ * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation.
+ * All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -83,12 +84,12 @@ public class AppConfigTest {
.password("izBJD8nLjawq0HMG") //
.build();
- private static final ImmutableFtpesConfig CORRECT_FTPES_CONFIGURATION = //
- new ImmutableFtpesConfig.Builder() //
+ private static final ImmutableCertificateConfig CORRECT_CERTIFICATE_CONFIGURATION = //
+ new ImmutableCertificateConfig.Builder() //
.keyCert("/src/test/resources/dfc.jks") //
.keyPasswordPath("/src/test/resources/dfc.jks.pass") //
- .trustedCa("/src/test/resources/ftp.jks") //
- .trustedCaPasswordPath("/src/test/resources/ftp.jks.pass") //
+ .trustedCa("/src/test/resources/cert.jks") //
+ .trustedCaPasswordPath("/src/test/resources/cert.jks.pass") //
.build();
private AppConfig appConfigUnderTest;
@@ -120,9 +121,9 @@ public class AppConfigTest {
Assertions.assertNotNull(publisherCfg);
assertThat(publisherCfg).isEqualToComparingFieldByField(CORRECT_PUBLISHER_CONFIG);
- FtpesConfig ftpesConfig = appConfigUnderTest.getFtpesConfiguration();
- assertThat(ftpesConfig).isNotNull();
- assertThat(ftpesConfig).isEqualToComparingFieldByField(CORRECT_FTPES_CONFIGURATION);
+ CertificateConfig certificateConfig = appConfigUnderTest.getCertificateConfiguration();
+ assertThat(certificateConfig).isNotNull();
+ assertThat(certificateConfig).isEqualToComparingFieldByField(CORRECT_CERTIFICATE_CONFIGURATION);
}
@Test
@@ -157,7 +158,7 @@ public class AppConfigTest {
assertThatThrownBy(() -> appConfigUnderTest.getPublisherConfiguration(CHANGE_IDENTIFIER))
.hasMessageContaining("No PublishingConfiguration loaded, changeIdentifier: PM_MEAS_FILES");
- Assertions.assertNull(appConfigUnderTest.getFtpesConfiguration());
+ Assertions.assertNull(appConfigUnderTest.getCertificateConfiguration());
}
@Test
@@ -172,7 +173,7 @@ public class AppConfigTest {
Assertions.assertNull(appConfigUnderTest.getDmaapConsumerConfiguration());
assertThatThrownBy(() -> appConfigUnderTest.getPublisherConfiguration(CHANGE_IDENTIFIER))
.hasMessageContaining(CHANGE_IDENTIFIER);
- Assertions.assertNull(appConfigUnderTest.getFtpesConfiguration());
+ Assertions.assertNull(appConfigUnderTest.getCertificateConfiguration());
}
@Test
@@ -190,7 +191,7 @@ public class AppConfigTest {
Assertions.assertNull(appConfigUnderTest.getDmaapConsumerConfiguration());
assertThatThrownBy(() -> appConfigUnderTest.getPublisherConfiguration(CHANGE_IDENTIFIER))
.hasMessageContaining(CHANGE_IDENTIFIER);
- Assertions.assertNull(appConfigUnderTest.getFtpesConfiguration());
+ Assertions.assertNull(appConfigUnderTest.getCertificateConfiguration());
}
@Test
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
index f49cd391..2013950a 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
@@ -1,6 +1,6 @@
/*-
* ============LICENSE_START======================================================================
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -43,7 +43,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
-public class DfcHttpClientTest {
+class DfcHttpClientTest {
private static final String USERNAME = "bob";
private static final String PASSWORD = "123";
@@ -55,28 +55,20 @@ public class DfcHttpClientTest {
DfcHttpClient dfcHttpClientSpy;
- private ImmutableFileServerData createFileServerData() {
- return ImmutableFileServerData.builder()
- .serverAddress(XNF_ADDRESS)
- .userId(USERNAME).password(PASSWORD)
- .port(PORT)
- .build();
- }
-
@BeforeEach
public void setup() {
dfcHttpClientSpy = spy(new DfcHttpClient(createFileServerData()));
}
@Test
- public void openConnection_successAuthSetup() throws DatafileTaskException {
+ void openConnection_successAuthSetup() throws DatafileTaskException {
dfcHttpClientSpy.open();
HttpClientConfig config = dfcHttpClientSpy.client.configuration();
assertEquals(HttpUtils.basicAuth(USERNAME, PASSWORD), config.headers().get("Authorization"));
}
@Test
- public void openConnection_failedBasicAuthSetupThrowException() {
+ void openConnection_failedBasicAuthSetupThrowException() {
ImmutableFileServerData serverData = ImmutableFileServerData.builder()
.serverAddress(XNF_ADDRESS)
.userId(USERNAME).password("")
@@ -90,7 +82,7 @@ public class DfcHttpClientTest {
}
@Test
- public void prepareUri_UriWithoutPort() {
+ void prepareUri_UriWithoutPort() {
ImmutableFileServerData serverData = ImmutableFileServerData.builder()
.serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD)
@@ -103,7 +95,7 @@ public class DfcHttpClientTest {
}
@Test
- public void collectFile_AllOk() throws Exception {
+ void collectFile_AllOk() throws Exception {
String REMOTE_FILE = "any";
Flux<InputStream> fis = Flux.just(new ByteArrayInputStream("ReturnedString".getBytes()));
@@ -121,7 +113,7 @@ public class DfcHttpClientTest {
}
@Test
- public void collectFile_No200ResponseWriteToErrorMessage() throws DatafileTaskException {
+ void collectFile_No200ResponseWriteToErrorMessage() throws DatafileTaskException {
String ERROR_RESPONSE = "This is unexpected message";
String REMOTE_FILE = "any";
Flux<Throwable> fis = Flux.error(new Throwable(ERROR_RESPONSE));
@@ -138,8 +130,16 @@ public class DfcHttpClientTest {
}
@Test
- public void isResponseOk_validateResponse() {
- assertTrue(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.RESPONSE_OK));
+ void isResponseOk_validateResponse() {
+ assertTrue(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.NETTY_RESPONSE_OK));
assertFalse(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.RESPONSE_ANY_NO_OK));
}
+
+ private ImmutableFileServerData createFileServerData() {
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId(USERNAME).password(PASSWORD)
+ .port(PORT)
+ .build();
+ }
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java
new file mode 100644
index 00000000..8df91d3a
--- /dev/null
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java
@@ -0,0 +1,164 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData;
+import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+@ExtendWith(MockitoExtension.class)
+class DfcHttpsClientTest {
+
+ private static final String USERNAME = "bob";
+ private static final String PASSWORD = "123";
+ private static final String XNF_ADDRESS = "127.0.0.1";
+ private static final int PORT = 443;
+ private static String remoteFile = "remoteFile";
+
+ @Mock
+ private PoolingHttpClientConnectionManager connectionManager;
+ @Mock
+ private Path localFile;
+
+ DfcHttpsClient dfcHttpsClientSpy;
+
+ @BeforeEach
+ public void setup() {
+ dfcHttpsClientSpy = spy(new DfcHttpsClient(createFileServerData(), connectionManager));
+ }
+
+ @Test
+ void fileServerData_properLocationBasicAuth() throws Exception {
+ boolean result = dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow();
+ assertEquals(true, result);
+ }
+
+ @Test
+ void fileServerData_properLocationNoBasicAuth() throws Exception {
+ dfcHttpsClientSpy = spy(new DfcHttpsClient(emptyUserInFileServerData(), connectionManager));
+
+ boolean result = dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow();
+ assertEquals(false, result);
+ }
+
+ @Test
+ void fileServerData_improperAuthDataExceptionOccurred() throws Exception {
+ dfcHttpsClientSpy = spy(new DfcHttpsClient(invalidUserInFileServerData(), connectionManager));
+
+ assertThrows(DatafileTaskException.class, () -> dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow());
+ }
+
+ @Test
+ void dfcHttpsClient_flow_successfulCallAndResponseProcessing() throws Exception {
+ doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy)
+ .executeHttpClient(any(HttpGet.class));
+ doReturn((long)3).when(dfcHttpsClientSpy).writeFile(eq(localFile), any(InputStream.class));
+
+ dfcHttpsClientSpy.open();
+ dfcHttpsClientSpy.collectFile(remoteFile, localFile);
+ dfcHttpsClientSpy.close();
+
+ verify(dfcHttpsClientSpy, times(1)).makeCall(any(HttpGet.class));
+ verify(dfcHttpsClientSpy, times(1))
+ .executeHttpClient(any(HttpGet.class));
+ verify(dfcHttpsClientSpy, times(1))
+ .processResponse(HttpClientResponseHelper.APACHE_RESPONSE_OK, localFile);
+ verify(dfcHttpsClientSpy, times(1))
+ .writeFile(eq(localFile), any(InputStream.class));
+ }
+
+ @Test
+ void dfcHttpsClient_flow_failedCallUnexpectedResponseCode() throws Exception {
+ doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy)
+ .executeHttpClient(any(HttpGet.class));
+ doReturn(false).when(dfcHttpsClientSpy).isResponseOk(any(HttpResponse.class));
+
+ dfcHttpsClientSpy.open();
+
+ assertThrows(NonRetryableDatafileTaskException.class,
+ () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile));
+ }
+
+ @Test
+ void dfcHttpsClient_flow_failedCallConnectionTimeout() throws Exception {
+ doThrow(ConnectTimeoutException.class).when(dfcHttpsClientSpy)
+ .executeHttpClient(any(HttpGet.class));
+
+ dfcHttpsClientSpy.open();
+
+ assertThrows(NonRetryableDatafileTaskException.class,
+ () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile));
+ }
+
+ @Test
+ void dfcHttpsClient_flow_failedCallIOExceptionForExecuteHttpClient() throws Exception {
+ doThrow(IOException.class).when(dfcHttpsClientSpy)
+ .executeHttpClient(any(HttpGet.class));
+
+ dfcHttpsClientSpy.open();
+
+ assertThrows(DatafileTaskException.class,
+ () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile));
+ }
+
+ private ImmutableFileServerData createFileServerData() {
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId(USERNAME).password(PASSWORD)
+ .port(PORT)
+ .build();
+ }
+
+ private ImmutableFileServerData emptyUserInFileServerData() {
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("").password("")
+ .port(PORT)
+ .build();
+ }
+
+ private ImmutableFileServerData invalidUserInFileServerData() {
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("demo").password("")
+ .port(PORT)
+ .build();
+ }
+}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java
index 42ab4b3a..0d52858d 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java
@@ -20,16 +20,28 @@ import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpResponseStatus;
import io.netty.handler.codec.http.HttpVersion;
import io.netty.handler.codec.http.cookie.Cookie;
+import org.apache.http.Header;
+import org.apache.http.HeaderIterator;
+import org.apache.http.HttpEntity;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.params.HttpParams;
import reactor.netty.http.client.HttpClientResponse;
import reactor.util.context.Context;
import reactor.util.context.ContextView;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
public class HttpClientResponseHelper {
- public static final HttpClientResponse RESPONSE_OK = new HttpClientResponse() {
+ public static final HttpClientResponse NETTY_RESPONSE_OK = new HttpClientResponse() {
@Override
public Map<CharSequence, Set<Cookie>> cookies() {
@@ -167,4 +179,166 @@ public class HttpClientResponseHelper {
return HttpResponseStatus.NOT_IMPLEMENTED;
}
};
+
+ public static final CloseableHttpResponse APACHE_RESPONSE_OK = new CloseableHttpResponse() {
+ @Override public void close() throws IOException {
+ getEntity().getContent().close();
+ }
+
+ @Override public StatusLine getStatusLine() {
+ return new StatusLine() {
+ @Override public ProtocolVersion getProtocolVersion() {
+ return null;
+ }
+
+ @Override public int getStatusCode() {
+ return 200;
+ }
+
+ @Override public String getReasonPhrase() {
+ return null;
+ }
+ };
+ }
+
+ @Override public void setStatusLine(StatusLine statusLine) {
+
+ }
+
+ @Override public void setStatusLine(ProtocolVersion protocolVersion, int i) {
+
+ }
+
+ @Override public void setStatusLine(ProtocolVersion protocolVersion, int i, String s) {
+
+ }
+
+ @Override public void setStatusCode(int i) throws IllegalStateException {
+
+ }
+
+ @Override public void setReasonPhrase(String s) throws IllegalStateException {
+
+ }
+
+ @Override public HttpEntity getEntity() {
+ return new HttpEntity() {
+ @Override public boolean isRepeatable() {
+ return false;
+ }
+
+ @Override public boolean isChunked() {
+ return false;
+ }
+
+ @Override public long getContentLength() {
+ return 0;
+ }
+
+ @Override public Header getContentType() {
+ return null;
+ }
+
+ @Override public Header getContentEncoding() {
+ return null;
+ }
+
+ @Override public InputStream getContent() throws IOException, UnsupportedOperationException {
+ return new ByteArrayInputStream("abc".getBytes());
+ }
+
+ @Override public void writeTo(OutputStream outputStream) throws IOException {
+
+ }
+
+ @Override public boolean isStreaming() {
+ return false;
+ }
+
+ @Override public void consumeContent() throws IOException {
+
+ }
+ };
+ }
+
+ @Override public void setEntity(HttpEntity httpEntity) {
+
+ }
+
+ @Override public Locale getLocale() {
+ return null;
+ }
+
+ @Override public void setLocale(Locale locale) {
+
+ }
+
+ @Override public ProtocolVersion getProtocolVersion() {
+ return null;
+ }
+
+ @Override public boolean containsHeader(String s) {
+ return false;
+ }
+
+ @Override public Header[] getHeaders(String s) {
+ return new Header[0];
+ }
+
+ @Override public Header getFirstHeader(String s) {
+ return null;
+ }
+
+ @Override public Header getLastHeader(String s) {
+ return null;
+ }
+
+ @Override public Header[] getAllHeaders() {
+ return new Header[0];
+ }
+
+ @Override public void addHeader(Header header) {
+
+ }
+
+ @Override public void addHeader(String s, String s1) {
+
+ }
+
+ @Override public void setHeader(Header header) {
+
+ }
+
+ @Override public void setHeader(String s, String s1) {
+
+ }
+
+ @Override public void setHeaders(Header[] headers) {
+
+ }
+
+ @Override public void removeHeader(Header header) {
+
+ }
+
+ @Override public void removeHeaders(String s) {
+
+ }
+
+ @Override public HeaderIterator headerIterator() {
+ return null;
+ }
+
+ @Override public HeaderIterator headerIterator(String s) {
+ return null;
+ }
+
+ @Override public HttpParams getParams() {
+ return null;
+ }
+
+ @Override public void setParams(HttpParams httpParams) {
+
+ }
+ };
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java
new file mode 100644
index 00000000..fa4473a6
--- /dev/null
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java
@@ -0,0 +1,52 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+import org.junit.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+@ExtendWith(MockitoExtension.class)
+public class HttpsClientConnectionManagerUtilTest {
+
+ private static final String KEY_PATH = "src/test/resources/keystore.p12";
+ private static final String KEY_PASSWORD = "src/test/resources/keystore.pass";
+ private static final String KEY_IMPROPER_PASSWORD = "src/test/resources/dfc.jks.pass";
+ private static final String TRUSTED_CA_PATH = "src/test/resources/trust.jks";
+ private static final String TRUSTED_CA_PASSWORD = "src/test/resources/trust.pass";
+
+ @Test
+ public void emptyManager_shouldThrowException() {
+ assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.instance());
+ }
+
+ @Test
+ public void creatingManager_successfulCase() throws Exception {
+ HttpsClientConnectionManagerUtil.setupOrUpdate(KEY_PATH, KEY_PASSWORD, TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD);
+ assertNotNull(HttpsClientConnectionManagerUtil.instance());
+ }
+
+ @Test
+ public void creatingManager_improperSecretShouldThrowException() {
+ assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.setupOrUpdate(KEY_PATH, KEY_IMPROPER_PASSWORD, TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD));
+ assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.instance());
+ }
+
+}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java
index fb475791..61adef9a 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java
@@ -34,6 +34,7 @@ public class SchemeTest {
assertEquals(Scheme.FTPES, Scheme.getSchemeFromString("FTPES"));
assertEquals(Scheme.SFTP, Scheme.getSchemeFromString("SFTP"));
assertEquals(Scheme.HTTP, Scheme.getSchemeFromString("HTTP"));
+ assertEquals(Scheme.HTTPS, Scheme.getSchemeFromString("HTTPS"));
}
@Test
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java
index 1146cf26..ceb8a989 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java
@@ -38,12 +38,14 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
-import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
+import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient;
import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient;
+import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient;
+import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient;
import org.onap.dcaegen2.collectors.datafile.model.Counters;
import org.onap.dcaegen2.collectors.datafile.model.FileData;
import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
@@ -64,6 +66,8 @@ public class FileCollectorTest {
private static final String FILE_READY_CHANGE_TYPE = "FileReady";
private static final String FTPES_SCHEME = "ftpes://";
private static final String SFTP_SCHEME = "sftp://";
+ private static final String HTTP_SCHEME = "http://";
+ private static final String HTTPS_SCHEME = "https://";
private static final String SERVER_ADDRESS = "192.168.0.101";
private static final int PORT_22 = 22;
private static final String PM_FILE_NAME = "A20161224.1030-1045.bin.gz";
@@ -79,24 +83,36 @@ public class FileCollectorTest {
private static final String SFTP_LOCATION = SFTP_SCHEME + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
private static final String SFTP_LOCATION_NO_PORT = SFTP_SCHEME + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
+ private static final String HTTP_LOCATION =
+ HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
+ private static final String HTTP_LOCATION_NO_PORT =
+ HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
+ private static final String HTTPS_LOCATION =
+ HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
+ private static final String HTTPS_LOCATION_NO_PORT =
+ HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
+
private static final String GZIP_COMPRESSION = "gzip";
private static final String MEAS_COLLECT_FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec";
private static final String FILE_FORMAT_VERSION = "V10";
- private static final String FTP_KEY_PATH = "ftpKeyPath";
- private static final String FTP_KEY_PASSWORD_PATH = "ftpKeyPassword";
+ private static final String CERTIFICATE_KEY_PATH = "certificateKeyPath";
+ private static final String CERTIFICATE_KEY_PASSWORD_PATH = "certificateKeyPassword";
private static final String TRUSTED_CA_PATH = "trustedCAPath";
private static final String TRUSTED_CA_PASSWORD_PATH = "trustedCAPassword";
private static final String CHANGE_IDENTIFIER = "PM_MEAS_FILES";
private static AppConfig appConfigMock = mock(AppConfig.class);
- private static FtpesConfig ftpesConfigMock = mock(FtpesConfig.class);
+ private static CertificateConfig certificateConfigMock = mock(CertificateConfig.class);
private FtpesClient ftpesClientMock = mock(FtpesClient.class);
private SftpClient sftpClientMock = mock(SftpClient.class);
private final Map<String, String> contextMap = new HashMap<>();
+ private DfcHttpClient dfcHttpClientMock = mock(DfcHttpClient.class);
+ private DfcHttpsClient dfcHttpsClientMock = mock(DfcHttpsClient.class);
+
private Counters counters;
private MessageMetaData createMessageMetaData() {
@@ -145,11 +161,11 @@ public class FileCollectorTest {
@BeforeAll
static void setUpConfiguration() {
- when(appConfigMock.getFtpesConfiguration()).thenReturn(ftpesConfigMock);
- when(ftpesConfigMock.keyCert()).thenReturn(FTP_KEY_PATH);
- when(ftpesConfigMock.keyPasswordPath()).thenReturn(FTP_KEY_PASSWORD_PATH);
- when(ftpesConfigMock.trustedCa()).thenReturn(TRUSTED_CA_PATH);
- when(ftpesConfigMock.trustedCaPasswordPath()).thenReturn(TRUSTED_CA_PASSWORD_PATH);
+ when(appConfigMock.getCertificateConfiguration()).thenReturn(certificateConfigMock);
+ when(certificateConfigMock.keyCert()).thenReturn(CERTIFICATE_KEY_PATH);
+ when(certificateConfigMock.keyPasswordPath()).thenReturn(CERTIFICATE_KEY_PASSWORD_PATH);
+ when(certificateConfigMock.trustedCa()).thenReturn(TRUSTED_CA_PATH);
+ when(certificateConfigMock.trustedCaPasswordPath()).thenReturn(TRUSTED_CA_PASSWORD_PATH);
}
@BeforeEach
@@ -178,6 +194,7 @@ public class FileCollectorTest {
assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
}
@Test
@@ -210,6 +227,70 @@ public class FileCollectorTest {
}
@Test
+ public void whenHttpFile_returnCorrectResponse() throws Exception {
+ FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
+ doReturn(dfcHttpClientMock).when(collectorUndetTest).createHttpClient(any());
+
+ FileData fileData = createFileData(HTTP_LOCATION_NO_PORT, Scheme.HTTP);
+
+ FilePublishInformation expectedfilePublishInformation =
+ createExpectedFilePublishInformation(HTTP_LOCATION_NO_PORT);
+
+ StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
+ .expectNext(expectedfilePublishInformation) //
+ .verifyComplete();
+
+ // The same again, but with port
+ fileData = createFileData(HTTP_LOCATION, Scheme.HTTP);
+ expectedfilePublishInformation = createExpectedFilePublishInformation(HTTP_LOCATION);
+
+ StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
+ .expectNext(expectedfilePublishInformation) //
+ .verifyComplete();
+
+ verify(dfcHttpClientMock, times(2)).open();
+ verify(dfcHttpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
+ verify(dfcHttpClientMock, times(2)).close();
+ verifyNoMoreInteractions(dfcHttpClientMock);
+
+ assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
+ assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
+ }
+
+ @Test
+ public void whenHttpsFile_returnCorrectResponse() throws Exception {
+ FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
+ doReturn(dfcHttpsClientMock).when(collectorUndetTest).createHttpsClient(any());
+
+ FileData fileData = createFileData(HTTPS_LOCATION_NO_PORT, Scheme.HTTPS);
+
+ FilePublishInformation expectedfilePublishInformation =
+ createExpectedFilePublishInformation(HTTPS_LOCATION_NO_PORT);
+
+ StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
+ .expectNext(expectedfilePublishInformation) //
+ .verifyComplete();
+
+ // The same again, but with port
+ fileData = createFileData(HTTPS_LOCATION, Scheme.HTTPS);
+ expectedfilePublishInformation = createExpectedFilePublishInformation(HTTPS_LOCATION);
+
+ StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
+ .expectNext(expectedfilePublishInformation) //
+ .verifyComplete();
+
+ verify(dfcHttpsClientMock, times(2)).open();
+ verify(dfcHttpsClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
+ verify(dfcHttpsClientMock, times(2)).close();
+ verifyNoMoreInteractions(dfcHttpsClientMock);
+
+ assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
+ assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
+ }
+
+ @Test
public void whenFtpesFileAlwaysFail_retryAndFail() throws Exception {
FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
@@ -226,6 +307,7 @@ public class FileCollectorTest {
assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0");
assertEquals(4, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 4");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
}
@Test
@@ -245,6 +327,7 @@ public class FileCollectorTest {
assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0");
assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
}
@Test
@@ -267,5 +350,6 @@ public class FileCollectorTest {
assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
}
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java
index 09d7627e..85f52ff8 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java
@@ -392,6 +392,7 @@ public class ScheduledTasksTest {
assertEquals(2, testedObject.getCounters().getTotalReceivedEvents(),"totalReceivedEvents should have been 2");
assertEquals(1, testedObject.getCounters().getNoOfFailedFtp(),"failedFtp should have been 1");
+ assertEquals(0, testedObject.getCounters().getNoOfFailedHttp(),"failedHttp should have been 0");
}
@Test
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java
new file mode 100644
index 00000000..75ad8a48
--- /dev/null
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java
@@ -0,0 +1,120 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.utils;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+/**
+ * Tests the SecurityUtil.
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(SecurityUtil.class)
+@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"})
+public class SecurityUtilTest {
+
+ @Mock
+ Path path;
+
+ private static final String expectedPassword = "password";
+ private static final String validPath = "/validPath";
+ private static final String invalidPath = "/invalidPath";
+
+ @Test
+ public void whenGetKeystorePasswordFromFile_passwordSuccessfullyReturned() throws Exception {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(validPath)).thenReturn(path);
+ when(Files.readAllBytes(path)).thenReturn("password".getBytes());
+
+ String result = SecurityUtil.getKeystorePasswordFromFile(validPath);
+ assertEquals(expectedPassword, result);
+
+ }
+
+ @Test
+ public void whenGetKeystorePasswordFromFile_IOExceptionForValidPath() throws Exception {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(validPath)).thenReturn(path);
+ when(Files.readAllBytes(path)).thenThrow(IOException.class);
+
+ assertEquals("", SecurityUtil.getKeystorePasswordFromFile(validPath));
+
+ }
+
+ @Test
+ public void whenGetKeystorePasswordFromFile_InvalidPathExceptionForInvalidPath() {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(invalidPath)).thenThrow(InvalidPathException.class);
+
+ assertThrows(InvalidPathException.class, () -> SecurityUtil.getKeystorePasswordFromFile(invalidPath));
+
+ }
+
+ @Test
+ public void whenGetTruststorePasswordFromFile_passwordSuccessfullyReturned() throws Exception {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(validPath)).thenReturn(path);
+ when(Files.readAllBytes(path)).thenReturn("password".getBytes());
+
+ String result = SecurityUtil.getTruststorePasswordFromFile(validPath);
+ assertEquals(expectedPassword, result);
+
+ }
+
+ @Test
+ public void whenGetTruststorePasswordFromFile_IOExceptionForValidPath() throws Exception {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(validPath)).thenReturn(path);
+ when(Files.readAllBytes(path)).thenThrow(IOException.class);
+
+ assertEquals("", SecurityUtil.getTruststorePasswordFromFile(validPath));
+
+ }
+
+ @Test
+ public void whenGetTruststorePasswordFromFile_InvalidPathExceptionForInvalidPath() {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(invalidPath)).thenThrow(InvalidPathException.class);
+
+ assertThrows(InvalidPathException.class, () -> SecurityUtil.getTruststorePasswordFromFile(invalidPath));
+
+ }
+
+}
diff --git a/datafile-app-server/src/test/resources/datafile_endpoints_test.json b/datafile-app-server/src/test/resources/datafile_endpoints_test.json
index 8e51b807..00b2488b 100644
--- a/datafile-app-server/src/test/resources/datafile_endpoints_test.json
+++ b/datafile-app-server/src/test/resources/datafile_endpoints_test.json
@@ -1,10 +1,10 @@
{
"config": {
"//description": "This file is only used for testing purposes",
- "dmaap.ftpesConfig.keyCert": "/src/test/resources/dfc.jks",
- "dmaap.ftpesConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass",
- "dmaap.ftpesConfig.trustedCa": "/src/test/resources/ftp.jks",
- "dmaap.ftpesConfig.trustedCaPasswordPath": "/src/test/resources/ftp.jks.pass",
+ "dmaap.certificateConfig.keyCert": "/src/test/resources/dfc.jks",
+ "dmaap.certificateConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass",
+ "dmaap.certificateConfig.trustedCa": "/src/test/resources/cert.jks",
+ "dmaap.certificateConfig.trustedCaPasswordPath": "/src/test/resources/cert.jks.pass",
"dmaap.security.trustStorePath": "src/test/resources/trust.jks",
"dmaap.security.trustStorePasswordPath": "src/test/resources/trust.pass",
"dmaap.security.keyStorePath": "src/test/resources/cert.jks",
diff --git a/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json b/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json
index e1327462..73bb7448 100644
--- a/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json
+++ b/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json
@@ -1,10 +1,10 @@
{
"config": {
"//description": "This file is only used for testing purposes",
- "dmaap.ftpesConfig.keyCert": "/src/test/resources/dfc.jks",
- "dmaap.ftpesConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass",
- "dmaap.ftpesConfig.trustedCa": "/src/test/resources/ftp.jks",
- "dmaap.ftpesConfig.trustedCaPasswordPath": "/src/test/resources/ftp.jks.pass",
+ "dmaap.certificateConfig.keyCert": "/src/test/resources/dfc.jks",
+ "dmaap.certificateConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass",
+ "dmaap.certificateConfig.trustedCa": "/src/test/resources/ftp.jks",
+ "dmaap.certificateConfig.trustedCaPasswordPath": "/src/test/resources/ftp.jks.pass",
"dmaap.security.trustStorePath": "src/test/resources/trust.jks",
"dmaap.security.trustStorePasswordPath": "src/test/resources/trust.pass",
"dmaap.security.keyStorePath": "src/test/resources/cert.jks",
diff --git a/datafile-app-server/src/test/resources/keystore.p12 b/datafile-app-server/src/test/resources/keystore.p12
new file mode 100644
index 00000000..b847707c
--- /dev/null
+++ b/datafile-app-server/src/test/resources/keystore.p12
Binary files differ
diff --git a/datafile-app-server/src/test/resources/keystore.pass b/datafile-app-server/src/test/resources/keystore.pass
new file mode 100644
index 00000000..1e7befca
--- /dev/null
+++ b/datafile-app-server/src/test/resources/keystore.pass
@@ -0,0 +1 @@
+HVpAf0kHGl4P#fdpblJLka6b \ No newline at end of file