diff options
Diffstat (limited to 'datafile-app-server')
28 files changed, 1109 insertions, 98 deletions
diff --git a/datafile-app-server/dpo/spec/datafile-component-spec.json b/datafile-app-server/dpo/spec/datafile-component-spec.json index a6505183..9de69138 100644 --- a/datafile-app-server/dpo/spec/datafile-component-spec.json +++ b/datafile-app-server/dpo/spec/datafile-component-spec.json @@ -61,7 +61,7 @@ ], "parameters": [ { - "name": "dmaap.ftpesConfig.keyCert", + "name": "dmaap.certificateConfig.keyCert", "value": "/opt/app/datafile/etc/cert/cert.p12", "description": "", "designer_editable": true, @@ -71,7 +71,7 @@ "required": true }, { - "name": "dmaap.ftpesConfig.keyPasswordPath", + "name": "dmaap.certificateConfig.keyPasswordPath", "value": "/opt/app/datafile/etc/cert/p12.pass", "description": "", "designer_editable": true, @@ -81,7 +81,7 @@ "required": true }, { - "name": "dmaap.ftpesConfig.trustedCa", + "name": "dmaap.certificateConfig.trustedCa", "value": "/opt/app/datafile/etc/cert/trust.jks", "description": "", "designer_editable": true, @@ -91,7 +91,7 @@ "required": true }, { - "name": "dmaap.ftpesConfig.trustedCaPasswordPath", + "name": "dmaap.certificateConfig.trustedCaPasswordPath", "value": "/opt/app/datafile/etc/cert/trust.pass", "description": "", "designer_editable": true, diff --git a/datafile-app-server/pom.xml b/datafile-app-server/pom.xml index 353b375b..5e0330b0 100644 --- a/datafile-app-server/pom.xml +++ b/datafile-app-server/pom.xml @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- ~ ============LICENSE_START======================================================= - ~ Copyright (C) 2018-2020 NOKIA Intellectual Property. All rights reserved. + ~ Copyright (C) 2018-2021 NOKIA Intellectual Property. All rights reserved. ~ Copyright (C) 2018-2021 Nordix Foundation. All rights reserved. ~ Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. ~ ================================================================================ @@ -26,7 +26,7 @@ <parent> <groupId>org.onap.dcaegen2.collectors</groupId> <artifactId>datafile</artifactId> - <version>1.5.2-SNAPSHOT</version> + <version>1.5.3-SNAPSHOT</version> </parent> <groupId>org.onap.dcaegen2.collectors.datafile</groupId> @@ -158,6 +158,16 @@ <artifactId>mockito-junit-jupiter</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito2</artifactId> + <scope>test</scope> + </dependency> <!--REQUIRED TO GENERATE DOCUMENTATION --> <dependency> diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java index afa3aaea..b9aa6449 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START======================================================= * Copyright (C) 2019 Nordix Foundation. All rights reserved. - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,10 +28,10 @@ import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; * */ public enum Scheme { - FTPES, SFTP, HTTP; + FTPES, SFTP, HTTP, HTTPS; public static final String DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG = "DFC does not support protocol "; - public static final String SUPPORTED_PROTOCOLS_ERROR_MESSAGE = ". Supported protocols are FTPeS, sFTP and HTTP"; + public static final String SUPPORTED_PROTOCOLS_ERROR_MESSAGE = ". Supported protocols are FTPeS, sFTP, HTTP and HTTPS"; /** * Get a <code>Scheme</code> from a string. @@ -48,10 +48,22 @@ public enum Scheme { result = Scheme.SFTP; } else if ("HTTP".equalsIgnoreCase(schemeString)) { result = Scheme.HTTP; + } else if ("HTTPS".equalsIgnoreCase(schemeString)) { + result = Scheme.HTTPS; } else { throw new DatafileTaskException( DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG + schemeString + SUPPORTED_PROTOCOLS_ERROR_MESSAGE); } return result; } + + /** + * Check if <code>Scheme</code> is FTP type or HTTP type. + * + * @param scheme the <code>Scheme</code> which has to be checked. + * @return true if <code>Scheme</code> is FTP type or false if it is HTTP type + */ + public static boolean isFtpScheme(Scheme scheme) { + return scheme == SFTP || scheme == FTPES; + } } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java new file mode 100644 index 00000000..79db32d3 --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java @@ -0,0 +1,50 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.commons; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +/** + * Utility class containing functions used for certificates configuration + * + * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a> + */ +public final class SecurityUtil { + private SecurityUtil() {} + private static final Logger logger = LoggerFactory.getLogger(SecurityUtil.class); + + public static String getKeystorePasswordFromFile(String passwordPath) { + return getPasswordFromFile(passwordPath, "Keystore"); + } + + public static String getTruststorePasswordFromFile(String passwordPath) { + return getPasswordFromFile(passwordPath, "Truststore"); + } + + public static String getPasswordFromFile(String passwordPath, String element) { + try { + return new String(Files.readAllBytes(Paths.get(passwordPath))); + } catch (IOException e) { + logger.error("{} password file at path: {} cannot be opened ", element, passwordPath); + } + return ""; + } +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java index d933e337..b381c021 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START====================================================================== - * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. + * All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -37,6 +38,7 @@ import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil; import org.onap.dcaegen2.collectors.datafile.model.logging.MappedDiagnosticContext; import org.onap.dcaegen2.services.sdk.rest.services.cbs.client.api.CbsClient; import org.onap.dcaegen2.services.sdk.rest.services.cbs.client.api.CbsClientFactory; @@ -76,7 +78,7 @@ public class AppConfig { Properties systemEnvironment; private ConsumerConfiguration dmaapConsumerConfiguration; private Map<String, PublisherConfiguration> publishingConfigurations; - private FtpesConfig ftpesConfiguration; + private CertificateConfig certificateConfiguration; private SftpConfig sftpConfiguration; private Disposable refreshConfigTask = null; @@ -163,8 +165,8 @@ public class AppConfig { return cfg; } - public synchronized FtpesConfig getFtpesConfiguration() { - return ftpesConfiguration; + public synchronized CertificateConfig getCertificateConfiguration() { + return certificateConfiguration; } public synchronized SftpConfig getSftpConfiguration() { @@ -193,7 +195,7 @@ public class AppConfig { CloudConfigParser parser = new CloudConfigParser(configurationObject, systemEnvironment); setConfiguration(parser.getConsumerConfiguration(), - parser.getDmaapPublisherConfigurations(), parser.getFtpesConfig(), + parser.getDmaapPublisherConfigurations(), parser.getCertificateConfig(), parser.getSftpConfig()); logConfig(); } catch (DatafileTaskException e) { @@ -204,7 +206,7 @@ public class AppConfig { private void logConfig() { logger.debug("Read and parsed sFTP configuration: [{}]", sftpConfiguration); - logger.debug("Read and parsed FTPes configuration: [{}]", ftpesConfiguration); + logger.debug("Read and parsed FTPes / HTTPS configuration: [{}]", certificateConfiguration); logger.debug("Read and parsed DMaaP configuration: [{}]", dmaapConsumerConfiguration); logger.debug("Read and parsed Publish configuration: [{}]", publishingConfigurations); } @@ -226,12 +228,14 @@ public class AppConfig { } private synchronized void setConfiguration(@NotNull ConsumerConfiguration consumerConfiguration, - @NotNull Map<String, PublisherConfiguration> publisherConfiguration, @NotNull FtpesConfig ftpesConfig, - @NotNull SftpConfig sftpConfig) { + @NotNull Map<String, PublisherConfiguration> publisherConfiguration, @NotNull CertificateConfig certificateConfig, + @NotNull SftpConfig sftpConfig) throws DatafileTaskException { this.dmaapConsumerConfiguration = consumerConfiguration; this.publishingConfigurations = publisherConfiguration; - this.ftpesConfiguration = ftpesConfig; + this.certificateConfiguration = certificateConfig; this.sftpConfiguration = sftpConfig; + HttpsClientConnectionManagerUtil.setupOrUpdate(certificateConfig.keyCert(), certificateConfig.keyPasswordPath(), + certificateConfig.trustedCa(), certificateConfig.trustedCaPasswordPath()); } JsonElement getJsonElement(InputStream inputStream) { diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java index 9e9da7db..1d8b6143 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java @@ -30,7 +30,7 @@ import org.springframework.stereotype.Component; @Value.Immutable @Value.Style(builder = "new", redactedMask = "####") @Gson.TypeAdapters -public abstract class FtpesConfig implements Serializable { +public abstract class CertificateConfig implements Serializable { private static final long serialVersionUID = 1L; diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java index 6ace4aae..d6b86433 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START======================================================= - * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. + * All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -187,12 +188,12 @@ public class CloudConfigParser { * @return the xNF communication security configuration. * @throws DatafileTaskException if a member of the configuration is missing. */ - public @NotNull FtpesConfig getFtpesConfig() throws DatafileTaskException { - return new ImmutableFtpesConfig.Builder() // - .keyCert(getAsString(jsonObject, "dmaap.ftpesConfig.keyCert")) - .keyPasswordPath(getAsString(jsonObject, "dmaap.ftpesConfig.keyPasswordPath")) - .trustedCa(getAsString(jsonObject, "dmaap.ftpesConfig.trustedCa")) - .trustedCaPasswordPath(getAsString(jsonObject, "dmaap.ftpesConfig.trustedCaPasswordPath")) // + public @NotNull CertificateConfig getCertificateConfig() throws DatafileTaskException { + return new ImmutableCertificateConfig.Builder() // + .keyCert(getAsString(jsonObject, "dmaap.certificateConfig.keyCert")) + .keyPasswordPath(getAsString(jsonObject, "dmaap.certificateConfig.keyPasswordPath")) + .trustedCa(getAsString(jsonObject, "dmaap.certificateConfig.trustedCa")) + .trustedCaPasswordPath(getAsString(jsonObject, "dmaap.certificateConfig.trustedCaPasswordPath")) // .build(); } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java index 9bacec88..a82b5478 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java @@ -1,7 +1,7 @@ /*- * ============LICENSE_START====================================================================== * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -22,9 +22,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.KeyStoreException; @@ -43,6 +41,7 @@ import org.apache.commons.net.ftp.FTPReply; import org.apache.commons.net.ftp.FTPSClient; import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; import org.slf4j.Logger; @@ -194,13 +193,7 @@ public class FtpesClient implements FileCollectClient { protected TrustManager getTrustManager(Path trustedCaPath, String trustedCaPasswordPath) throws KeyStoreException, NoSuchAlgorithmException, IOException, CertificateException { - String trustedCaPassword = ""; - try { - trustedCaPassword = new String(Files.readAllBytes(Paths.get(trustedCaPasswordPath))); - } catch (IOException e) { - logger.error("Truststore password file at path: {} cannot be opened ", trustedCaPasswordPath); - e.printStackTrace(); - } + String trustedCaPassword = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath); synchronized (FtpesClient.class) { if (theTrustManager == null) { theTrustManager = createTrustManager(trustedCaPath, trustedCaPassword); @@ -211,14 +204,7 @@ public class FtpesClient implements FileCollectClient { protected KeyManager getKeyManager(Path keyCertPath, String keyCertPasswordPath) throws IOException, GeneralSecurityException { - String keyCertPassword = ""; - try { - keyCertPassword = new String(Files.readAllBytes(Paths.get(keyCertPasswordPath))); - } catch (IOException e) { - logger.error("Keystore password file at path: {} cannot be opened ", keyCertPasswordPath); - e.printStackTrace(); - } - + String keyCertPassword = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath); synchronized (FtpesClient.class) { if (theKeyManager == null) { theKeyManager = createKeyManager(keyCertPath, keyCertPassword); diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java index 86bfc210..3ccc9fb2 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START====================================================================== - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -37,6 +37,11 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +/** + * Gets file from PNF with HTTP protocol. + * + * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a> + */ public class DfcHttpClient implements FileCollectClient { //Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS @@ -111,7 +116,7 @@ public class DfcHttpClient implements FileCollectClient { try { long numBytes = Files.copy(response, localFile); logger.trace("Transmission was successful - {} bytes downloaded.", numBytes); - logger.trace("CollectFile fetched: {}", localFile.toString()); + logger.trace("CollectFile fetched: {}", localFile); response.close(); } catch (IOException e) { errorMessages.set(new Exception("Error fetching file with", e)); @@ -139,7 +144,7 @@ public class DfcHttpClient implements FileCollectClient { } @NotNull protected String prepareUri(String remoteFile) { - int port = fileServerData.port().isPresent() ? fileServerData.port().get() : HttpUtils.HTTP_DEFAULT_PORT; + int port = fileServerData.port().orElse(HttpUtils.HTTP_DEFAULT_PORT); return "http://" + fileServerData.serverAddress() + ":" + port + remoteFile; } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java new file mode 100644 index 00000000..3090815a --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java @@ -0,0 +1,170 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.conn.HttpHostConnectException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.util.EntityUtils; +import org.jetbrains.annotations.NotNull; +import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.service.HttpUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLHandshakeException; +import java.io.IOException; +import java.io.InputStream; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +/** + * Gets file from PNF with HTTPS protocol. + * + * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a> + */ +public class DfcHttpsClient implements FileCollectClient { + + protected CloseableHttpClient httpsClient; + + private static final Logger logger = LoggerFactory.getLogger(DfcHttpsClient.class); + private static final int FIFTEEN_SECONDS = 15 * 1000; + + private final FileServerData fileServerData; + private final PoolingHttpClientConnectionManager connectionManager; + + public DfcHttpsClient(FileServerData fileServerData, PoolingHttpClientConnectionManager connectionManager) { + this.fileServerData = fileServerData; + this.connectionManager = connectionManager; + } + + @Override public void open() { + logger.trace("Setting httpsClient for file download."); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(true) + .build(); + + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(FIFTEEN_SECONDS) + .build(); + + httpsClient = HttpClients.custom() + .setConnectionManager(connectionManager) + .setDefaultSocketConfig(socketConfig) + .setDefaultRequestConfig(requestConfig) + .build(); + + logger.trace("httpsClient prepared for connection."); + } + + @Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException { + logger.trace("Prepare to collectFile {}", localFile); + HttpGet httpGet = new HttpGet(prepareUri(remoteFile)); + if (basicAuthValidNotPresentOrThrow()) { + httpGet.addHeader("Authorization", + HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password())); + } + try { + HttpResponse httpResponse = makeCall(httpGet); + processResponse(httpResponse, localFile); + } catch (IOException e) { + throw new DatafileTaskException("Error downloading file from server. ", e); + } + logger.trace("HTTPS collectFile OK"); + } + + protected boolean basicAuthValidNotPresentOrThrow() throws DatafileTaskException { + if (isAuthDataEmpty()) { + return false; + } + if (isAuthDataFilled()) { + return true; + } + throw new DatafileTaskException("Not sufficient basic auth data for file."); + } + + private boolean isAuthDataEmpty() { + return this.fileServerData.userId().isEmpty() && this.fileServerData.password().isEmpty(); + } + + private boolean isAuthDataFilled() { + return !this.fileServerData.userId().isEmpty() && !this.fileServerData.password().isEmpty(); + } + + @NotNull protected String prepareUri(String remoteFile) { + int port = fileServerData.port().orElse(HttpUtils.HTTPS_DEFAULT_PORT); + return "https://" + fileServerData.serverAddress() + ":" + port + remoteFile; + } + + protected HttpResponse makeCall(HttpGet httpGet) + throws IOException, DatafileTaskException { + try { + HttpResponse httpResponse = executeHttpClient(httpGet); + if (isResponseOk(httpResponse)) { + return httpResponse; + } + + EntityUtils.consume(httpResponse.getEntity()); + throw new NonRetryableDatafileTaskException( + "Unexpected response code - " + httpResponse.getStatusLine().getStatusCode() + + ". No retry attempts will be done."); + + } catch (ConnectTimeoutException | UnknownHostException | HttpHostConnectException | SSLHandshakeException e) { + throw new NonRetryableDatafileTaskException( + "Unable to get file from xNF. No retry attempts will be done.", e); + } + } + + protected CloseableHttpResponse executeHttpClient(HttpGet httpGet) + throws IOException { + return httpsClient.execute(httpGet); + } + + protected boolean isResponseOk(HttpResponse response) { + return response.getStatusLine().getStatusCode() == 200; + } + + protected void processResponse(HttpResponse response, Path localFile) throws IOException { + logger.trace("Starting to process response."); + HttpEntity entity = response.getEntity(); + InputStream stream = entity.getContent(); + long numBytes = writeFile(localFile, stream); + stream.close(); + EntityUtils.consume(entity); + logger.trace("Transmission was successful - {} bytes downloaded.", numBytes); + } + + protected long writeFile(Path localFile, InputStream stream) throws IOException { + return Files.copy(stream, localFile, StandardCopyOption.REPLACE_EXISTING); + } + + @Override public void close() { + logger.trace("Https client has ended downloading process."); + } +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java new file mode 100644 index 00000000..e60ec0f4 --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java @@ -0,0 +1,132 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; +import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.FileSystemResource; + +import javax.net.ssl.SSLContext; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; + +/** + * Utility class supplying connection manager for HTTPS protocol. + * + * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a> + */ +public class HttpsClientConnectionManagerUtil { + + private HttpsClientConnectionManagerUtil() { + } + + private static final Logger logger = LoggerFactory.getLogger(HttpsClientConnectionManagerUtil.class); + //Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS + private static final int MAX_NUMBER_OF_CONNECTIONS = 200; + private static PoolingHttpClientConnectionManager connectionManager; + + public static PoolingHttpClientConnectionManager instance() throws DatafileTaskException { + if (connectionManager == null) { + throw new DatafileTaskException("ConnectionManager has to be set or update first"); + } + return connectionManager; + } + + public static void setupOrUpdate(String keyCertPath, String keyCertPasswordPath, String trustedCaPath, + String trustedCaPasswordPath) throws DatafileTaskException { + synchronized (HttpsClientConnectionManagerUtil.class) { + if (connectionManager != null) { + connectionManager.close(); + connectionManager = null; + } + setup(keyCertPath, keyCertPasswordPath, trustedCaPath, trustedCaPasswordPath); + } + logger.trace("HttpsConnectionManager setup or updated"); + } + + private static void setup(String keyCertPath, String keyCertPasswordPath, String trustedCaPath, + String trustedCaPasswordPath) throws DatafileTaskException { + try { + SSLContextBuilder sslBuilder = SSLContexts.custom(); + sslBuilder = supplyKeyInfo(keyCertPath, keyCertPasswordPath, sslBuilder); + sslBuilder = supplyTrustInfo(trustedCaPath, trustedCaPasswordPath, sslBuilder); + + SSLContext sslContext = sslBuilder.build(); + + SSLConnectionSocketFactory sslConnectionSocketFactory = + new SSLConnectionSocketFactory(sslContext, new String[] {"TLSv1.2"}, null, + (hostname, session) -> true); + + Registry<ConnectionSocketFactory> socketFactoryRegistry = + RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslConnectionSocketFactory) + .build(); + + connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connectionManager.setMaxTotal(MAX_NUMBER_OF_CONNECTIONS); + + } catch (Exception e) { + throw new DatafileTaskException("Unable to prepare HttpsConnectionManager : ", e); + } + } + + private static SSLContextBuilder supplyKeyInfo(String keyCertPath, String keyCertPasswordPath, + SSLContextBuilder sslBuilder) + throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException, + UnrecoverableKeyException { + String keyPass = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath); + KeyStore keyFile = createKeyStore(keyCertPath, keyPass); + return sslBuilder.loadKeyMaterial(keyFile, keyPass.toCharArray()); + } + + private static KeyStore createKeyStore(String trustedCaPath, String trustedCaPassword) + throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException { + logger.trace("Creating trust manager from file: {}", trustedCaPath); + try (InputStream fis = createInputStream(trustedCaPath)) { + KeyStore keyStore = KeyStore.getInstance("PKCS12"); + keyStore.load(fis, trustedCaPassword.toCharArray()); + return keyStore; + } + } + + private static InputStream createInputStream(String localFileName) throws IOException { + FileSystemResource realResource = new FileSystemResource(Paths.get(localFileName)); + return realResource.getInputStream(); + } + + private static SSLContextBuilder supplyTrustInfo(String trustedCaPath, String trustedCaPasswordPath, + SSLContextBuilder sslBuilder) + throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException { + String trustPass = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath); + File trustStoreFile = new File(trustedCaPath); + return sslBuilder.loadTrustMaterial(trustStoreFile, trustPass.toCharArray()); + } +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java index 8e8d847c..d5587e97 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java @@ -33,7 +33,9 @@ public class Counters { private final AtomicInteger numberOfSubscriptions = new AtomicInteger(); private int noOfCollectedFiles = 0; private int noOfFailedFtpAttempts = 0; + private int noOfFailedHttpAttempts = 0; private int noOfFailedFtp = 0; + private int noOfFailedHttp = 0; private int noOfFailedPublishAttempts = 0; private int totalPublishedFiles = 0; private int noOfFailedPublish = 0; @@ -62,10 +64,18 @@ public class Counters { noOfFailedFtpAttempts++; } + public synchronized void incNoOfFailedHttpAttempts() { + noOfFailedHttpAttempts++; + } + public synchronized void incNoOfFailedFtp() { noOfFailedFtp++; } + public synchronized void incNoOfFailedHttp() { + noOfFailedHttp++; + } + public synchronized void incNoOfFailedPublishAttempts() { noOfFailedPublishAttempts++; } @@ -89,7 +99,9 @@ public class Counters { str.append("\n"); str.append(format("collectedFiles", noOfCollectedFiles)); str.append(format("failedFtpAttempts", noOfFailedFtpAttempts)); + str.append(format("failedHttpAttempts", noOfFailedHttpAttempts)); str.append(format("failedFtp", noOfFailedFtp)); + str.append(format("failedHttp", noOfFailedHttp)); str.append("\n"); str.append(format("totalPublishedFiles", totalPublishedFiles)); str.append(format("lastPublishedTime", lastPublishedTime)); @@ -113,10 +125,18 @@ public class Counters { return noOfFailedFtpAttempts; } + public int getNoOfFailedHttpAttempts() { + return noOfFailedHttpAttempts; + } + public int getNoOfFailedFtp() { return noOfFailedFtp; } + public int getNoOfFailedHttp() { + return noOfFailedHttp; + } + public int getNoOfFailedPublishAttempts() { return noOfFailedPublishAttempts; } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java index 1dca0058..e2c1e2ff 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java @@ -26,6 +26,7 @@ import java.util.Base64; public final class HttpUtils implements HttpStatus { public static final int HTTP_DEFAULT_PORT = 80; + public static final int HTTPS_DEFAULT_PORT = 443; private HttpUtils() { } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java index e76d4156..cfc77549 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java @@ -23,8 +23,9 @@ import java.time.Duration; import java.util.Map; import java.util.Optional; +import org.onap.dcaegen2.collectors.datafile.commons.Scheme; import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; -import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig; +import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; @@ -32,6 +33,8 @@ import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient; import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient; import org.onap.dcaegen2.collectors.datafile.ftp.SftpClientSettings; import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient; +import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient; +import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil; import org.onap.dcaegen2.collectors.datafile.model.Counters; import org.onap.dcaegen2.collectors.datafile.model.FileData; import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation; @@ -109,11 +112,11 @@ public class FileCollector { return Mono.just(Optional.of(getFilePublishInformation(fileData, localFile, context))); } catch (NonRetryableDatafileTaskException nre) { logger.warn("Failed to download file: {} {}, reason: {}", fileData.sourceName(), fileData.name(), nre); - counters.incNoOfFailedFtpAttempts(); + incFailedAttemptsCounter(fileData); return Mono.just(Optional.empty()); // Give up } catch (DatafileTaskException e) { logger.warn("Failed to download file: {} {}, reason: ", fileData.sourceName(), fileData.name(), e); - counters.incNoOfFailedFtpAttempts(); + incFailedAttemptsCounter(fileData); return Mono.error(e); } catch (Exception throwable) { logger.warn("Failed to close client: {} {}, reason: {}", fileData.sourceName(), fileData.name(), @@ -122,6 +125,14 @@ public class FileCollector { } } + private void incFailedAttemptsCounter(FileData fileData) { + if (Scheme.isFtpScheme(fileData.scheme())) { + counters.incNoOfFailedFtpAttempts(); + } else { + counters.incNoOfFailedHttpAttempts(); + } + } + private FileCollectClient createClient(FileData fileData) throws DatafileTaskException { switch (fileData.scheme()) { case SFTP: @@ -130,6 +141,8 @@ public class FileCollector { return createFtpesClient(fileData); case HTTP: return createHttpClient(fileData); + case HTTPS: + return createHttpsClient(fileData); default: throw new DatafileTaskException("Unhandled protocol: " + fileData.scheme()); } @@ -163,7 +176,7 @@ public class FileCollector { } protected FtpesClient createFtpesClient(FileData fileData) { - FtpesConfig config = datafileAppConfig.getFtpesConfiguration(); + CertificateConfig config = datafileAppConfig.getCertificateConfiguration(); return new FtpesClient(fileData.fileServerData(), Paths.get(config.keyCert()), config.keyPasswordPath(), Paths.get(config.trustedCa()), config.trustedCaPasswordPath()); } @@ -171,4 +184,8 @@ public class FileCollector { protected FileCollectClient createHttpClient(FileData fileData) { return new DfcHttpClient(fileData.fileServerData()); } + + protected FileCollectClient createHttpsClient(FileData fileData) throws DatafileTaskException { + return new DfcHttpsClient(fileData.fileServerData(), HttpsClientConnectionManagerUtil.instance()); + } } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java index eba0a6cb..fa1757e2 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java @@ -23,6 +23,7 @@ import java.time.Instant; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import org.onap.dcaegen2.collectors.datafile.commons.Scheme; import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.model.Counters; @@ -257,7 +258,11 @@ public class ScheduledTasks { deleteFile(localFilePath, fileData.context); publishedFilesCache.remove(localFilePath); currentNumberOfTasks.decrementAndGet(); - counters.incNoOfFailedFtp(); + if (Scheme.isFtpScheme(fileData.fileData.scheme())) { + counters.incNoOfFailedFtp(); + } else { + counters.incNoOfFailedHttp(); + } return Mono.empty(); } diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java index 7f0c6422..bcbe7f82 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START====================================================================== - * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. + * All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -83,12 +84,12 @@ public class AppConfigTest { .password("izBJD8nLjawq0HMG") // .build(); - private static final ImmutableFtpesConfig CORRECT_FTPES_CONFIGURATION = // - new ImmutableFtpesConfig.Builder() // + private static final ImmutableCertificateConfig CORRECT_CERTIFICATE_CONFIGURATION = // + new ImmutableCertificateConfig.Builder() // .keyCert("/src/test/resources/dfc.jks") // .keyPasswordPath("/src/test/resources/dfc.jks.pass") // - .trustedCa("/src/test/resources/ftp.jks") // - .trustedCaPasswordPath("/src/test/resources/ftp.jks.pass") // + .trustedCa("/src/test/resources/cert.jks") // + .trustedCaPasswordPath("/src/test/resources/cert.jks.pass") // .build(); private AppConfig appConfigUnderTest; @@ -120,9 +121,9 @@ public class AppConfigTest { Assertions.assertNotNull(publisherCfg); assertThat(publisherCfg).isEqualToComparingFieldByField(CORRECT_PUBLISHER_CONFIG); - FtpesConfig ftpesConfig = appConfigUnderTest.getFtpesConfiguration(); - assertThat(ftpesConfig).isNotNull(); - assertThat(ftpesConfig).isEqualToComparingFieldByField(CORRECT_FTPES_CONFIGURATION); + CertificateConfig certificateConfig = appConfigUnderTest.getCertificateConfiguration(); + assertThat(certificateConfig).isNotNull(); + assertThat(certificateConfig).isEqualToComparingFieldByField(CORRECT_CERTIFICATE_CONFIGURATION); } @Test @@ -157,7 +158,7 @@ public class AppConfigTest { assertThatThrownBy(() -> appConfigUnderTest.getPublisherConfiguration(CHANGE_IDENTIFIER)) .hasMessageContaining("No PublishingConfiguration loaded, changeIdentifier: PM_MEAS_FILES"); - Assertions.assertNull(appConfigUnderTest.getFtpesConfiguration()); + Assertions.assertNull(appConfigUnderTest.getCertificateConfiguration()); } @Test @@ -172,7 +173,7 @@ public class AppConfigTest { Assertions.assertNull(appConfigUnderTest.getDmaapConsumerConfiguration()); assertThatThrownBy(() -> appConfigUnderTest.getPublisherConfiguration(CHANGE_IDENTIFIER)) .hasMessageContaining(CHANGE_IDENTIFIER); - Assertions.assertNull(appConfigUnderTest.getFtpesConfiguration()); + Assertions.assertNull(appConfigUnderTest.getCertificateConfiguration()); } @Test @@ -190,7 +191,7 @@ public class AppConfigTest { Assertions.assertNull(appConfigUnderTest.getDmaapConsumerConfiguration()); assertThatThrownBy(() -> appConfigUnderTest.getPublisherConfiguration(CHANGE_IDENTIFIER)) .hasMessageContaining(CHANGE_IDENTIFIER); - Assertions.assertNull(appConfigUnderTest.getFtpesConfiguration()); + Assertions.assertNull(appConfigUnderTest.getCertificateConfiguration()); } @Test diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java index f49cd391..2013950a 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START====================================================================== - * Copyright (C) 2020 Nokia. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -43,7 +43,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) -public class DfcHttpClientTest { +class DfcHttpClientTest { private static final String USERNAME = "bob"; private static final String PASSWORD = "123"; @@ -55,28 +55,20 @@ public class DfcHttpClientTest { DfcHttpClient dfcHttpClientSpy; - private ImmutableFileServerData createFileServerData() { - return ImmutableFileServerData.builder() - .serverAddress(XNF_ADDRESS) - .userId(USERNAME).password(PASSWORD) - .port(PORT) - .build(); - } - @BeforeEach public void setup() { dfcHttpClientSpy = spy(new DfcHttpClient(createFileServerData())); } @Test - public void openConnection_successAuthSetup() throws DatafileTaskException { + void openConnection_successAuthSetup() throws DatafileTaskException { dfcHttpClientSpy.open(); HttpClientConfig config = dfcHttpClientSpy.client.configuration(); assertEquals(HttpUtils.basicAuth(USERNAME, PASSWORD), config.headers().get("Authorization")); } @Test - public void openConnection_failedBasicAuthSetupThrowException() { + void openConnection_failedBasicAuthSetupThrowException() { ImmutableFileServerData serverData = ImmutableFileServerData.builder() .serverAddress(XNF_ADDRESS) .userId(USERNAME).password("") @@ -90,7 +82,7 @@ public class DfcHttpClientTest { } @Test - public void prepareUri_UriWithoutPort() { + void prepareUri_UriWithoutPort() { ImmutableFileServerData serverData = ImmutableFileServerData.builder() .serverAddress(XNF_ADDRESS) .userId(USERNAME).password(PASSWORD) @@ -103,7 +95,7 @@ public class DfcHttpClientTest { } @Test - public void collectFile_AllOk() throws Exception { + void collectFile_AllOk() throws Exception { String REMOTE_FILE = "any"; Flux<InputStream> fis = Flux.just(new ByteArrayInputStream("ReturnedString".getBytes())); @@ -121,7 +113,7 @@ public class DfcHttpClientTest { } @Test - public void collectFile_No200ResponseWriteToErrorMessage() throws DatafileTaskException { + void collectFile_No200ResponseWriteToErrorMessage() throws DatafileTaskException { String ERROR_RESPONSE = "This is unexpected message"; String REMOTE_FILE = "any"; Flux<Throwable> fis = Flux.error(new Throwable(ERROR_RESPONSE)); @@ -138,8 +130,16 @@ public class DfcHttpClientTest { } @Test - public void isResponseOk_validateResponse() { - assertTrue(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.RESPONSE_OK)); + void isResponseOk_validateResponse() { + assertTrue(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.NETTY_RESPONSE_OK)); assertFalse(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.RESPONSE_ANY_NO_OK)); } + + private ImmutableFileServerData createFileServerData() { + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId(USERNAME).password(PASSWORD) + .port(PORT) + .build(); + } } diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java new file mode 100644 index 00000000..8df91d3a --- /dev/null +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java @@ -0,0 +1,164 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + + +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +class DfcHttpsClientTest { + + private static final String USERNAME = "bob"; + private static final String PASSWORD = "123"; + private static final String XNF_ADDRESS = "127.0.0.1"; + private static final int PORT = 443; + private static String remoteFile = "remoteFile"; + + @Mock + private PoolingHttpClientConnectionManager connectionManager; + @Mock + private Path localFile; + + DfcHttpsClient dfcHttpsClientSpy; + + @BeforeEach + public void setup() { + dfcHttpsClientSpy = spy(new DfcHttpsClient(createFileServerData(), connectionManager)); + } + + @Test + void fileServerData_properLocationBasicAuth() throws Exception { + boolean result = dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow(); + assertEquals(true, result); + } + + @Test + void fileServerData_properLocationNoBasicAuth() throws Exception { + dfcHttpsClientSpy = spy(new DfcHttpsClient(emptyUserInFileServerData(), connectionManager)); + + boolean result = dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow(); + assertEquals(false, result); + } + + @Test + void fileServerData_improperAuthDataExceptionOccurred() throws Exception { + dfcHttpsClientSpy = spy(new DfcHttpsClient(invalidUserInFileServerData(), connectionManager)); + + assertThrows(DatafileTaskException.class, () -> dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow()); + } + + @Test + void dfcHttpsClient_flow_successfulCallAndResponseProcessing() throws Exception { + doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy) + .executeHttpClient(any(HttpGet.class)); + doReturn((long)3).when(dfcHttpsClientSpy).writeFile(eq(localFile), any(InputStream.class)); + + dfcHttpsClientSpy.open(); + dfcHttpsClientSpy.collectFile(remoteFile, localFile); + dfcHttpsClientSpy.close(); + + verify(dfcHttpsClientSpy, times(1)).makeCall(any(HttpGet.class)); + verify(dfcHttpsClientSpy, times(1)) + .executeHttpClient(any(HttpGet.class)); + verify(dfcHttpsClientSpy, times(1)) + .processResponse(HttpClientResponseHelper.APACHE_RESPONSE_OK, localFile); + verify(dfcHttpsClientSpy, times(1)) + .writeFile(eq(localFile), any(InputStream.class)); + } + + @Test + void dfcHttpsClient_flow_failedCallUnexpectedResponseCode() throws Exception { + doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy) + .executeHttpClient(any(HttpGet.class)); + doReturn(false).when(dfcHttpsClientSpy).isResponseOk(any(HttpResponse.class)); + + dfcHttpsClientSpy.open(); + + assertThrows(NonRetryableDatafileTaskException.class, + () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile)); + } + + @Test + void dfcHttpsClient_flow_failedCallConnectionTimeout() throws Exception { + doThrow(ConnectTimeoutException.class).when(dfcHttpsClientSpy) + .executeHttpClient(any(HttpGet.class)); + + dfcHttpsClientSpy.open(); + + assertThrows(NonRetryableDatafileTaskException.class, + () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile)); + } + + @Test + void dfcHttpsClient_flow_failedCallIOExceptionForExecuteHttpClient() throws Exception { + doThrow(IOException.class).when(dfcHttpsClientSpy) + .executeHttpClient(any(HttpGet.class)); + + dfcHttpsClientSpy.open(); + + assertThrows(DatafileTaskException.class, + () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile)); + } + + private ImmutableFileServerData createFileServerData() { + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId(USERNAME).password(PASSWORD) + .port(PORT) + .build(); + } + + private ImmutableFileServerData emptyUserInFileServerData() { + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId("").password("") + .port(PORT) + .build(); + } + + private ImmutableFileServerData invalidUserInFileServerData() { + return ImmutableFileServerData.builder() + .serverAddress(XNF_ADDRESS) + .userId("demo").password("") + .port(PORT) + .build(); + } +} diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java index 42ab4b3a..0d52858d 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java @@ -20,16 +20,28 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http.cookie.Cookie; +import org.apache.http.Header; +import org.apache.http.HeaderIterator; +import org.apache.http.HttpEntity; +import org.apache.http.ProtocolVersion; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.params.HttpParams; import reactor.netty.http.client.HttpClientResponse; import reactor.util.context.Context; import reactor.util.context.ContextView; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Locale; import java.util.Map; import java.util.Set; public class HttpClientResponseHelper { - public static final HttpClientResponse RESPONSE_OK = new HttpClientResponse() { + public static final HttpClientResponse NETTY_RESPONSE_OK = new HttpClientResponse() { @Override public Map<CharSequence, Set<Cookie>> cookies() { @@ -167,4 +179,166 @@ public class HttpClientResponseHelper { return HttpResponseStatus.NOT_IMPLEMENTED; } }; + + public static final CloseableHttpResponse APACHE_RESPONSE_OK = new CloseableHttpResponse() { + @Override public void close() throws IOException { + getEntity().getContent().close(); + } + + @Override public StatusLine getStatusLine() { + return new StatusLine() { + @Override public ProtocolVersion getProtocolVersion() { + return null; + } + + @Override public int getStatusCode() { + return 200; + } + + @Override public String getReasonPhrase() { + return null; + } + }; + } + + @Override public void setStatusLine(StatusLine statusLine) { + + } + + @Override public void setStatusLine(ProtocolVersion protocolVersion, int i) { + + } + + @Override public void setStatusLine(ProtocolVersion protocolVersion, int i, String s) { + + } + + @Override public void setStatusCode(int i) throws IllegalStateException { + + } + + @Override public void setReasonPhrase(String s) throws IllegalStateException { + + } + + @Override public HttpEntity getEntity() { + return new HttpEntity() { + @Override public boolean isRepeatable() { + return false; + } + + @Override public boolean isChunked() { + return false; + } + + @Override public long getContentLength() { + return 0; + } + + @Override public Header getContentType() { + return null; + } + + @Override public Header getContentEncoding() { + return null; + } + + @Override public InputStream getContent() throws IOException, UnsupportedOperationException { + return new ByteArrayInputStream("abc".getBytes()); + } + + @Override public void writeTo(OutputStream outputStream) throws IOException { + + } + + @Override public boolean isStreaming() { + return false; + } + + @Override public void consumeContent() throws IOException { + + } + }; + } + + @Override public void setEntity(HttpEntity httpEntity) { + + } + + @Override public Locale getLocale() { + return null; + } + + @Override public void setLocale(Locale locale) { + + } + + @Override public ProtocolVersion getProtocolVersion() { + return null; + } + + @Override public boolean containsHeader(String s) { + return false; + } + + @Override public Header[] getHeaders(String s) { + return new Header[0]; + } + + @Override public Header getFirstHeader(String s) { + return null; + } + + @Override public Header getLastHeader(String s) { + return null; + } + + @Override public Header[] getAllHeaders() { + return new Header[0]; + } + + @Override public void addHeader(Header header) { + + } + + @Override public void addHeader(String s, String s1) { + + } + + @Override public void setHeader(Header header) { + + } + + @Override public void setHeader(String s, String s1) { + + } + + @Override public void setHeaders(Header[] headers) { + + } + + @Override public void removeHeader(Header header) { + + } + + @Override public void removeHeaders(String s) { + + } + + @Override public HeaderIterator headerIterator() { + return null; + } + + @Override public HeaderIterator headerIterator(String s) { + return null; + } + + @Override public HttpParams getParams() { + return null; + } + + @Override public void setParams(HttpParams httpParams) { + + } + }; } diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java new file mode 100644 index 00000000..fa4473a6 --- /dev/null +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java @@ -0,0 +1,52 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import org.junit.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +@ExtendWith(MockitoExtension.class) +public class HttpsClientConnectionManagerUtilTest { + + private static final String KEY_PATH = "src/test/resources/keystore.p12"; + private static final String KEY_PASSWORD = "src/test/resources/keystore.pass"; + private static final String KEY_IMPROPER_PASSWORD = "src/test/resources/dfc.jks.pass"; + private static final String TRUSTED_CA_PATH = "src/test/resources/trust.jks"; + private static final String TRUSTED_CA_PASSWORD = "src/test/resources/trust.pass"; + + @Test + public void emptyManager_shouldThrowException() { + assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.instance()); + } + + @Test + public void creatingManager_successfulCase() throws Exception { + HttpsClientConnectionManagerUtil.setupOrUpdate(KEY_PATH, KEY_PASSWORD, TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD); + assertNotNull(HttpsClientConnectionManagerUtil.instance()); + } + + @Test + public void creatingManager_improperSecretShouldThrowException() { + assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.setupOrUpdate(KEY_PATH, KEY_IMPROPER_PASSWORD, TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD)); + assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.instance()); + } + +} diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java index fb475791..61adef9a 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java @@ -34,6 +34,7 @@ public class SchemeTest { assertEquals(Scheme.FTPES, Scheme.getSchemeFromString("FTPES")); assertEquals(Scheme.SFTP, Scheme.getSchemeFromString("SFTP")); assertEquals(Scheme.HTTP, Scheme.getSchemeFromString("HTTP")); + assertEquals(Scheme.HTTPS, Scheme.getSchemeFromString("HTTPS")); } @Test diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java index 1146cf26..ceb8a989 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java @@ -38,12 +38,14 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; -import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig; +import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient; import org.onap.dcaegen2.collectors.datafile.commons.Scheme; import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient; +import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient; +import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient; import org.onap.dcaegen2.collectors.datafile.model.Counters; import org.onap.dcaegen2.collectors.datafile.model.FileData; import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation; @@ -64,6 +66,8 @@ public class FileCollectorTest { private static final String FILE_READY_CHANGE_TYPE = "FileReady"; private static final String FTPES_SCHEME = "ftpes://"; private static final String SFTP_SCHEME = "sftp://"; + private static final String HTTP_SCHEME = "http://"; + private static final String HTTPS_SCHEME = "https://"; private static final String SERVER_ADDRESS = "192.168.0.101"; private static final int PORT_22 = 22; private static final String PM_FILE_NAME = "A20161224.1030-1045.bin.gz"; @@ -79,24 +83,36 @@ public class FileCollectorTest { private static final String SFTP_LOCATION = SFTP_SCHEME + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION; private static final String SFTP_LOCATION_NO_PORT = SFTP_SCHEME + SERVER_ADDRESS + REMOTE_FILE_LOCATION; + private static final String HTTP_LOCATION = + HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION; + private static final String HTTP_LOCATION_NO_PORT = + HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION; + private static final String HTTPS_LOCATION = + HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION; + private static final String HTTPS_LOCATION_NO_PORT = + HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION; + private static final String GZIP_COMPRESSION = "gzip"; private static final String MEAS_COLLECT_FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec"; private static final String FILE_FORMAT_VERSION = "V10"; - private static final String FTP_KEY_PATH = "ftpKeyPath"; - private static final String FTP_KEY_PASSWORD_PATH = "ftpKeyPassword"; + private static final String CERTIFICATE_KEY_PATH = "certificateKeyPath"; + private static final String CERTIFICATE_KEY_PASSWORD_PATH = "certificateKeyPassword"; private static final String TRUSTED_CA_PATH = "trustedCAPath"; private static final String TRUSTED_CA_PASSWORD_PATH = "trustedCAPassword"; private static final String CHANGE_IDENTIFIER = "PM_MEAS_FILES"; private static AppConfig appConfigMock = mock(AppConfig.class); - private static FtpesConfig ftpesConfigMock = mock(FtpesConfig.class); + private static CertificateConfig certificateConfigMock = mock(CertificateConfig.class); private FtpesClient ftpesClientMock = mock(FtpesClient.class); private SftpClient sftpClientMock = mock(SftpClient.class); private final Map<String, String> contextMap = new HashMap<>(); + private DfcHttpClient dfcHttpClientMock = mock(DfcHttpClient.class); + private DfcHttpsClient dfcHttpsClientMock = mock(DfcHttpsClient.class); + private Counters counters; private MessageMetaData createMessageMetaData() { @@ -145,11 +161,11 @@ public class FileCollectorTest { @BeforeAll static void setUpConfiguration() { - when(appConfigMock.getFtpesConfiguration()).thenReturn(ftpesConfigMock); - when(ftpesConfigMock.keyCert()).thenReturn(FTP_KEY_PATH); - when(ftpesConfigMock.keyPasswordPath()).thenReturn(FTP_KEY_PASSWORD_PATH); - when(ftpesConfigMock.trustedCa()).thenReturn(TRUSTED_CA_PATH); - when(ftpesConfigMock.trustedCaPasswordPath()).thenReturn(TRUSTED_CA_PASSWORD_PATH); + when(appConfigMock.getCertificateConfiguration()).thenReturn(certificateConfigMock); + when(certificateConfigMock.keyCert()).thenReturn(CERTIFICATE_KEY_PATH); + when(certificateConfigMock.keyPasswordPath()).thenReturn(CERTIFICATE_KEY_PASSWORD_PATH); + when(certificateConfigMock.trustedCa()).thenReturn(TRUSTED_CA_PATH); + when(certificateConfigMock.trustedCaPasswordPath()).thenReturn(TRUSTED_CA_PASSWORD_PATH); } @BeforeEach @@ -178,6 +194,7 @@ public class FileCollectorTest { assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1"); assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0"); } @Test @@ -210,6 +227,70 @@ public class FileCollectorTest { } @Test + public void whenHttpFile_returnCorrectResponse() throws Exception { + FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); + doReturn(dfcHttpClientMock).when(collectorUndetTest).createHttpClient(any()); + + FileData fileData = createFileData(HTTP_LOCATION_NO_PORT, Scheme.HTTP); + + FilePublishInformation expectedfilePublishInformation = + createExpectedFilePublishInformation(HTTP_LOCATION_NO_PORT); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap)) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + // The same again, but with port + fileData = createFileData(HTTP_LOCATION, Scheme.HTTP); + expectedfilePublishInformation = createExpectedFilePublishInformation(HTTP_LOCATION); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap)) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + verify(dfcHttpClientMock, times(2)).open(); + verify(dfcHttpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + verify(dfcHttpClientMock, times(2)).close(); + verifyNoMoreInteractions(dfcHttpClientMock); + + assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1"); + assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0"); + } + + @Test + public void whenHttpsFile_returnCorrectResponse() throws Exception { + FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); + doReturn(dfcHttpsClientMock).when(collectorUndetTest).createHttpsClient(any()); + + FileData fileData = createFileData(HTTPS_LOCATION_NO_PORT, Scheme.HTTPS); + + FilePublishInformation expectedfilePublishInformation = + createExpectedFilePublishInformation(HTTPS_LOCATION_NO_PORT); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap)) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + // The same again, but with port + fileData = createFileData(HTTPS_LOCATION, Scheme.HTTPS); + expectedfilePublishInformation = createExpectedFilePublishInformation(HTTPS_LOCATION); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap)) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + verify(dfcHttpsClientMock, times(2)).open(); + verify(dfcHttpsClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + verify(dfcHttpsClientMock, times(2)).close(); + verifyNoMoreInteractions(dfcHttpsClientMock); + + assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1"); + assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0"); + } + + @Test public void whenFtpesFileAlwaysFail_retryAndFail() throws Exception { FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any()); @@ -226,6 +307,7 @@ public class FileCollectorTest { assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0"); assertEquals(4, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 4"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0"); } @Test @@ -245,6 +327,7 @@ public class FileCollectorTest { assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0"); assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0"); } @Test @@ -267,5 +350,6 @@ public class FileCollectorTest { assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1"); assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0"); } } diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java index 09d7627e..85f52ff8 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java @@ -392,6 +392,7 @@ public class ScheduledTasksTest { assertEquals(2, testedObject.getCounters().getTotalReceivedEvents(),"totalReceivedEvents should have been 2"); assertEquals(1, testedObject.getCounters().getNoOfFailedFtp(),"failedFtp should have been 1"); + assertEquals(0, testedObject.getCounters().getNoOfFailedHttp(),"failedHttp should have been 0"); } @Test diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java new file mode 100644 index 00000000..75ad8a48 --- /dev/null +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java @@ -0,0 +1,120 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.utils; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static org.powermock.api.mockito.PowerMockito.when; + +/** + * Tests the SecurityUtil. + * + * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a> + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest(SecurityUtil.class) +@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"}) +public class SecurityUtilTest { + + @Mock + Path path; + + private static final String expectedPassword = "password"; + private static final String validPath = "/validPath"; + private static final String invalidPath = "/invalidPath"; + + @Test + public void whenGetKeystorePasswordFromFile_passwordSuccessfullyReturned() throws Exception { + mockStatic(Paths.class); + mockStatic(Files.class); + when(Paths.get(validPath)).thenReturn(path); + when(Files.readAllBytes(path)).thenReturn("password".getBytes()); + + String result = SecurityUtil.getKeystorePasswordFromFile(validPath); + assertEquals(expectedPassword, result); + + } + + @Test + public void whenGetKeystorePasswordFromFile_IOExceptionForValidPath() throws Exception { + mockStatic(Paths.class); + mockStatic(Files.class); + when(Paths.get(validPath)).thenReturn(path); + when(Files.readAllBytes(path)).thenThrow(IOException.class); + + assertEquals("", SecurityUtil.getKeystorePasswordFromFile(validPath)); + + } + + @Test + public void whenGetKeystorePasswordFromFile_InvalidPathExceptionForInvalidPath() { + mockStatic(Paths.class); + mockStatic(Files.class); + when(Paths.get(invalidPath)).thenThrow(InvalidPathException.class); + + assertThrows(InvalidPathException.class, () -> SecurityUtil.getKeystorePasswordFromFile(invalidPath)); + + } + + @Test + public void whenGetTruststorePasswordFromFile_passwordSuccessfullyReturned() throws Exception { + mockStatic(Paths.class); + mockStatic(Files.class); + when(Paths.get(validPath)).thenReturn(path); + when(Files.readAllBytes(path)).thenReturn("password".getBytes()); + + String result = SecurityUtil.getTruststorePasswordFromFile(validPath); + assertEquals(expectedPassword, result); + + } + + @Test + public void whenGetTruststorePasswordFromFile_IOExceptionForValidPath() throws Exception { + mockStatic(Paths.class); + mockStatic(Files.class); + when(Paths.get(validPath)).thenReturn(path); + when(Files.readAllBytes(path)).thenThrow(IOException.class); + + assertEquals("", SecurityUtil.getTruststorePasswordFromFile(validPath)); + + } + + @Test + public void whenGetTruststorePasswordFromFile_InvalidPathExceptionForInvalidPath() { + mockStatic(Paths.class); + mockStatic(Files.class); + when(Paths.get(invalidPath)).thenThrow(InvalidPathException.class); + + assertThrows(InvalidPathException.class, () -> SecurityUtil.getTruststorePasswordFromFile(invalidPath)); + + } + +} diff --git a/datafile-app-server/src/test/resources/datafile_endpoints_test.json b/datafile-app-server/src/test/resources/datafile_endpoints_test.json index 8e51b807..00b2488b 100644 --- a/datafile-app-server/src/test/resources/datafile_endpoints_test.json +++ b/datafile-app-server/src/test/resources/datafile_endpoints_test.json @@ -1,10 +1,10 @@ { "config": { "//description": "This file is only used for testing purposes", - "dmaap.ftpesConfig.keyCert": "/src/test/resources/dfc.jks", - "dmaap.ftpesConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass", - "dmaap.ftpesConfig.trustedCa": "/src/test/resources/ftp.jks", - "dmaap.ftpesConfig.trustedCaPasswordPath": "/src/test/resources/ftp.jks.pass", + "dmaap.certificateConfig.keyCert": "/src/test/resources/dfc.jks", + "dmaap.certificateConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass", + "dmaap.certificateConfig.trustedCa": "/src/test/resources/cert.jks", + "dmaap.certificateConfig.trustedCaPasswordPath": "/src/test/resources/cert.jks.pass", "dmaap.security.trustStorePath": "src/test/resources/trust.jks", "dmaap.security.trustStorePasswordPath": "src/test/resources/trust.pass", "dmaap.security.keyStorePath": "src/test/resources/cert.jks", diff --git a/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json b/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json index e1327462..73bb7448 100644 --- a/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json +++ b/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json @@ -1,10 +1,10 @@ { "config": { "//description": "This file is only used for testing purposes", - "dmaap.ftpesConfig.keyCert": "/src/test/resources/dfc.jks", - "dmaap.ftpesConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass", - "dmaap.ftpesConfig.trustedCa": "/src/test/resources/ftp.jks", - "dmaap.ftpesConfig.trustedCaPasswordPath": "/src/test/resources/ftp.jks.pass", + "dmaap.certificateConfig.keyCert": "/src/test/resources/dfc.jks", + "dmaap.certificateConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass", + "dmaap.certificateConfig.trustedCa": "/src/test/resources/ftp.jks", + "dmaap.certificateConfig.trustedCaPasswordPath": "/src/test/resources/ftp.jks.pass", "dmaap.security.trustStorePath": "src/test/resources/trust.jks", "dmaap.security.trustStorePasswordPath": "src/test/resources/trust.pass", "dmaap.security.keyStorePath": "src/test/resources/cert.jks", diff --git a/datafile-app-server/src/test/resources/keystore.p12 b/datafile-app-server/src/test/resources/keystore.p12 Binary files differnew file mode 100644 index 00000000..b847707c --- /dev/null +++ b/datafile-app-server/src/test/resources/keystore.p12 diff --git a/datafile-app-server/src/test/resources/keystore.pass b/datafile-app-server/src/test/resources/keystore.pass new file mode 100644 index 00000000..1e7befca --- /dev/null +++ b/datafile-app-server/src/test/resources/keystore.pass @@ -0,0 +1 @@ +HVpAf0kHGl4P#fdpblJLka6b
\ No newline at end of file |