diff options
Diffstat (limited to 'datafile-app-server/src/main')
9 files changed, 218 insertions, 43 deletions
diff --git a/datafile-app-server/src/main/docker/Dockerfile b/datafile-app-server/src/main/docker/Dockerfile index 0763e2bf..313c8d07 100755 --- a/datafile-app-server/src/main/docker/Dockerfile +++ b/datafile-app-server/src/main/docker/Dockerfile @@ -1,6 +1,7 @@ # # ============LICENSE_START======================================================= # Copyright (C) 2019 Nordix Foundation. +# Copyright (C) 2020 Nokia. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,19 +20,26 @@ # FROM openjdk:11.0.6-jre-slim +EXPOSE 8100 8433 + +RUN apt-get update && apt-get install -y \ + vim \ + curl + WORKDIR /opt/app/datafile RUN mkdir -p /var/log/ONAP RUN mkdir -p /opt/app/datafile/etc/cert/ -ADD /target/datafile-app-server.jar /opt/app/datafile/ -ADD /config/application.yaml /opt/app/datafile/config/ - -EXPOSE 8100 8433 - RUN groupadd -r onap && useradd -ms /bin/bash datafile -g onap RUN chown -R datafile:onap /var/log/ONAP + +ADD /config/application.yaml /opt/app/datafile/config/ RUN chmod -R 777 /opt/app/datafile/config/ USER datafile +RUN mkdir -p /home/datafile/.ssh + +ADD /target/datafile-app-server.jar /opt/app/datafile/ + ENTRYPOINT ["/usr/local/openjdk-11/bin/java", "-jar", "/opt/app/datafile/datafile-app-server.jar"] diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java index 21c51566..c257ceed 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java @@ -68,11 +68,13 @@ import reactor.core.publisher.Mono; @EnableConfigurationProperties @ConfigurationProperties("app") public class AppConfig { + private static final Logger logger = LoggerFactory.getLogger(AppConfig.class); private ConsumerConfiguration dmaapConsumerConfiguration; private Map<String, PublisherConfiguration> publishingConfigurations; private FtpesConfig ftpesConfiguration; + private SftpConfig sftpConfiguration; @Value("#{systemEnvironment}") Properties systemEnvironment; private Disposable refreshConfigTask = null; @@ -90,6 +92,7 @@ public class AppConfig { public void initialize() { stop(); Map<String, String> context = MappedDiagnosticContext.initializeTraceContext(); + loadConfigurationFromFile(); refreshConfigTask = createRefreshTask(context) // @@ -131,7 +134,6 @@ public class AppConfig { * Checks if there is a configuration for the given feed. * * @param changeIdentifier the change identifier the feed is configured to belong to. - * * @return true if a feed is configured for the given change identifier, false if not. */ public synchronized boolean isFeedConfigured(String changeIdentifier) { @@ -143,7 +145,6 @@ public class AppConfig { * * @param changeIdentifier the change identifier the feed is configured to belong to. * @return the <code>PublisherConfiguration</code> for the feed belonging to the given change identifier. - * * @throws DatafileTaskException if no configuration has been loaded or the configuration is missing for the given * change identifier. */ @@ -165,6 +166,10 @@ public class AppConfig { return ftpesConfiguration; } + public synchronized SftpConfig getSftpConfiguration() { + return sftpConfiguration; + } + private <R> Mono<R> onErrorResume(Throwable trowable) { logger.error("Could not refresh application configuration {}", trowable.toString()); return Mono.empty(); @@ -178,27 +183,25 @@ public class AppConfig { return CbsClientFactory.createCbsClient(env); } - /** - * Parse configuration. - * - * @param jsonObject the DFC service's configuration - * @return this which is updated if successful - */ - private AppConfig parseCloudConfig(JsonObject jsonObject) { + private AppConfig parseCloudConfig(JsonObject configurationObject) { try { - CloudConfigParser parser = new CloudConfigParser(jsonObject); + CloudConfigParser parser = new CloudConfigParser(configurationObject, systemEnvironment); setConfiguration(parser.getDmaapConsumerConfig(), parser.getDmaapPublisherConfigurations(), - parser.getFtpesConfig()); - + parser.getFtpesConfig(), parser.getSftpConfig()); + logConfig(); } catch (DatafileTaskException e) { logger.error("Could not parse configuration {}", e.toString(), e); } return this; } - /** - * Reads the configuration from file. - */ + private void logConfig() { + logger.debug("Read and parsed sFTP configuration: [{}]", sftpConfiguration); + logger.debug("Read and parsed FTPes configuration: [{}]", ftpesConfiguration); + logger.debug("Read and parsed DMaaP configuration: [{}]", dmaapConsumerConfiguration); + logger.debug("Read and parsed Publish configuration: [{}]", publishingConfigurations); + } + void loadConfigurationFromFile() { GsonBuilder gsonBuilder = new GsonBuilder(); ServiceLoader.load(TypeAdapterFactory.class).forEach(gsonBuilder::registerTypeAdapterFactory); @@ -217,10 +220,12 @@ public class AppConfig { } private synchronized void setConfiguration(@NotNull ConsumerConfiguration consumerConfiguration, - @NotNull Map<String, PublisherConfiguration> publisherConfiguration, @NotNull FtpesConfig ftpesConfig) { + @NotNull Map<String, PublisherConfiguration> publisherConfiguration, @NotNull FtpesConfig ftpesConfig, + @NotNull SftpConfig sftpConfig) { this.dmaapConsumerConfiguration = consumerConfiguration; this.publishingConfigurations = publisherConfiguration; this.ftpesConfiguration = ftpesConfig; + this.sftpConfiguration = sftpConfig; } JsonElement getJsonElement(JsonParser parser, InputStream inputStream) { diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java index 23197025..a86a32b8 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; +import java.util.Properties; import java.util.Set; import javax.validation.constraints.NotNull; @@ -38,6 +39,7 @@ import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; * @author <a href="mailto:henrik.b.andersson@est.tech">Henrik Andersson</a> */ public class CloudConfigParser { + private static final String DMAAP_SECURITY_TRUST_STORE_PATH = "dmaap.security.trustStorePath"; private static final String DMAAP_SECURITY_TRUST_STORE_PASS_PATH = "dmaap.security.trustStorePasswordPath"; private static final String DMAAP_SECURITY_KEY_STORE_PATH = "dmaap.security.keyStorePath"; @@ -45,19 +47,23 @@ public class CloudConfigParser { private static final String DMAAP_SECURITY_ENABLE_DMAAP_CERT_AUTH = "dmaap.security.enableDmaapCertAuth"; private static final String CONFIG = "config"; + private static final String KNOWN_HOSTS_FILE_PATH_ENV_PROPERTY = "KNOWN_HOSTS_FILE_PATH"; + private static final String CBS_PROPERTY_SFTP_SECURITY_STRICT_HOST_KEY_CHECKING = + "sftp.security.strictHostKeyChecking"; + + private final Properties systemEnvironment; + private final JsonObject jsonObject; - public CloudConfigParser(JsonObject jsonObject) { + public CloudConfigParser(JsonObject jsonObject, Properties systemEnvironment) { this.jsonObject = jsonObject.getAsJsonObject(CONFIG); - + this.systemEnvironment = systemEnvironment; } /** * Get the publisher configurations. * - * @return a map with change identifier as key and the connected publisher configuration as - * value. - * + * @return a map with change identifier as key and the connected publisher configuration as value. * @throws DatafileTaskException if a member of the configuration is missing. */ public @NotNull Map<String, PublisherConfiguration> getDmaapPublisherConfigurations() throws DatafileTaskException { @@ -114,6 +120,19 @@ public class CloudConfigParser { } /** + * Get the sFTP configuration. + * + * @return the sFTP configuration. + * @throws DatafileTaskException if a member of the configuration is missing. + */ + public @NotNull SftpConfig getSftpConfig() throws DatafileTaskException { + String filePath = determineKnownHostsFilePath(); + return new ImmutableSftpConfig.Builder() // + .strictHostKeyChecking(getAsBoolean(jsonObject, CBS_PROPERTY_SFTP_SECURITY_STRICT_HOST_KEY_CHECKING)) + .knownHostsFilePath(filePath).build(); + } + + /** * Get the security configuration for communication with the xNF. * * @return the xNF communication security configuration. @@ -128,6 +147,15 @@ public class CloudConfigParser { .build(); } + private String determineKnownHostsFilePath() { + String filePath = ""; + if (systemEnvironment != null) { + filePath = + systemEnvironment.getProperty(KNOWN_HOSTS_FILE_PATH_ENV_PROPERTY, "/home/datafile/.ssh/known_hosts"); + } + return filePath; + } + private static @NotNull JsonElement get(JsonObject obj, String memberName) throws DatafileTaskException { JsonElement elem = obj.get(memberName); if (elem == null) { @@ -140,6 +168,10 @@ public class CloudConfigParser { return get(obj, memberName).getAsString(); } + private static @NotNull Boolean getAsBoolean(JsonObject obj, String memberName) throws DatafileTaskException { + return get(obj, memberName).getAsBoolean(); + } + private static @NotNull JsonObject getAsJson(JsonObject obj, String memberName) throws DatafileTaskException { return get(obj, memberName).getAsJsonObject(); } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SftpConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SftpConfig.java new file mode 100644 index 00000000..6acc2d56 --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SftpConfig.java @@ -0,0 +1,42 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2020 NOKIA Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.configuration; + +import java.io.Serializable; + +import org.immutables.gson.Gson; +import org.immutables.value.Value; +import org.springframework.stereotype.Component; + +@Component +@Value.Style(builder = "new") +@Value.Immutable +@Gson.TypeAdapters +public abstract class SftpConfig implements Serializable { + + private static final long serialVersionUID = 1L; + + @Value.Parameter + public abstract Boolean strictHostKeyChecking(); + + @Value.Parameter + public abstract String knownHostsFilePath(); +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java index f7121efc..fea578ba 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java @@ -31,10 +31,12 @@ import java.security.NoSuchAlgorithmException; import java.security.UnrecoverableKeyException; import java.security.cert.CertificateException; import java.util.Optional; + import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; + import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPReply; import org.apache.commons.net.ftp.FTPSClient; @@ -222,8 +224,8 @@ public class FtpsClient implements FileCollectClient { } } - private KeyManager createKeyManager(Path keyCertPath, String keyCertPassword) - throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException, UnrecoverableKeyException { + private KeyManager createKeyManager(Path keyCertPath, String keyCertPassword) throws IOException, KeyStoreException, + NoSuchAlgorithmException, CertificateException, UnrecoverableKeyException { logger.trace("Creating key manager from file: {}", keyCertPath); try (InputStream fis = createInputStream(keyCertPath)) { KeyStore keyStore = KeyStore.getInstance("JKS"); diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java index da8361ff..d1685203 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START====================================================================== - * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2018-2019 Nordix Foundation, 2020 Nokia. All rights reserved. * =============================================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -26,6 +26,7 @@ import com.jcraft.jsch.SftpException; import java.nio.file.Path; import java.util.Optional; +import org.jetbrains.annotations.NotNull; import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; import org.slf4j.Logger; @@ -35,19 +36,22 @@ import org.slf4j.LoggerFactory; * Gets file from xNF with SFTP protocol. * * @author <a href="mailto:martin.c.yan@est.tech">Martin Yan</a> - * */ public class SftpClient implements FileCollectClient { + private static final Logger logger = LoggerFactory.getLogger(SftpClient.class); private static final int SFTP_DEFAULT_PORT = 22; + private static final String STRICT_HOST_KEY_CHECKING = "StrictHostKeyChecking"; private final FileServerData fileServerData; protected Session session = null; protected ChannelSftp sftpChannel = null; + private final SftpClientSettings settings; - public SftpClient(FileServerData fileServerData) { + public SftpClient(FileServerData fileServerData, SftpClientSettings sftpConfig) { this.fileServerData = fileServerData; + this.settings = sftpConfig; } @Override @@ -56,7 +60,7 @@ public class SftpClient implements FileCollectClient { try { sftpChannel.get(remoteFile, localFile.toString()); - logger.trace("File {} Download Successfull from xNF", localFile.getFileName()); + logger.trace("File {} Download successful from xNF", localFile.getFileName()); } catch (SftpException e) { boolean retry = e.id != ChannelSftp.SSH_FX_NO_SUCH_FILE && e.id != ChannelSftp.SSH_FX_PERMISSION_DENIED && e.id != ChannelSftp.SSH_FX_OP_UNSUPPORTED; @@ -101,29 +105,47 @@ public class SftpClient implements FileCollectClient { } } } + JSch createJsch() { + return new JSch(); + } private int getPort(Optional<Integer> port) { return port.isPresent() ? port.get() : SFTP_DEFAULT_PORT; } private Session setUpSession(FileServerData fileServerData) throws JSchException { + boolean useStrictHostChecking = this.settings.shouldUseStrictHostChecking(); + JSch jsch = createJschClient(useStrictHostChecking); + return createJshSession(jsch, fileServerData, useStrictHostChecking); + } + + private JSch createJschClient(boolean useStrictHostChecking) throws JSchException { JSch jsch = createJsch(); + if (useStrictHostChecking) { + jsch.setKnownHosts(this.settings.getKnownHostsFilePath()); + } + return jsch; + } + private Session createJshSession(JSch jsch, FileServerData fileServerData, boolean useStrictHostKeyChecking) + throws JSchException { Session newSession = jsch.getSession(fileServerData.userId(), fileServerData.serverAddress(), getPort(fileServerData.port())); - newSession.setConfig("StrictHostKeyChecking", "no"); + newSession.setConfig(STRICT_HOST_KEY_CHECKING, toYesNo(useStrictHostKeyChecking)); newSession.setPassword(fileServerData.password()); newSession.connect(); return newSession; } + @NotNull + private String toYesNo(boolean useStrictHostKeyChecking) { + return useStrictHostKeyChecking ? "yes" : "no"; + } + private ChannelSftp getChannel(Session session) throws JSchException { Channel channel = session.openChannel("sftp"); channel.connect(); return (ChannelSftp) channel; } - protected JSch createJsch() { - return new JSch(); - } } diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettings.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettings.java new file mode 100644 index 00000000..8cab4327 --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettings.java @@ -0,0 +1,63 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2020 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +import java.io.File; +import org.onap.dcaegen2.collectors.datafile.configuration.SftpConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SftpClientSettings { + + private static final Logger logger = LoggerFactory.getLogger(SftpClientSettings.class); + + private final SftpConfig sftpConfig; + + public SftpClientSettings(SftpConfig sftpConfig) { + this.sftpConfig = sftpConfig; + } + + public boolean shouldUseStrictHostChecking() { + boolean strictHostKeyChecking = false; + if (isStrictHostKeyCheckingEnabled()) { + File file = new File(getKnownHostsFilePath()); + strictHostKeyChecking = file.isFile(); + logUsageOfStrictHostCheckingFlag(strictHostKeyChecking, file.getAbsolutePath()); + } else { + logger.info("StrictHostKeyChecking will be disabled."); + } + return strictHostKeyChecking; + } + + public String getKnownHostsFilePath() { + return this.sftpConfig.knownHostsFilePath(); + } + + private boolean isStrictHostKeyCheckingEnabled() { + return Boolean.TRUE.equals(this.sftpConfig.strictHostKeyChecking()); + } + + private void logUsageOfStrictHostCheckingFlag(boolean strictHostKeyChecking, String filePath) { + if (strictHostKeyChecking) { + logger.info("StrictHostKeyChecking will be enabled with KNOW_HOSTS_FILE_PATH [{}].", filePath); + } else { + logger.warn( + "StrictHostKeyChecking is enabled but environment variable KNOW_HOSTS_FILE_PATH is not set or points to not existing file [{}] --> falling back to StrictHostKeyChecking='no'.", + filePath); + } + } +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java index 4805cb47..8721f61e 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java @@ -130,12 +130,12 @@ public abstract class FileData { String[] userAndPassword = userInfoString.split(":"); if (userAndPassword.length == 2) { return Optional.of(userAndPassword); - }else if(userAndPassword.length == 1)//if just user + } else if (userAndPassword.length == 1)// if just user { - String[] tab = new String[2]; - tab[0] = userAndPassword[0]; - tab[1] = "";//add empty password - return Optional.of(tab); + String[] tab = new String[2]; + tab[0] = userAndPassword[0]; + tab[1] = "";// add empty password + return Optional.of(tab); } } return Optional.empty(); diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java index 3e292975..e9c4aceb 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java @@ -29,6 +29,7 @@ import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTask import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectClient; import org.onap.dcaegen2.collectors.datafile.ftp.FtpsClient; import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient; +import org.onap.dcaegen2.collectors.datafile.ftp.SftpClientSettings; import org.onap.dcaegen2.collectors.datafile.model.Counters; import org.onap.dcaegen2.collectors.datafile.model.FileData; import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation; @@ -67,7 +68,6 @@ public class FileCollector { * @param numRetries the number of retries if the publishing fails * @param firstBackoff the time to delay the first retry * @param contextMap context for logging. - * * @return the data needed to publish the file. */ public Mono<FilePublishInformation> collectFile(FileData fileData, long numRetries, Duration firstBackoff, @@ -154,7 +154,8 @@ public class FileCollector { } protected SftpClient createSftpClient(FileData fileData) { - return new SftpClient(fileData.fileServerData()); + return new SftpClient(fileData.fileServerData(), + new SftpClientSettings(datafileAppConfig.getSftpConfiguration())); } protected FtpsClient createFtpsClient(FileData fileData) { |