diff options
16 files changed, 368 insertions, 91 deletions
diff --git a/datafile-app-server/pom.xml b/datafile-app-server/pom.xml index f5626af7..4c716bd6 100644 --- a/datafile-app-server/pom.xml +++ b/datafile-app-server/pom.xml @@ -24,12 +24,11 @@ <parent> <groupId>org.onap.dcaegen2.collectors</groupId> <artifactId>datafile</artifactId> - <version>1.0.0-SNAPSHOT</version> + <version>1.0.2-SNAPSHOT</version> </parent> <groupId>org.onap.dcaegen2.collectors.datafile</groupId> <artifactId>datafile-app-server</artifactId> - <version>1.0.1-SNAPSHOT</version> <packaging>jar</packaging> <properties> diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/RetryTimer.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/RetryTimer.java new file mode 100644 index 00000000..c2b97da8 --- /dev/null +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/RetryTimer.java @@ -0,0 +1,30 @@ +/* + * ============LICENSE_START====================================================================== + * Copyright (C) 2018 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.tasks; + +public class RetryTimer { + public void waitRetryTime() { + try { + Thread.sleep(60000); + } catch (InterruptedException e) { + // Nothing, no one will interrupt. + } + + } +} diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImpl.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImpl.java index be6ac9cc..306c2ded 100644 --- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImpl.java +++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImpl.java @@ -22,6 +22,8 @@ import java.net.URI; import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; import org.onap.dcaegen2.collectors.datafile.configuration.Config; import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig; +import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectClient; +import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectResult; import org.onap.dcaegen2.collectors.datafile.ftp.FileServerData; import org.onap.dcaegen2.collectors.datafile.ftp.FtpsClient; import org.onap.dcaegen2.collectors.datafile.ftp.ImmutableFileServerData; @@ -51,6 +53,7 @@ public class XnfCollectorTaskImpl implements XnfCollectorTask { private final FtpsClient ftpsClient; private final SftpClient sftpClient; + private RetryTimer retryTimer; @Autowired protected XnfCollectorTaskImpl(AppConfig datafileAppConfig, FtpsClient ftpsCleint, SftpClient sftpClient) { @@ -95,19 +98,20 @@ public class XnfCollectorTaskImpl implements XnfCollectorTask { FileServerData fileServerData = getFileServerData(uri); String remoteFile = uri.getPath(); String localFile = "target" + File.separator + fileData.name(); - String scheme = uri.getScheme(); - boolean fileDownloaded = false; - if (FTPES.equals(scheme) || FTPS.equals(scheme)) { - fileDownloaded = ftpsClient.collectFile(fileServerData, remoteFile, localFile); - } else if (SFTP.equals(scheme)) { - fileDownloaded = sftpClient.collectFile(fileServerData, remoteFile, localFile); - } else { - logger.error("DFC does not support protocol {}. Supported protocols are {}, {}, and {}. Data: {}", scheme, - FTPES, FTPS, SFTP, fileData); - localFile = null; - } - if (!fileDownloaded) { + FileCollectClient currentClient = selectClient(fileData, uri); + + if (currentClient != null) { + FileCollectResult fileCollectResult = currentClient.collectFile(fileServerData, remoteFile, localFile); + if (!fileCollectResult.downloadSuccessful()) { + fileCollectResult = retry(fileCollectResult, currentClient); + } + if (!fileCollectResult.downloadSuccessful()) { + localFile = null; + logger.error("Download of file aborted after maximum number of retries. Data: {} Error causes {}", + fileServerData, fileCollectResult.getErrorData()); + } + } else { localFile = null; } return localFile; @@ -128,6 +132,30 @@ public class XnfCollectorTaskImpl implements XnfCollectorTask { return userInfo; } + private FileCollectClient selectClient(FileData fileData, URI uri) { + FileCollectClient selectedClient = null; + String scheme = uri.getScheme(); + if (FTPES.equals(scheme) || FTPS.equals(scheme)) { + selectedClient = ftpsClient; + } else if (SFTP.equals(scheme)) { + selectedClient = sftpClient; + } else { + logger.error("DFC does not support protocol {}. Supported protocols are {}, {}, and {}. Data: {}", scheme, + FTPES, FTPS, SFTP, fileData); + } + return selectedClient; + } + + private FileCollectResult retry(FileCollectResult fileCollectResult, FileCollectClient fileCollectClient) { + int retryCount = 1; + FileCollectResult newResult = fileCollectResult; + while (!newResult.downloadSuccessful() && retryCount++ < 3) { + getRetryTimer().waitRetryTime(); + newResult = fileCollectClient.retryCollectFile(); + } + return newResult; + } + private ConsumerDmaapModel getConsumerDmaapModel(FileData fileData, String localFile) { String name = fileData.name(); String compression = fileData.compression(); @@ -137,4 +165,15 @@ public class XnfCollectorTaskImpl implements XnfCollectorTask { return ImmutableConsumerDmaapModel.builder().name(name).location(localFile).compression(compression) .fileFormatType(fileFormatType).fileFormatVersion(fileFormatVersion).build(); } + + private RetryTimer getRetryTimer() { + if (retryTimer == null) { + retryTimer = new RetryTimer(); + } + return retryTimer; + } + + protected void setRetryTimer(RetryTimer retryTimer) { + this.retryTimer = retryTimer; + } } diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImplTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImplTest.java index 3a3f16c0..8251a65a 100644 --- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImplTest.java +++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImplTest.java @@ -16,6 +16,7 @@ package org.onap.dcaegen2.collectors.datafile.tasks; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -28,6 +29,8 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig; +import org.onap.dcaegen2.collectors.datafile.ftp.ErrorData; +import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectResult; import org.onap.dcaegen2.collectors.datafile.ftp.FileServerData; import org.onap.dcaegen2.collectors.datafile.ftp.FtpsClient; import org.onap.dcaegen2.collectors.datafile.ftp.ImmutableFileServerData; @@ -74,8 +77,8 @@ public class XnfCollectorTaskImplTest { private FtpsClient ftpsClientMock = mock(FtpsClient.class); private SftpClient sftpClientMock = mock(SftpClient.class); + private RetryTimer retryTimerMock = mock(RetryTimer.class); - private XnfCollectorTask collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock); @BeforeAll public static void setUpConfiguration() { @@ -88,6 +91,8 @@ public class XnfCollectorTaskImplTest { @Test public void whenFtpesFile_returnCorrectResponse() { + XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock); + FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER) .changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(FTPES_LOCATION) .compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) @@ -96,7 +101,7 @@ public class XnfCollectorTaskImplTest { FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId(USER) .password(PWD).port(PORT_22).build(); when(ftpsClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION)) - .thenReturn(Boolean.TRUE); + .thenReturn(new FileCollectResult()); ConsumerDmaapModel expectedConsumerDmaapModel = ImmutableConsumerDmaapModel.builder().name(PM_FILE_NAME) .location(LOCAL_FILE_LOCATION).compression(GZIP_COMPRESSION) @@ -115,6 +120,8 @@ public class XnfCollectorTaskImplTest { @Test public void whenSftpFile_returnCorrectResponse() { + XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock); + FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER) .changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(SFTP_LOCATION) .compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) @@ -123,7 +130,7 @@ public class XnfCollectorTaskImplTest { FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId("") .password("").port(PORT_22).build(); when(sftpClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION)) - .thenReturn(Boolean.TRUE); + .thenReturn(new FileCollectResult()); ConsumerDmaapModel expectedConsumerDmaapModel = ImmutableConsumerDmaapModel.builder().name(PM_FILE_NAME) .location(LOCAL_FILE_LOCATION).compression(GZIP_COMPRESSION) @@ -133,15 +140,67 @@ public class XnfCollectorTaskImplTest { .verifyComplete(); verify(sftpClientMock, times(1)).collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); - verify(ftpsClientMock).setKeyCertPath(FTP_KEY_PATH); - verify(ftpsClientMock).setKeyCertPassword(FTP_KEY_PASSWORD); - verify(ftpsClientMock).setTrustedCAPath(TRUSTED_CA_PATH); - verify(ftpsClientMock).setTrustedCAPassword(TRUSTED_CA_PASSWORD); - verifyNoMoreInteractions(ftpsClientMock); + verifyNoMoreInteractions(sftpClientMock); + } + + @Test + public void whenFtpesFileAlwaysFail_retryAndReturnEmpty() { + XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock); + collectorUndetTest.setRetryTimer(retryTimerMock); + + FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER) + .changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(FTPES_LOCATION) + .compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) + .fileFormatVersion(FILE_FORMAT_VERSION).build(); + + FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId(USER) + .password(PWD).port(PORT_22).build(); + ErrorData errorData = new ErrorData(); + errorData.addError("Unable to collect file.", new Exception()); + when(ftpsClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION)) + .thenReturn(new FileCollectResult(errorData)); + doReturn(new FileCollectResult(errorData)).when(ftpsClientMock).retryCollectFile(); + + StepVerifier.create(collectorUndetTest.execute(fileData)).expectNextCount(0).verifyComplete(); + + verify(ftpsClientMock, times(1)).collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + verify(ftpsClientMock, times(2)).retryCollectFile(); + } + + @Test + public void whenFtpesFileFailOnce_retryAndReturnCorrectResponse() { + XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock); + collectorUndetTest.setRetryTimer(retryTimerMock); + + FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER) + .changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(FTPES_LOCATION) + .compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) + .fileFormatVersion(FILE_FORMAT_VERSION).build(); + + FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId(USER) + .password(PWD).port(PORT_22).build(); + ErrorData errorData = new ErrorData(); + errorData.addError("Unable to collect file.", new Exception()); + when(ftpsClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION)) + .thenReturn(new FileCollectResult(errorData)); + doReturn(new FileCollectResult()).when(ftpsClientMock).retryCollectFile(); + + ConsumerDmaapModel expectedConsumerDmaapModel = ImmutableConsumerDmaapModel.builder().name(PM_FILE_NAME) + .location(LOCAL_FILE_LOCATION).compression(GZIP_COMPRESSION) + .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE).fileFormatVersion(FILE_FORMAT_VERSION).build(); + + StepVerifier.create(collectorUndetTest.execute(fileData)).expectNext(expectedConsumerDmaapModel) + .verifyComplete(); + + + verify(ftpsClientMock, times(1)).collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + verify(ftpsClientMock, times(1)).retryCollectFile(); } @Test public void whenWrongScheme_returnEmpty() { + XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock); + FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER) .changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location("http://host.com/file.zip") .compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) @@ -150,14 +209,10 @@ public class XnfCollectorTaskImplTest { FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId("") .password("").port(PORT_22).build(); when(sftpClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION)) - .thenReturn(Boolean.TRUE); + .thenReturn(new FileCollectResult()); StepVerifier.create(collectorUndetTest.execute(fileData)).expectNextCount(0).verifyComplete(); - verify(ftpsClientMock).setKeyCertPath(FTP_KEY_PATH); - verify(ftpsClientMock).setKeyCertPassword(FTP_KEY_PASSWORD); - verify(ftpsClientMock).setTrustedCAPath(TRUSTED_CA_PATH); - verify(ftpsClientMock).setTrustedCAPassword(TRUSTED_CA_PASSWORD); - verifyNoMoreInteractions(ftpsClientMock); + verifyNoMoreInteractions(sftpClientMock); } } diff --git a/datafile-commons/pom.xml b/datafile-commons/pom.xml index 4b812aa2..9029769e 100644 --- a/datafile-commons/pom.xml +++ b/datafile-commons/pom.xml @@ -24,7 +24,7 @@ <parent> <groupId>org.onap.dcaegen2.collectors</groupId> <artifactId>datafile</artifactId> - <version>1.0.0-SNAPSHOT</version> + <version>1.0.2-SNAPSHOT</version> </parent> <groupId>org.onap.dcaegen2.collectors.datafile</groupId> diff --git a/datafile-dmaap-client/pom.xml b/datafile-dmaap-client/pom.xml index 21839ab1..9eb232ba 100644 --- a/datafile-dmaap-client/pom.xml +++ b/datafile-dmaap-client/pom.xml @@ -24,7 +24,7 @@ <parent> <groupId>org.onap.dcaegen2.collectors</groupId> <artifactId>datafile</artifactId> - <version>1.0.0-SNAPSHOT</version> + <version>1.0.2-SNAPSHOT</version> </parent> <groupId>org.onap.dcaegen2.collectors.datafile</groupId> diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/ErrorData.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/ErrorData.java new file mode 100644 index 00000000..2585bbce --- /dev/null +++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/ErrorData.java @@ -0,0 +1,41 @@ +/* + * ============LICENSE_START====================================================================== + * Copyright (C) 2018 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +import java.util.ArrayList; +import java.util.List; + +public class ErrorData { + private List<String> errorMessages = new ArrayList<>(); + private List<Throwable> errorCauses = new ArrayList<>(); + + public void addError(String errorMessage, Throwable errorCause) { + errorMessages.add(errorMessage); + errorCauses.add(errorCause); + } + + @Override + public String toString() { + StringBuilder message = new StringBuilder(); + for (int i = 0; i < errorMessages.size(); i++) { + message.append(errorMessages.get(i)).append(" Cause: ").append(errorCauses.get(i)).append("\n"); + } + return message.toString(); + } +} diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectClient.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectClient.java new file mode 100644 index 00000000..b50b0457 --- /dev/null +++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectClient.java @@ -0,0 +1,53 @@ +/* + * ============LICENSE_START====================================================================== + * Copyright (C) 2018 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author + * + */ +public abstract class FileCollectClient { + protected static final Logger logger = LoggerFactory.getLogger(FtpsClient.class); + + protected FileServerData fileServerData; + protected String remoteFile; + protected String localFile; + protected ErrorData errorData; + + public FileCollectResult collectFile(FileServerData fileServerData, String remoteFile, String localFile) { + logger.trace("collectFile called with fileServerData: {}, remoteFile: {}, localFile: {}", fileServerData, + remoteFile, localFile); + + this.fileServerData = fileServerData; + this.remoteFile = remoteFile; + this.localFile = localFile; + + return retryCollectFile(); + } + + public abstract FileCollectResult retryCollectFile(); + + protected void addError(String errorMessage, Throwable errorCause) { + if (errorData == null) { + errorData = new ErrorData(); + } + errorData.addError(errorMessage, errorCause); + } +} diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectResult.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectResult.java new file mode 100644 index 00000000..6cd048ac --- /dev/null +++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectResult.java @@ -0,0 +1,46 @@ +/* + * ============LICENSE_START====================================================================== + * Copyright (C) 2018 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +public class FileCollectResult { + private boolean result; + private ErrorData errorData; + + public FileCollectResult() { + this.result = true; + } + + public FileCollectResult(ErrorData errorData) { + this.errorData = errorData; + result = false; + } + + public boolean downloadSuccessful() { + return result; + } + + public String getErrorData() { + return errorData.toString(); + } + + @Override + public String toString() { + return "Download successful: " + result + " Error data: " + getErrorData(); + } +} diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java index 719013ea..a88072c7 100644 --- a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java +++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java @@ -36,8 +36,6 @@ import org.onap.dcaegen2.collectors.datafile.ssl.ITrustManagerFactory; import org.onap.dcaegen2.collectors.datafile.ssl.KeyManagerUtilsWrapper; import org.onap.dcaegen2.collectors.datafile.ssl.KeyStoreWrapper; import org.onap.dcaegen2.collectors.datafile.ssl.TrustManagerFactoryWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; /** @@ -46,9 +44,7 @@ import org.springframework.stereotype.Component; * @author <a href="mailto:martin.c.yan@est.tech">Martin Yan</a> */ @Component -public class FtpsClient { - private static final Logger logger = LoggerFactory.getLogger(FtpsClient.class); - +public class FtpsClient extends FileCollectClient { private String keyCertPath; private String keyCertPassword; private String trustedCAPath; @@ -58,27 +54,32 @@ public class FtpsClient { private IKeyManagerUtils kmu; private IKeyStore keyStore; private ITrustManagerFactory trustManagerFactory; - private IFile localFile; + private IFile lf; private IFileSystemResource fileResource; private IOutputStream os; - public boolean collectFile(FileServerData fileServerData, String remoteFile, String localFile) { - logger.trace("collectFile called with fileServerData: {}, remoteFile: {}, localFile: {}", fileServerData, - remoteFile, localFile); - boolean result = true; + @Override + public FileCollectResult retryCollectFile() { + logger.trace("retryCollectFile called"); + + FileCollectResult fileCollectResult; + IFTPSClient ftps = getFtpsClient(); ftps.setNeedClientAuth(true); - if (setUpKeyManager(ftps) && setUpTrustedCA(ftps) && setUpConnection(fileServerData, ftps)) { - result = getFileFromxNF(remoteFile, localFile, ftps, fileServerData); - - closeDownConnection(ftps); + if (setUpKeyManager(ftps) && setUpTrustedCA(ftps) && setUpConnection(ftps)) { + if (getFileFromxNF(ftps)) { + closeDownConnection(ftps); + fileCollectResult = new FileCollectResult(); + } else { + fileCollectResult = new FileCollectResult(errorData); + } } else { - result = false; + fileCollectResult = new FileCollectResult(errorData); } - logger.trace("collectFile left with result: {}", result); - return result; + logger.trace("retryCollectFile left with result: {}", fileCollectResult); + return fileCollectResult; } private boolean setUpKeyManager(IFTPSClient ftps) { @@ -88,7 +89,7 @@ public class FtpsClient { keyManagerUtils.setCredentials(keyCertPath, keyCertPassword); ftps.setKeyManager(keyManagerUtils.getClientKeyManager()); } catch (GeneralSecurityException | IOException e) { - logger.error("Unable to use own key store {}", keyCertPath, e); + addError("Unable to use own key store " + keyCertPath, e); result = false; } return result; @@ -108,13 +109,13 @@ public class FtpsClient { ftps.setTrustManager(tmf.getTrustManagers()[0]); } catch (Exception e) { - logger.error("Unable to trust xNF's CA, {}", trustedCAPath, e); + addError("Unable to trust xNF's CA, " + trustedCAPath, e); result = false; } return result; } - private boolean setUpConnection(FileServerData fileServerData, IFTPSClient ftps) { + private boolean setUpConnection(IFTPSClient ftps) { boolean result = true; try { ftps.connect(fileServerData.serverAddress(), fileServerData.port()); @@ -122,7 +123,7 @@ public class FtpsClient { boolean loginSuccesful = ftps.login(fileServerData.userId(), fileServerData.password()); if (!loginSuccesful) { ftps.logout(); - logger.error("Unable to log in to xNF. {}", fileServerData); + addError("Unable to log in to xNF. " + fileServerData, null); result = false; } @@ -134,24 +135,23 @@ public class FtpsClient { ftps.execPROT("P"); } else { ftps.disconnect(); - logger.error("Unable to connect to xNF. {}", fileServerData); + addError("Unable to connect to xNF. " + fileServerData + "xNF reply code: " + ftps.getReplyCode(), null); result = false; } } catch (Exception ex) { - logger.error("Unable to connect to xNF. Data: {}", fileServerData, ex); + addError("Unable to connect to xNF. Data: " + fileServerData, ex); result = false; } logger.trace("setUpConnection return value: {}", result); return result; } - private boolean getFileFromxNF(String remoteFile, String localFilePath, IFTPSClient ftps, - FileServerData fileServerData) { + private boolean getFileFromxNF(IFTPSClient ftps) { logger.trace("starting to getFile"); boolean result = true; + IFile outfile = getFile(); try { - IFile outfile = getFile(); - outfile.setPath(localFilePath); + outfile.setPath(localFile); outfile.createNewFile(); IOutputStream outputStream = getOutputStream(); @@ -160,9 +160,14 @@ public class FtpsClient { ftps.retrieveFile(remoteFile, output); output.close(); - logger.debug("File {} Download Successfull from xNF", localFilePath); + logger.debug("File {} Download Successfull from xNF", localFile); } catch (IOException ex) { - logger.error("Unable to collect file from xNF. Data: {}", fileServerData, ex); + addError("Unable to collect file from xNF. Data: " + fileServerData, ex); + try { + outfile.delete(); + } catch (Exception e) { + // Nothing + } result = false; } return result; @@ -227,11 +232,11 @@ public class FtpsClient { } private IFile getFile() { - if (localFile == null) { - localFile = new FileWrapper(); + if (lf == null) { + lf = new FileWrapper(); } - return localFile; + return lf; } private IOutputStream getOutputStream() { @@ -266,7 +271,7 @@ public class FtpsClient { } protected void setFile(IFile file) { - localFile = file; + lf = file; } protected void setOutputStream(IOutputStream outputStream) { diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java index 5bd95b16..18b28346 100644 --- a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java +++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java @@ -24,8 +24,6 @@ import com.jcraft.jsch.Session; import com.jcraft.jsch.SftpException; import org.apache.commons.io.FilenameUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; /** @@ -35,11 +33,12 @@ import org.springframework.stereotype.Component; * */ @Component -public class SftpClient { - private static final Logger logger = LoggerFactory.getLogger(SftpClient.class); +public class SftpClient extends FileCollectClient { + @Override + public FileCollectResult retryCollectFile() { + logger.trace("retryCollectFile called"); - public boolean collectFile(FileServerData fileServerData, String remoteFile, String localFile) { - boolean result = true; + FileCollectResult result; Session session = setUpSession(fileServerData); if (session != null) { @@ -47,20 +46,22 @@ public class SftpClient { if (sftpChannel != null) { try { sftpChannel.get(remoteFile, localFile); + result = new FileCollectResult(); logger.debug("File {} Download Successfull from xNF", FilenameUtils.getName(localFile)); } catch (SftpException e) { - logger.error("Unable to get file from xNF. Data: {}", fileServerData, e); - result = false; + addError("Unable to get file from xNF. Data: " + fileServerData, e); + result = new FileCollectResult(errorData); } sftpChannel.exit(); } else { - result = false; + result = new FileCollectResult(errorData); } session.disconnect(); } else { - result = false; + result = new FileCollectResult(errorData); } + logger.trace("retryCollectFile left with result: {}", result); return result; } @@ -74,7 +75,7 @@ public class SftpClient { session.setPassword(fileServerData.password()); session.connect(); } catch (JSchException e) { - logger.error("Unable to set up SFTP connection to xNF. Data: {}", fileServerData, e); + addError("Unable to set up SFTP connection to xNF. Data: " + fileServerData, e); } return session; } @@ -87,7 +88,7 @@ public class SftpClient { channel.connect(); sftpChannel = (ChannelSftp) channel; } catch (JSchException e) { - logger.error("Unable to get sftp channel to xNF. Data: {}", fileServerData, e); + addError("Unable to get sftp channel to xNF. Data: " + fileServerData, e); } return sftpChannel; } diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/FileWrapper.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/FileWrapper.java index f8c02f09..32b6c72f 100644 --- a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/FileWrapper.java +++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/FileWrapper.java @@ -41,4 +41,9 @@ public class FileWrapper implements IFile { public File getFile() { return file; } + + @Override + public boolean delete() { + return file.delete(); + } } diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/IFile.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/IFile.java index 47d868a0..a7094f69 100644 --- a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/IFile.java +++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/IFile.java @@ -27,4 +27,6 @@ public interface IFile { public boolean createNewFile() throws IOException; public File getFile(); + + public boolean delete(); } diff --git a/datafile-dmaap-client/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClientTest.java b/datafile-dmaap-client/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClientTest.java index 5d716a9b..e9e00c38 100644 --- a/datafile-dmaap-client/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClientTest.java +++ b/datafile-dmaap-client/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClientTest.java @@ -106,9 +106,9 @@ public class FtpsClientTest { ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS) .userId(USERNAME).password(PASSWORD).port(PORT).build(); - boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); + FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); - assertTrue(result); + assertTrue(result.downloadSuccessful()); verify(ftpsClientMock).setNeedClientAuth(true); verify(keyManagerUtilsMock).setCredentials(FTP_KEY_PATH, FTP_KEY_PASSWORD); verify(ftpsClientMock).setKeyManager(keyManagerMock); @@ -140,9 +140,9 @@ public class FtpsClientTest { ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS) .userId(USERNAME).password(PASSWORD).port(PORT).build(); - boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); + FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); - assertFalse(result); + assertFalse(result.downloadSuccessful()); } @Test @@ -156,9 +156,9 @@ public class FtpsClientTest { ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS) .userId(USERNAME).password(PASSWORD).port(PORT).build(); - boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); + FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); - assertFalse(result); + assertFalse(result.downloadSuccessful()); } @Test @@ -172,10 +172,10 @@ public class FtpsClientTest { ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS) .userId(USERNAME).password(PASSWORD).port(PORT).build(); - boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); + FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); verify(ftpsClientMock, times(1)).logout(); - assertFalse(result); + assertFalse(result.downloadSuccessful()); } @Test @@ -190,10 +190,10 @@ public class FtpsClientTest { ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS) .userId(USERNAME).password(PASSWORD).port(PORT).build(); - boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); + FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); verify(ftpsClientMock, times(1)).disconnect(); - assertFalse(result); + assertFalse(result.downloadSuccessful()); } @Test @@ -208,9 +208,9 @@ public class FtpsClientTest { ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS) .userId(USERNAME).password(PASSWORD).port(PORT).build(); - boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); + FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); - assertFalse(result); + assertFalse(result.downloadSuccessful()); } @Test @@ -227,8 +227,9 @@ public class FtpsClientTest { ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS) .userId(USERNAME).password(PASSWORD).port(PORT).build(); - boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); + FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH); - assertFalse(result); + assertFalse(result.downloadSuccessful()); + verify(localFileMock, times(1)).delete(); } }
\ No newline at end of file @@ -30,7 +30,7 @@ <groupId>org.onap.dcaegen2.collectors</groupId> <artifactId>datafile</artifactId> - <version>1.0.0-SNAPSHOT</version> + <version>1.0.2-SNAPSHOT</version> <name>dcaegen2-collectors.datafile</name> <description>datafile collector</description> diff --git a/version.properties b/version.properties index 1b043102..22632725 100644 --- a/version.properties +++ b/version.properties @@ -1,6 +1,6 @@ major=1
minor=0
-patch=0
+patch=2
base_version=${major}.${minor}.${patch}
release_version=${base_version}
snapshot_version=${base_version}-SNAPSHOT
|