summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorelinuxhenrik <henrik.b.andersson@est.tech>2018-10-09 16:28:29 +0200
committerelinuxhenrik <henrik.b.andersson@est.tech>2018-10-09 16:34:22 +0200
commitb47ed92edbf3b4f50b814a4b1912b5ef6185c294 (patch)
tree36acfe8dcb86fab3c6365b19198358357fbb6973
parent0ae7a7f6b40ff13a21f33e3e573d6b3ebde6bf90 (diff)
Fix retry when file download fails1.0.2
When the Datafile collector is unable to download the file from an xNF it now retries to collect the file. Change-Id: I61f68f9cf7af1a7fab160b0e936daafd1a23aaf8 Issue-ID: DCAEGEN2-864 Signed-off-by: elinuxhenrik <henrik.b.andersson@est.tech>
-rw-r--r--datafile-app-server/pom.xml3
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/RetryTimer.java30
-rw-r--r--datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImpl.java63
-rw-r--r--datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImplTest.java83
-rw-r--r--datafile-commons/pom.xml2
-rw-r--r--datafile-dmaap-client/pom.xml2
-rw-r--r--datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/ErrorData.java41
-rw-r--r--datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectClient.java53
-rw-r--r--datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectResult.java46
-rw-r--r--datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java71
-rw-r--r--datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java25
-rw-r--r--datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/FileWrapper.java5
-rw-r--r--datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/IFile.java2
-rw-r--r--datafile-dmaap-client/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClientTest.java29
-rw-r--r--pom.xml2
-rw-r--r--version.properties2
16 files changed, 368 insertions, 91 deletions
diff --git a/datafile-app-server/pom.xml b/datafile-app-server/pom.xml
index f5626af7..4c716bd6 100644
--- a/datafile-app-server/pom.xml
+++ b/datafile-app-server/pom.xml
@@ -24,12 +24,11 @@
<parent>
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.0.0-SNAPSHOT</version>
+ <version>1.0.2-SNAPSHOT</version>
</parent>
<groupId>org.onap.dcaegen2.collectors.datafile</groupId>
<artifactId>datafile-app-server</artifactId>
- <version>1.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/RetryTimer.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/RetryTimer.java
new file mode 100644
index 00000000..c2b97da8
--- /dev/null
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/RetryTimer.java
@@ -0,0 +1,30 @@
+/*
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2018 Nordix Foundation. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END========================================================================
+ */
+
+package org.onap.dcaegen2.collectors.datafile.tasks;
+
+public class RetryTimer {
+ public void waitRetryTime() {
+ try {
+ Thread.sleep(60000);
+ } catch (InterruptedException e) {
+ // Nothing, no one will interrupt.
+ }
+
+ }
+}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImpl.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImpl.java
index be6ac9cc..306c2ded 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImpl.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImpl.java
@@ -22,6 +22,8 @@ import java.net.URI;
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
import org.onap.dcaegen2.collectors.datafile.configuration.Config;
import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
+import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectClient;
+import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectResult;
import org.onap.dcaegen2.collectors.datafile.ftp.FileServerData;
import org.onap.dcaegen2.collectors.datafile.ftp.FtpsClient;
import org.onap.dcaegen2.collectors.datafile.ftp.ImmutableFileServerData;
@@ -51,6 +53,7 @@ public class XnfCollectorTaskImpl implements XnfCollectorTask {
private final FtpsClient ftpsClient;
private final SftpClient sftpClient;
+ private RetryTimer retryTimer;
@Autowired
protected XnfCollectorTaskImpl(AppConfig datafileAppConfig, FtpsClient ftpsCleint, SftpClient sftpClient) {
@@ -95,19 +98,20 @@ public class XnfCollectorTaskImpl implements XnfCollectorTask {
FileServerData fileServerData = getFileServerData(uri);
String remoteFile = uri.getPath();
String localFile = "target" + File.separator + fileData.name();
- String scheme = uri.getScheme();
- boolean fileDownloaded = false;
- if (FTPES.equals(scheme) || FTPS.equals(scheme)) {
- fileDownloaded = ftpsClient.collectFile(fileServerData, remoteFile, localFile);
- } else if (SFTP.equals(scheme)) {
- fileDownloaded = sftpClient.collectFile(fileServerData, remoteFile, localFile);
- } else {
- logger.error("DFC does not support protocol {}. Supported protocols are {}, {}, and {}. Data: {}", scheme,
- FTPES, FTPS, SFTP, fileData);
- localFile = null;
- }
- if (!fileDownloaded) {
+ FileCollectClient currentClient = selectClient(fileData, uri);
+
+ if (currentClient != null) {
+ FileCollectResult fileCollectResult = currentClient.collectFile(fileServerData, remoteFile, localFile);
+ if (!fileCollectResult.downloadSuccessful()) {
+ fileCollectResult = retry(fileCollectResult, currentClient);
+ }
+ if (!fileCollectResult.downloadSuccessful()) {
+ localFile = null;
+ logger.error("Download of file aborted after maximum number of retries. Data: {} Error causes {}",
+ fileServerData, fileCollectResult.getErrorData());
+ }
+ } else {
localFile = null;
}
return localFile;
@@ -128,6 +132,30 @@ public class XnfCollectorTaskImpl implements XnfCollectorTask {
return userInfo;
}
+ private FileCollectClient selectClient(FileData fileData, URI uri) {
+ FileCollectClient selectedClient = null;
+ String scheme = uri.getScheme();
+ if (FTPES.equals(scheme) || FTPS.equals(scheme)) {
+ selectedClient = ftpsClient;
+ } else if (SFTP.equals(scheme)) {
+ selectedClient = sftpClient;
+ } else {
+ logger.error("DFC does not support protocol {}. Supported protocols are {}, {}, and {}. Data: {}", scheme,
+ FTPES, FTPS, SFTP, fileData);
+ }
+ return selectedClient;
+ }
+
+ private FileCollectResult retry(FileCollectResult fileCollectResult, FileCollectClient fileCollectClient) {
+ int retryCount = 1;
+ FileCollectResult newResult = fileCollectResult;
+ while (!newResult.downloadSuccessful() && retryCount++ < 3) {
+ getRetryTimer().waitRetryTime();
+ newResult = fileCollectClient.retryCollectFile();
+ }
+ return newResult;
+ }
+
private ConsumerDmaapModel getConsumerDmaapModel(FileData fileData, String localFile) {
String name = fileData.name();
String compression = fileData.compression();
@@ -137,4 +165,15 @@ public class XnfCollectorTaskImpl implements XnfCollectorTask {
return ImmutableConsumerDmaapModel.builder().name(name).location(localFile).compression(compression)
.fileFormatType(fileFormatType).fileFormatVersion(fileFormatVersion).build();
}
+
+ private RetryTimer getRetryTimer() {
+ if (retryTimer == null) {
+ retryTimer = new RetryTimer();
+ }
+ return retryTimer;
+ }
+
+ protected void setRetryTimer(RetryTimer retryTimer) {
+ this.retryTimer = retryTimer;
+ }
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImplTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImplTest.java
index 3a3f16c0..8251a65a 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImplTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/XnfCollectorTaskImplTest.java
@@ -16,6 +16,7 @@
package org.onap.dcaegen2.collectors.datafile.tasks;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -28,6 +29,8 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
+import org.onap.dcaegen2.collectors.datafile.ftp.ErrorData;
+import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectResult;
import org.onap.dcaegen2.collectors.datafile.ftp.FileServerData;
import org.onap.dcaegen2.collectors.datafile.ftp.FtpsClient;
import org.onap.dcaegen2.collectors.datafile.ftp.ImmutableFileServerData;
@@ -74,8 +77,8 @@ public class XnfCollectorTaskImplTest {
private FtpsClient ftpsClientMock = mock(FtpsClient.class);
private SftpClient sftpClientMock = mock(SftpClient.class);
+ private RetryTimer retryTimerMock = mock(RetryTimer.class);
- private XnfCollectorTask collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
@BeforeAll
public static void setUpConfiguration() {
@@ -88,6 +91,8 @@ public class XnfCollectorTaskImplTest {
@Test
public void whenFtpesFile_returnCorrectResponse() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+
FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
.changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(FTPES_LOCATION)
.compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
@@ -96,7 +101,7 @@ public class XnfCollectorTaskImplTest {
FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId(USER)
.password(PWD).port(PORT_22).build();
when(ftpsClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
- .thenReturn(Boolean.TRUE);
+ .thenReturn(new FileCollectResult());
ConsumerDmaapModel expectedConsumerDmaapModel = ImmutableConsumerDmaapModel.builder().name(PM_FILE_NAME)
.location(LOCAL_FILE_LOCATION).compression(GZIP_COMPRESSION)
@@ -115,6 +120,8 @@ public class XnfCollectorTaskImplTest {
@Test
public void whenSftpFile_returnCorrectResponse() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+
FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
.changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(SFTP_LOCATION)
.compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
@@ -123,7 +130,7 @@ public class XnfCollectorTaskImplTest {
FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId("")
.password("").port(PORT_22).build();
when(sftpClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
- .thenReturn(Boolean.TRUE);
+ .thenReturn(new FileCollectResult());
ConsumerDmaapModel expectedConsumerDmaapModel = ImmutableConsumerDmaapModel.builder().name(PM_FILE_NAME)
.location(LOCAL_FILE_LOCATION).compression(GZIP_COMPRESSION)
@@ -133,15 +140,67 @@ public class XnfCollectorTaskImplTest {
.verifyComplete();
verify(sftpClientMock, times(1)).collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
- verify(ftpsClientMock).setKeyCertPath(FTP_KEY_PATH);
- verify(ftpsClientMock).setKeyCertPassword(FTP_KEY_PASSWORD);
- verify(ftpsClientMock).setTrustedCAPath(TRUSTED_CA_PATH);
- verify(ftpsClientMock).setTrustedCAPassword(TRUSTED_CA_PASSWORD);
- verifyNoMoreInteractions(ftpsClientMock);
+ verifyNoMoreInteractions(sftpClientMock);
+ }
+
+ @Test
+ public void whenFtpesFileAlwaysFail_retryAndReturnEmpty() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+ collectorUndetTest.setRetryTimer(retryTimerMock);
+
+ FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
+ .changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(FTPES_LOCATION)
+ .compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
+ .fileFormatVersion(FILE_FORMAT_VERSION).build();
+
+ FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId(USER)
+ .password(PWD).port(PORT_22).build();
+ ErrorData errorData = new ErrorData();
+ errorData.addError("Unable to collect file.", new Exception());
+ when(ftpsClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
+ .thenReturn(new FileCollectResult(errorData));
+ doReturn(new FileCollectResult(errorData)).when(ftpsClientMock).retryCollectFile();
+
+ StepVerifier.create(collectorUndetTest.execute(fileData)).expectNextCount(0).verifyComplete();
+
+ verify(ftpsClientMock, times(1)).collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
+ verify(ftpsClientMock, times(2)).retryCollectFile();
+ }
+
+ @Test
+ public void whenFtpesFileFailOnce_retryAndReturnCorrectResponse() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+ collectorUndetTest.setRetryTimer(retryTimerMock);
+
+ FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
+ .changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(FTPES_LOCATION)
+ .compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
+ .fileFormatVersion(FILE_FORMAT_VERSION).build();
+
+ FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId(USER)
+ .password(PWD).port(PORT_22).build();
+ ErrorData errorData = new ErrorData();
+ errorData.addError("Unable to collect file.", new Exception());
+ when(ftpsClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
+ .thenReturn(new FileCollectResult(errorData));
+ doReturn(new FileCollectResult()).when(ftpsClientMock).retryCollectFile();
+
+ ConsumerDmaapModel expectedConsumerDmaapModel = ImmutableConsumerDmaapModel.builder().name(PM_FILE_NAME)
+ .location(LOCAL_FILE_LOCATION).compression(GZIP_COMPRESSION)
+ .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE).fileFormatVersion(FILE_FORMAT_VERSION).build();
+
+ StepVerifier.create(collectorUndetTest.execute(fileData)).expectNext(expectedConsumerDmaapModel)
+ .verifyComplete();
+
+
+ verify(ftpsClientMock, times(1)).collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
+ verify(ftpsClientMock, times(1)).retryCollectFile();
}
@Test
public void whenWrongScheme_returnEmpty() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+
FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
.changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location("http://host.com/file.zip")
.compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
@@ -150,14 +209,10 @@ public class XnfCollectorTaskImplTest {
FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId("")
.password("").port(PORT_22).build();
when(sftpClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
- .thenReturn(Boolean.TRUE);
+ .thenReturn(new FileCollectResult());
StepVerifier.create(collectorUndetTest.execute(fileData)).expectNextCount(0).verifyComplete();
- verify(ftpsClientMock).setKeyCertPath(FTP_KEY_PATH);
- verify(ftpsClientMock).setKeyCertPassword(FTP_KEY_PASSWORD);
- verify(ftpsClientMock).setTrustedCAPath(TRUSTED_CA_PATH);
- verify(ftpsClientMock).setTrustedCAPassword(TRUSTED_CA_PASSWORD);
- verifyNoMoreInteractions(ftpsClientMock);
+ verifyNoMoreInteractions(sftpClientMock);
}
}
diff --git a/datafile-commons/pom.xml b/datafile-commons/pom.xml
index 4b812aa2..9029769e 100644
--- a/datafile-commons/pom.xml
+++ b/datafile-commons/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.0.0-SNAPSHOT</version>
+ <version>1.0.2-SNAPSHOT</version>
</parent>
<groupId>org.onap.dcaegen2.collectors.datafile</groupId>
diff --git a/datafile-dmaap-client/pom.xml b/datafile-dmaap-client/pom.xml
index 21839ab1..9eb232ba 100644
--- a/datafile-dmaap-client/pom.xml
+++ b/datafile-dmaap-client/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.0.0-SNAPSHOT</version>
+ <version>1.0.2-SNAPSHOT</version>
</parent>
<groupId>org.onap.dcaegen2.collectors.datafile</groupId>
diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/ErrorData.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/ErrorData.java
new file mode 100644
index 00000000..2585bbce
--- /dev/null
+++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/ErrorData.java
@@ -0,0 +1,41 @@
+/*
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2018 Nordix Foundation. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END========================================================================
+ */
+
+package org.onap.dcaegen2.collectors.datafile.ftp;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ErrorData {
+ private List<String> errorMessages = new ArrayList<>();
+ private List<Throwable> errorCauses = new ArrayList<>();
+
+ public void addError(String errorMessage, Throwable errorCause) {
+ errorMessages.add(errorMessage);
+ errorCauses.add(errorCause);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder message = new StringBuilder();
+ for (int i = 0; i < errorMessages.size(); i++) {
+ message.append(errorMessages.get(i)).append(" Cause: ").append(errorCauses.get(i)).append("\n");
+ }
+ return message.toString();
+ }
+}
diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectClient.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectClient.java
new file mode 100644
index 00000000..b50b0457
--- /dev/null
+++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectClient.java
@@ -0,0 +1,53 @@
+/*
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2018 Nordix Foundation. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+
+package org.onap.dcaegen2.collectors.datafile.ftp;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author
+ *
+ */
+public abstract class FileCollectClient {
+ protected static final Logger logger = LoggerFactory.getLogger(FtpsClient.class);
+
+ protected FileServerData fileServerData;
+ protected String remoteFile;
+ protected String localFile;
+ protected ErrorData errorData;
+
+ public FileCollectResult collectFile(FileServerData fileServerData, String remoteFile, String localFile) {
+ logger.trace("collectFile called with fileServerData: {}, remoteFile: {}, localFile: {}", fileServerData,
+ remoteFile, localFile);
+
+ this.fileServerData = fileServerData;
+ this.remoteFile = remoteFile;
+ this.localFile = localFile;
+
+ return retryCollectFile();
+ }
+
+ public abstract FileCollectResult retryCollectFile();
+
+ protected void addError(String errorMessage, Throwable errorCause) {
+ if (errorData == null) {
+ errorData = new ErrorData();
+ }
+ errorData.addError(errorMessage, errorCause);
+ }
+}
diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectResult.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectResult.java
new file mode 100644
index 00000000..6cd048ac
--- /dev/null
+++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FileCollectResult.java
@@ -0,0 +1,46 @@
+/*
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2018 Nordix Foundation. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END========================================================================
+ */
+
+package org.onap.dcaegen2.collectors.datafile.ftp;
+
+public class FileCollectResult {
+ private boolean result;
+ private ErrorData errorData;
+
+ public FileCollectResult() {
+ this.result = true;
+ }
+
+ public FileCollectResult(ErrorData errorData) {
+ this.errorData = errorData;
+ result = false;
+ }
+
+ public boolean downloadSuccessful() {
+ return result;
+ }
+
+ public String getErrorData() {
+ return errorData.toString();
+ }
+
+ @Override
+ public String toString() {
+ return "Download successful: " + result + " Error data: " + getErrorData();
+ }
+}
diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java
index 719013ea..a88072c7 100644
--- a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java
+++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClient.java
@@ -36,8 +36,6 @@ import org.onap.dcaegen2.collectors.datafile.ssl.ITrustManagerFactory;
import org.onap.dcaegen2.collectors.datafile.ssl.KeyManagerUtilsWrapper;
import org.onap.dcaegen2.collectors.datafile.ssl.KeyStoreWrapper;
import org.onap.dcaegen2.collectors.datafile.ssl.TrustManagerFactoryWrapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
/**
@@ -46,9 +44,7 @@ import org.springframework.stereotype.Component;
* @author <a href="mailto:martin.c.yan@est.tech">Martin Yan</a>
*/
@Component
-public class FtpsClient {
- private static final Logger logger = LoggerFactory.getLogger(FtpsClient.class);
-
+public class FtpsClient extends FileCollectClient {
private String keyCertPath;
private String keyCertPassword;
private String trustedCAPath;
@@ -58,27 +54,32 @@ public class FtpsClient {
private IKeyManagerUtils kmu;
private IKeyStore keyStore;
private ITrustManagerFactory trustManagerFactory;
- private IFile localFile;
+ private IFile lf;
private IFileSystemResource fileResource;
private IOutputStream os;
- public boolean collectFile(FileServerData fileServerData, String remoteFile, String localFile) {
- logger.trace("collectFile called with fileServerData: {}, remoteFile: {}, localFile: {}", fileServerData,
- remoteFile, localFile);
- boolean result = true;
+ @Override
+ public FileCollectResult retryCollectFile() {
+ logger.trace("retryCollectFile called");
+
+ FileCollectResult fileCollectResult;
+
IFTPSClient ftps = getFtpsClient();
ftps.setNeedClientAuth(true);
- if (setUpKeyManager(ftps) && setUpTrustedCA(ftps) && setUpConnection(fileServerData, ftps)) {
- result = getFileFromxNF(remoteFile, localFile, ftps, fileServerData);
-
- closeDownConnection(ftps);
+ if (setUpKeyManager(ftps) && setUpTrustedCA(ftps) && setUpConnection(ftps)) {
+ if (getFileFromxNF(ftps)) {
+ closeDownConnection(ftps);
+ fileCollectResult = new FileCollectResult();
+ } else {
+ fileCollectResult = new FileCollectResult(errorData);
+ }
} else {
- result = false;
+ fileCollectResult = new FileCollectResult(errorData);
}
- logger.trace("collectFile left with result: {}", result);
- return result;
+ logger.trace("retryCollectFile left with result: {}", fileCollectResult);
+ return fileCollectResult;
}
private boolean setUpKeyManager(IFTPSClient ftps) {
@@ -88,7 +89,7 @@ public class FtpsClient {
keyManagerUtils.setCredentials(keyCertPath, keyCertPassword);
ftps.setKeyManager(keyManagerUtils.getClientKeyManager());
} catch (GeneralSecurityException | IOException e) {
- logger.error("Unable to use own key store {}", keyCertPath, e);
+ addError("Unable to use own key store " + keyCertPath, e);
result = false;
}
return result;
@@ -108,13 +109,13 @@ public class FtpsClient {
ftps.setTrustManager(tmf.getTrustManagers()[0]);
} catch (Exception e) {
- logger.error("Unable to trust xNF's CA, {}", trustedCAPath, e);
+ addError("Unable to trust xNF's CA, " + trustedCAPath, e);
result = false;
}
return result;
}
- private boolean setUpConnection(FileServerData fileServerData, IFTPSClient ftps) {
+ private boolean setUpConnection(IFTPSClient ftps) {
boolean result = true;
try {
ftps.connect(fileServerData.serverAddress(), fileServerData.port());
@@ -122,7 +123,7 @@ public class FtpsClient {
boolean loginSuccesful = ftps.login(fileServerData.userId(), fileServerData.password());
if (!loginSuccesful) {
ftps.logout();
- logger.error("Unable to log in to xNF. {}", fileServerData);
+ addError("Unable to log in to xNF. " + fileServerData, null);
result = false;
}
@@ -134,24 +135,23 @@ public class FtpsClient {
ftps.execPROT("P");
} else {
ftps.disconnect();
- logger.error("Unable to connect to xNF. {}", fileServerData);
+ addError("Unable to connect to xNF. " + fileServerData + "xNF reply code: " + ftps.getReplyCode(), null);
result = false;
}
} catch (Exception ex) {
- logger.error("Unable to connect to xNF. Data: {}", fileServerData, ex);
+ addError("Unable to connect to xNF. Data: " + fileServerData, ex);
result = false;
}
logger.trace("setUpConnection return value: {}", result);
return result;
}
- private boolean getFileFromxNF(String remoteFile, String localFilePath, IFTPSClient ftps,
- FileServerData fileServerData) {
+ private boolean getFileFromxNF(IFTPSClient ftps) {
logger.trace("starting to getFile");
boolean result = true;
+ IFile outfile = getFile();
try {
- IFile outfile = getFile();
- outfile.setPath(localFilePath);
+ outfile.setPath(localFile);
outfile.createNewFile();
IOutputStream outputStream = getOutputStream();
@@ -160,9 +160,14 @@ public class FtpsClient {
ftps.retrieveFile(remoteFile, output);
output.close();
- logger.debug("File {} Download Successfull from xNF", localFilePath);
+ logger.debug("File {} Download Successfull from xNF", localFile);
} catch (IOException ex) {
- logger.error("Unable to collect file from xNF. Data: {}", fileServerData, ex);
+ addError("Unable to collect file from xNF. Data: " + fileServerData, ex);
+ try {
+ outfile.delete();
+ } catch (Exception e) {
+ // Nothing
+ }
result = false;
}
return result;
@@ -227,11 +232,11 @@ public class FtpsClient {
}
private IFile getFile() {
- if (localFile == null) {
- localFile = new FileWrapper();
+ if (lf == null) {
+ lf = new FileWrapper();
}
- return localFile;
+ return lf;
}
private IOutputStream getOutputStream() {
@@ -266,7 +271,7 @@ public class FtpsClient {
}
protected void setFile(IFile file) {
- localFile = file;
+ lf = file;
}
protected void setOutputStream(IOutputStream outputStream) {
diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java
index 5bd95b16..18b28346 100644
--- a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java
+++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java
@@ -24,8 +24,6 @@ import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
import org.apache.commons.io.FilenameUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
/**
@@ -35,11 +33,12 @@ import org.springframework.stereotype.Component;
*
*/
@Component
-public class SftpClient {
- private static final Logger logger = LoggerFactory.getLogger(SftpClient.class);
+public class SftpClient extends FileCollectClient {
+ @Override
+ public FileCollectResult retryCollectFile() {
+ logger.trace("retryCollectFile called");
- public boolean collectFile(FileServerData fileServerData, String remoteFile, String localFile) {
- boolean result = true;
+ FileCollectResult result;
Session session = setUpSession(fileServerData);
if (session != null) {
@@ -47,20 +46,22 @@ public class SftpClient {
if (sftpChannel != null) {
try {
sftpChannel.get(remoteFile, localFile);
+ result = new FileCollectResult();
logger.debug("File {} Download Successfull from xNF", FilenameUtils.getName(localFile));
} catch (SftpException e) {
- logger.error("Unable to get file from xNF. Data: {}", fileServerData, e);
- result = false;
+ addError("Unable to get file from xNF. Data: " + fileServerData, e);
+ result = new FileCollectResult(errorData);
}
sftpChannel.exit();
} else {
- result = false;
+ result = new FileCollectResult(errorData);
}
session.disconnect();
} else {
- result = false;
+ result = new FileCollectResult(errorData);
}
+ logger.trace("retryCollectFile left with result: {}", result);
return result;
}
@@ -74,7 +75,7 @@ public class SftpClient {
session.setPassword(fileServerData.password());
session.connect();
} catch (JSchException e) {
- logger.error("Unable to set up SFTP connection to xNF. Data: {}", fileServerData, e);
+ addError("Unable to set up SFTP connection to xNF. Data: " + fileServerData, e);
}
return session;
}
@@ -87,7 +88,7 @@ public class SftpClient {
channel.connect();
sftpChannel = (ChannelSftp) channel;
} catch (JSchException e) {
- logger.error("Unable to get sftp channel to xNF. Data: {}", fileServerData, e);
+ addError("Unable to get sftp channel to xNF. Data: " + fileServerData, e);
}
return sftpChannel;
}
diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/FileWrapper.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/FileWrapper.java
index f8c02f09..32b6c72f 100644
--- a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/FileWrapper.java
+++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/FileWrapper.java
@@ -41,4 +41,9 @@ public class FileWrapper implements IFile {
public File getFile() {
return file;
}
+
+ @Override
+ public boolean delete() {
+ return file.delete();
+ }
}
diff --git a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/IFile.java b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/IFile.java
index 47d868a0..a7094f69 100644
--- a/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/IFile.java
+++ b/datafile-dmaap-client/src/main/java/org/onap/dcaegen2/collectors/datafile/io/IFile.java
@@ -27,4 +27,6 @@ public interface IFile {
public boolean createNewFile() throws IOException;
public File getFile();
+
+ public boolean delete();
}
diff --git a/datafile-dmaap-client/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClientTest.java b/datafile-dmaap-client/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClientTest.java
index 5d716a9b..e9e00c38 100644
--- a/datafile-dmaap-client/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClientTest.java
+++ b/datafile-dmaap-client/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpsClientTest.java
@@ -106,9 +106,9 @@ public class FtpsClientTest {
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertTrue(result);
+ assertTrue(result.downloadSuccessful());
verify(ftpsClientMock).setNeedClientAuth(true);
verify(keyManagerUtilsMock).setCredentials(FTP_KEY_PATH, FTP_KEY_PASSWORD);
verify(ftpsClientMock).setKeyManager(keyManagerMock);
@@ -140,9 +140,9 @@ public class FtpsClientTest {
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
@@ -156,9 +156,9 @@ public class FtpsClientTest {
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
@@ -172,10 +172,10 @@ public class FtpsClientTest {
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
verify(ftpsClientMock, times(1)).logout();
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
@@ -190,10 +190,10 @@ public class FtpsClientTest {
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
verify(ftpsClientMock, times(1)).disconnect();
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
@@ -208,9 +208,9 @@ public class FtpsClientTest {
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
@@ -227,8 +227,9 @@ public class FtpsClientTest {
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
+ verify(localFileMock, times(1)).delete();
}
} \ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 8ed28c97..b04a798f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,7 @@
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.0.0-SNAPSHOT</version>
+ <version>1.0.2-SNAPSHOT</version>
<name>dcaegen2-collectors.datafile</name>
<description>datafile collector</description>
diff --git a/version.properties b/version.properties
index 1b043102..22632725 100644
--- a/version.properties
+++ b/version.properties
@@ -1,6 +1,6 @@
major=1
minor=0
-patch=0
+patch=2
base_version=${major}.${minor}.${patch}
release_version=${base_version}
snapshot_version=${base_version}-SNAPSHOT