Add HTTP as new protocol to collect files from xNFs

- HTTP basic auth included
- small code refactoring related to the task

Issue-ID: DCAEGEN2-2527

Signed-off-by: Krzysztof Gajewski <krzysztof.gajewski@nokia.com>
Change-Id: I13ec80e996861e14d2c561087c4af3b34d861030

10 files changed, 332 insertions, 12 deletions

diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClientTest.java
index 8e6ff947..d8daa56a 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClientTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClientTest.java
@@ -42,6 +42,7 @@ import org.apache.commons.net.ftp.FTPSClient;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentMatchers;
+import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData;
 import org.springframework.http.HttpStatus;
 
 public class FtpesClientTest {
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientTest.java
index d50bfc8d..49fd3652 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientTest.java
@@ -41,6 +41,8 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
+import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData;
 import org.onap.dcaegen2.collectors.datafile.configuration.ImmutableSftpConfig;
 import org.onap.dcaegen2.collectors.datafile.configuration.SftpConfig;
 import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
new file mode 100644
index 00000000..f49cd391
--- /dev/null
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
@@ -0,0 +1,145 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2020 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData;
+import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.service.HttpUtils;
+import reactor.core.publisher.Flux;
+import reactor.netty.http.client.HttpClientConfig;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.file.Path;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class DfcHttpClientTest {
+
+    private static final String USERNAME = "bob";
+    private static final String PASSWORD = "123";
+    private static final String XNF_ADDRESS = "127.0.0.1";
+    private static final int PORT = 80;
+
+    @Mock
+    private Path pathMock;
+
+    DfcHttpClient dfcHttpClientSpy;
+
+    private ImmutableFileServerData createFileServerData() {
+        return ImmutableFileServerData.builder()
+            .serverAddress(XNF_ADDRESS)
+            .userId(USERNAME).password(PASSWORD)
+            .port(PORT)
+            .build();
+    }
+
+    @BeforeEach
+    public void setup() {
+        dfcHttpClientSpy = spy(new DfcHttpClient(createFileServerData()));
+    }
+
+    @Test
+    public void openConnection_successAuthSetup() throws DatafileTaskException {
+        dfcHttpClientSpy.open();
+        HttpClientConfig config = dfcHttpClientSpy.client.configuration();
+        assertEquals(HttpUtils.basicAuth(USERNAME, PASSWORD), config.headers().get("Authorization"));
+    }
+
+    @Test
+    public void openConnection_failedBasicAuthSetupThrowException() {
+        ImmutableFileServerData serverData = ImmutableFileServerData.builder()
+            .serverAddress(XNF_ADDRESS)
+            .userId(USERNAME).password("")
+            .port(PORT)
+            .build();
+
+        DfcHttpClient dfcHttpClientSpy = spy(new DfcHttpClient(serverData));
+
+        assertThatThrownBy(() -> dfcHttpClientSpy.open())
+            .hasMessageContaining("Not sufficient basic auth data for file.");
+    }
+
+    @Test
+    public void prepareUri_UriWithoutPort() {
+        ImmutableFileServerData serverData = ImmutableFileServerData.builder()
+            .serverAddress(XNF_ADDRESS)
+            .userId(USERNAME).password(PASSWORD)
+            .build();
+        DfcHttpClient clientNoPortSpy = spy(new DfcHttpClient(serverData));
+        String REMOTE_FILE = "any";
+
+        String retrievedUri = clientNoPortSpy.prepareUri(REMOTE_FILE);
+        assertTrue(retrievedUri.startsWith("http://" + XNF_ADDRESS + ":80"));
+    }
+
+    @Test
+    public void collectFile_AllOk() throws Exception {
+        String REMOTE_FILE = "any";
+        Flux<InputStream> fis = Flux.just(new ByteArrayInputStream("ReturnedString".getBytes()));
+
+        dfcHttpClientSpy.open();
+
+        when(dfcHttpClientSpy.getServerResponse(any())).thenReturn(fis);
+        doReturn(false).when(dfcHttpClientSpy).isDownloadFailed(any());
+
+        dfcHttpClientSpy.collectFile(REMOTE_FILE, pathMock);
+        dfcHttpClientSpy.close();
+
+        verify(dfcHttpClientSpy, times(1)).getServerResponse(ArgumentMatchers.eq(REMOTE_FILE));
+        verify(dfcHttpClientSpy, times(1)).processDataFromServer(any(), any(), any());
+        verify(dfcHttpClientSpy, times(1)).isDownloadFailed(any());
+    }
+
+    @Test
+    public void collectFile_No200ResponseWriteToErrorMessage() throws DatafileTaskException {
+        String ERROR_RESPONSE = "This is unexpected message";
+        String REMOTE_FILE = "any";
+        Flux<Throwable> fis = Flux.error(new Throwable(ERROR_RESPONSE));
+
+        dfcHttpClientSpy.open();
+
+        doReturn(fis).when(dfcHttpClientSpy).getServerResponse(any());
+
+        assertThatThrownBy(() -> dfcHttpClientSpy.collectFile(REMOTE_FILE, pathMock))
+            .hasMessageContaining("Error occured during datafile download: ");
+        verify(dfcHttpClientSpy, times(1)).getServerResponse(REMOTE_FILE);
+        verify(dfcHttpClientSpy, times(1)).processFailedConnectionWithServer(any(), any());
+        dfcHttpClientSpy.close();
+    }
+
+    @Test
+    public void isResponseOk_validateResponse() {
+        assertTrue(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.RESPONSE_OK));
+        assertFalse(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.RESPONSE_ANY_NO_OK));
+    }
+}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java
new file mode 100644
index 00000000..42ab4b3a
--- /dev/null
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java
@@ -0,0 +1,170 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2020 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.HttpVersion;
+import io.netty.handler.codec.http.cookie.Cookie;
+import reactor.netty.http.client.HttpClientResponse;
+import reactor.util.context.Context;
+import reactor.util.context.ContextView;
+
+import java.util.Map;
+import java.util.Set;
+
+public class HttpClientResponseHelper {
+
+    public static final HttpClientResponse RESPONSE_OK = new HttpClientResponse() {
+
+        @Override
+        public Map<CharSequence, Set<Cookie>> cookies() {
+            return null;
+        }
+
+        @Override
+        public boolean isKeepAlive() {
+            return false;
+        }
+
+        @Override
+        public boolean isWebsocket() {
+            return false;
+        }
+
+        @Override
+        public HttpMethod method() {
+            return null;
+        }
+
+        @Override
+        public String fullPath() {
+            return null;
+        }
+
+        @Override
+        public String uri() {
+            return null;
+        }
+
+        @Override
+        public HttpVersion version() {
+            return null;
+        }
+
+        @Override
+        public Context currentContext() {
+            return null;
+        }
+
+        @Override
+        public ContextView currentContextView() {
+            return null;
+        }
+
+        @Override
+        public String[] redirectedFrom() {
+            return new String[0];
+        }
+
+        @Override
+        public HttpHeaders requestHeaders() {
+            return null;
+        }
+
+        @Override
+        public String resourceUrl() {
+            return null;
+        }
+
+        @Override
+        public HttpHeaders responseHeaders() {
+            return null;
+        }
+
+        @Override
+        public HttpResponseStatus status() {
+            return HttpResponseStatus.OK;
+        }
+    };
+
+    public static final HttpClientResponse RESPONSE_ANY_NO_OK = new HttpClientResponse() {
+
+        @Override
+        public Map<CharSequence, Set<Cookie>> cookies() {
+            return null;
+        }
+
+        @Override
+        public boolean isKeepAlive() {
+            return false;
+        }
+
+        @Override
+        public boolean isWebsocket() {
+            return false;
+        }
+
+        @Override
+        public HttpMethod method() {
+            return null;
+        }
+
+        @Override
+        public String fullPath() {
+            return null;
+        }
+
+        @Override
+        public String uri() {
+            return null;
+        }
+
+        @Override
+        public HttpVersion version() {
+            return null;
+        }
+
+        @Override public Context currentContext() {
+            return null;
+        }
+
+        @Override public ContextView currentContextView() {
+            return null;
+        }
+
+        @Override public String[] redirectedFrom() {
+            return new String[0];
+        }
+
+        @Override public HttpHeaders requestHeaders() {
+            return null;
+        }
+
+        @Override public String resourceUrl() {
+            return null;
+        }
+
+        @Override public HttpHeaders responseHeaders() {
+            return null;
+        }
+
+        @Override public HttpResponseStatus status() {
+            return HttpResponseStatus.NOT_IMPLEMENTED;
+        }
+    };
+}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/model/FileDataTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/model/FileDataTest.java
index feeeb474..a446050e 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/model/FileDataTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/model/FileDataTest.java
@@ -24,9 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.junit.jupiter.api.Test;
-import org.onap.dcaegen2.collectors.datafile.ftp.FileServerData;
-import org.onap.dcaegen2.collectors.datafile.ftp.ImmutableFileServerData;
-import org.onap.dcaegen2.collectors.datafile.ftp.Scheme;
+import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
+import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
 
 public class FileDataTest {
     private static final String FTPES_SCHEME = "ftpes://";
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SchemeTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java
index b695f106..fb475791 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SchemeTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java
@@ -17,13 +17,15 @@
  * ============LICENSE_END=========================================================
  */
 
-package org.onap.dcaegen2.collectors.datafile.ftp;
+package org.onap.dcaegen2.collectors.datafile.scheme;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
 
 public class SchemeTest {
 
@@ -31,6 +33,7 @@ public class SchemeTest {
     public void shouldReturnSchemeForSupportedProtocol() throws DatafileTaskException {
         assertEquals(Scheme.FTPES, Scheme.getSchemeFromString("FTPES"));
         assertEquals(Scheme.SFTP, Scheme.getSchemeFromString("SFTP"));
+        assertEquals(Scheme.HTTP, Scheme.getSchemeFromString("HTTP"));
     }
 
     @Test
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/JsonMessageParserTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/JsonMessageParserTest.java
index 9e642b7d..c7ef8dac 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/JsonMessageParserTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/JsonMessageParserTest.java
@@ -35,7 +35,7 @@ import java.util.Optional;
 
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
-import org.onap.dcaegen2.collectors.datafile.ftp.Scheme;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
 import org.onap.dcaegen2.collectors.datafile.model.FileData;
 import org.onap.dcaegen2.collectors.datafile.model.FileReadyMessage;
 import org.onap.dcaegen2.collectors.datafile.model.ImmutableFileData;
@@ -217,7 +217,7 @@ class JsonMessageParserTest {
     void whenPassingCorrectJsonWrongScheme_noMessage() {
         AdditionalField additionalField = new JsonMessage.AdditionalFieldBuilder() //
             .name(PM_FILE_NAME) //
-            .location("http://location.xml") //
+            .location("unreal://location.xml") //
             .compression(GZIP_COMPRESSION) //
             .fileFormatType(FILE_FORMAT_TYPE) //
             .fileFormatVersion(FILE_FORMAT_VERSION) //
@@ -242,8 +242,8 @@ class JsonMessageParserTest {
 
         assertTrue(logAppender.list.toString()
                 .contains(ERROR_LOG_TAG + JsonMessageParser.ERROR_MSG_VES_EVENT_PARSING
-                    + Scheme.DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG + "http" + Scheme.SUPPORTED_PROTOCOLS_ERROR_MESSAGE
-                    + ". Location: http://location.xml"),"Error missing in log");
+                    + Scheme.DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG + "unreal" + Scheme.SUPPORTED_PROTOCOLS_ERROR_MESSAGE
+                    + ". Location: unreal://location.xml"),"Error missing in log");
         assertTrue(logAppender.list.toString().contains("sourceName=5GRAN_DU"),"Missing sourceName in log");
     }
 
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/DMaaPMessageConsumerTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/DMaaPMessageConsumerTest.java
index f0c8e3b3..e68913f5 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/DMaaPMessageConsumerTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/DMaaPMessageConsumerTest.java
@@ -28,7 +28,7 @@ import org.junit.jupiter.api.Test;
 import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
 import org.onap.dcaegen2.collectors.datafile.configuration.ConsumerConfiguration;
 import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
-import org.onap.dcaegen2.collectors.datafile.ftp.Scheme;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
 import org.onap.dcaegen2.collectors.datafile.model.FileData;
 import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
 import org.onap.dcaegen2.collectors.datafile.model.FileReadyMessage;
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java
index a98e2baf..1146cf26 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java
@@ -42,7 +42,7 @@ import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
 import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
 import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
 import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient;
-import org.onap.dcaegen2.collectors.datafile.ftp.Scheme;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
 import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient;
 import org.onap.dcaegen2.collectors.datafile.model.Counters;
 import org.onap.dcaegen2.collectors.datafile.model.FileData;
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java
index d5a9a92c..09d7627e 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java
@@ -55,7 +55,7 @@ import org.onap.dcaegen2.collectors.datafile.configuration.ConsumerConfiguration
 import org.onap.dcaegen2.collectors.datafile.configuration.ImmutablePublisherConfiguration;
 import org.onap.dcaegen2.collectors.datafile.configuration.PublisherConfiguration;
 import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
-import org.onap.dcaegen2.collectors.datafile.ftp.Scheme;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
 import org.onap.dcaegen2.collectors.datafile.model.FileData;
 import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
 import org.onap.dcaegen2.collectors.datafile.model.FileReadyMessage;
@@ -74,7 +74,6 @@ import org.slf4j.MDC;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.test.StepVerifier;
-import reactor.test.scheduler.VirtualTimeScheduler;
 
 public class ScheduledTasksTest {