Merge "ZIP archive support for multiple YANG files delivery on Schema Set creation using REST"

6 files changed, 197 insertions, 27 deletions

diff --git a/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java b/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java
index c53d1a42a6..532a0ca848 100644
--- a/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java
+++ b/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java
@@ -19,13 +19,18 @@
 
 package org.onap.cps.rest.utils;
 
-import static com.google.common.base.Preconditions.checkNotNull;
 import static org.opendaylight.yangtools.yang.common.YangConstants.RFC6020_YANG_FILE_EXTENSION;
 
 import com.google.common.collect.ImmutableMap;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Locale;
 import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 import lombok.AccessLevel;
 import lombok.NoArgsConstructor;
 import org.onap.cps.spi.exceptions.CpsException;
@@ -35,26 +40,81 @@ import org.springframework.web.multipart.MultipartFile;
 @NoArgsConstructor(access = AccessLevel.PRIVATE)
 public class MultipartFileUtil {
 
+    private static final String ZIP_FILE_EXTENSION = ".zip";
+    private static final String YANG_FILE_EXTENSION = RFC6020_YANG_FILE_EXTENSION;
+    private static final int READ_BUFFER_SIZE = 1024;
+
     /**
      * Extracts yang resources from multipart file instance.
      *
      * @param multipartFile the yang file uploaded
      * @return yang resources as {map} where the key is original file name, and the value is file content
-     * @throws ModelValidationException if the file name extension is not '.yang'
+     * @throws ModelValidationException if the file name extension is not '.yang' or '.zip'
+     *                                  or if zip archive contain no yang files
      * @throws CpsException             if the file content cannot be read
      */
 
     public static Map<String, String> extractYangResourcesMap(final MultipartFile multipartFile) {
-        return ImmutableMap.of(extractYangResourceName(multipartFile), extractYangResourceContent(multipartFile));
+        final String originalFileName = multipartFile.getOriginalFilename();
+        if (resourceNameEndsWithExtension(originalFileName, YANG_FILE_EXTENSION)) {
+            return ImmutableMap.of(originalFileName, extractYangResourceContent(multipartFile));
+        }
+        if (resourceNameEndsWithExtension(originalFileName, ZIP_FILE_EXTENSION)) {
+            return extractYangResourcesMapFromZipArchive(multipartFile);
+        }
+        throw new ModelValidationException("Unsupported file type.",
+            String.format("Filename %s matches none of expected extensions: %s", originalFileName,
+                Arrays.asList(YANG_FILE_EXTENSION, ZIP_FILE_EXTENSION)));
+    }
+
+    private static Map<String, String> extractYangResourcesMapFromZipArchive(final MultipartFile multipartFile) {
+        final ImmutableMap.Builder<String, String> yangResourceMapBuilder = ImmutableMap.builder();
+
+        try (
+            final InputStream inputStream = multipartFile.getInputStream();
+            final ZipInputStream zipInputStream = new ZipInputStream(inputStream);
+        ) {
+            ZipEntry zipEntry;
+            while ((zipEntry = zipInputStream.getNextEntry()) != null) {
+                extractZipEntryToMapIfApplicable(yangResourceMapBuilder, zipEntry, zipInputStream);
+            }
+            zipInputStream.closeEntry();
+
+        } catch (final IOException e) {
+            throw new CpsException("Cannot extract resources from zip archive.", e.getMessage(), e);
+        }
+
+        try {
+            final Map<String, String> yangResourceMap = yangResourceMapBuilder.build();
+            if (yangResourceMap.isEmpty()) {
+                throw new ModelValidationException("Archive contains no YANG resources.",
+                    String.format("Archive contains no files having %s extension.", YANG_FILE_EXTENSION));
+            }
+            return yangResourceMap;
+
+        } catch (final IllegalArgumentException e) {
+            throw new ModelValidationException("Invalid ZIP archive content.",
+                "Multiple resources with same name detected.", e);
+        }
     }
 
-    private static String extractYangResourceName(final MultipartFile multipartFile) {
-        final String fileName = checkNotNull(multipartFile.getOriginalFilename(), "Missing filename.");
-        if (!fileName.endsWith(RFC6020_YANG_FILE_EXTENSION)) {
-            throw new ModelValidationException("Unsupported file type.",
-                String.format("Filename %s does not end with '%s'", fileName, RFC6020_YANG_FILE_EXTENSION));
+    private static void extractZipEntryToMapIfApplicable(
+        final ImmutableMap.Builder<String, String> yangResourceMapBuilder, final ZipEntry zipEntry,
+        final ZipInputStream zipInputStream) throws IOException {
+
+        final String yangResourceName = extractResourceNameFromPath(zipEntry.getName());
+        if (zipEntry.isDirectory() || !resourceNameEndsWithExtension(yangResourceName, YANG_FILE_EXTENSION)) {
+            return;
         }
-        return fileName;
+        yangResourceMapBuilder.put(yangResourceName, extractYangResourceContent(zipInputStream));
+    }
+
+    private static boolean resourceNameEndsWithExtension(final String resourceName, final String extension) {
+        return resourceName != null && resourceName.toLowerCase(Locale.ENGLISH).endsWith(extension);
+    }
+
+    private static String extractResourceNameFromPath(final String path) {
+        return path == null ? "" : path.replaceAll("^.*[\\\\/]", "");
     }
 
     private static String extractYangResourceContent(final MultipartFile multipartFile) {
@@ -65,4 +125,14 @@ public class MultipartFileUtil {
         }
     }
 
+    private static String extractYangResourceContent(final ZipInputStream zipInputStream) throws IOException {
+        try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
+            final byte[] buffer = new byte[READ_BUFFER_SIZE];
+            int numberOfBytesRead;
+            while ((numberOfBytesRead = zipInputStream.read(buffer, 0, READ_BUFFER_SIZE)) > 0) {
+                byteArrayOutputStream.write(buffer, 0, numberOfBytesRead);
+            }
+            return byteArrayOutputStream.toString(StandardCharsets.UTF_8);
+        }
+    }
 }
diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy
index a95d606a35..60f54bfa7b 100644
--- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy
+++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy
@@ -37,6 +37,7 @@ import org.springframework.test.web.servlet.MockMvc
 import org.springframework.util.LinkedMultiValueMap
 import org.springframework.util.MultiValueMap
 import spock.lang.Specification
+import spock.lang.Unroll
 
 import static org.onap.cps.spi.CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete
@@ -85,29 +86,66 @@ class AdminRestControllerSpec extends Specification {
             response.status == HttpStatus.BAD_REQUEST.value()
     }
 
-    def 'Create schema set from yang file'() {
+    def 'Create schema set from yang file.'() {
         def yangResourceMapCapture
-        given:
+        given: 'single yang file'
             def multipartFile = createMultipartFile("filename.yang", "content")
-        when:
+        when: 'file uploaded with schema set create request'
             def response = performCreateSchemaSetRequest(multipartFile)
-        then: 'Service method is invoked with expected parameters'
+        then: 'associated service method is invoked with expected parameters'
             1 * mockCpsModuleService.createSchemaSet('test-dataspace', 'test-schema-set', _) >>
                     { args -> yangResourceMapCapture = args[2] }
             yangResourceMapCapture['filename.yang'] == 'content'
-        and: 'Response code indicates success'
+        and: 'response code indicates success'
             response.status == HttpStatus.CREATED.value()
     }
 
-    def 'Create schema set from file with invalid filename extension'() {
-        given:
+    def 'Create schema set from zip archive.'() {
+        def yangResourceMapCapture
+        given: 'zip archive with multiple .yang files inside'
+            def multipartFile = createZipMultipartFileFromResource("/yang-files-set.zip")
+        when: 'file uploaded with schema set create request'
+            def response = performCreateSchemaSetRequest(multipartFile)
+        then: 'associated service method is invoked with expected parameters'
+            1 * mockCpsModuleService.createSchemaSet('test-dataspace', 'test-schema-set', _) >>
+                    { args -> yangResourceMapCapture = args[2] }
+            yangResourceMapCapture['assembly.yang'] == "fake assembly content 1\n"
+            yangResourceMapCapture['component.yang'] == "fake component content 1\n"
+        and: 'response code indicates success'
+            response.status == HttpStatus.CREATED.value()
+    }
+
+    @Unroll
+    def 'Create schema set from zip archive having #caseDescriptor.'() {
+        when: 'zip archive having #caseDescriptor is uploaded with create schema set request'
+            def response = performCreateSchemaSetRequest(multipartFile)
+        then: 'create schema set rejected'
+            response.status == HttpStatus.BAD_REQUEST.value()
+        where: 'following cases are tested'
+            caseDescriptor                        | multipartFile
+            'no .yang files inside'               | createZipMultipartFileFromResource("/no-yang-files.zip")
+            'multiple .yang files with same name' | createZipMultipartFileFromResource("/yang-files-multiple-sets.zip")
+    }
+
+    def 'Create schema set from file with unsupported filename extension.'() {
+        given: 'file with unsupported filename extension (.doc)'
             def multipartFile = createMultipartFile("filename.doc", "content")
-        when:
+        when: 'file uploaded with schema set create request'
             def response = performCreateSchemaSetRequest(multipartFile)
-        then: 'Create schema fails'
+        then: 'create schema set rejected'
             response.status == HttpStatus.BAD_REQUEST.value()
     }
 
+    @Unroll
+    def 'Create schema set from #fileType file with IOException occurrence on processing.'() {
+        when: 'file uploaded with schema set create request'
+            def response = performCreateSchemaSetRequest(createMultipartFileForIOException(fileType))
+        then: 'the error response returned indicating internal server error occurrence'
+            response.status == HttpStatus.INTERNAL_SERVER_ERROR.value()
+        where: 'following file types are used'
+            fileType << ['YANG', 'ZIP']
+    }
+
     def 'Delete schema set.'() {
         when: 'delete schema set endpoint is invoked'
             def response = performDeleteRequest(schemaSetEndpoint)
@@ -138,6 +176,19 @@ class AdminRestControllerSpec extends Specification {
         return new MockMultipartFile("file", filename, "text/plain", content.getBytes())
     }
 
+    def createZipMultipartFileFromResource(resourcePath) {
+        return new MockMultipartFile("file", "test.zip", "application/zip",
+                getClass().getResource(resourcePath).getBytes())
+    }
+
+    def createMultipartFileForIOException(extension) {
+        def multipartFile = Mock(MockMultipartFile)
+        multipartFile.getOriginalFilename() >> "TEST." + extension
+        multipartFile.getBytes() >> { throw new IOException() }
+        multipartFile.getInputStream() >> { throw new IOException() }
+        return multipartFile
+    }
+
     def performCreateSchemaSetRequest(multipartFile) {
         return mvc.perform(
                 multipart(schemaSetsEndpoint)
diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy
index ba5aa4cac0..3e2bdec37e 100644
--- a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy
+++ b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy
@@ -19,30 +19,79 @@
 
 package org.onap.cps.rest.utils
 
+import org.onap.cps.spi.exceptions.CpsException
 import org.onap.cps.spi.exceptions.ModelValidationException
 import org.springframework.mock.web.MockMultipartFile
+import org.springframework.web.multipart.MultipartFile
 import spock.lang.Specification
+import spock.lang.Unroll
 
 class MultipartFileUtilSpec extends Specification {
 
-    def 'Extract yang resource from multipart file'() {
-        given:
+    def 'Extract yang resource from yang file.'() {
+        given: 'uploaded yang file'
             def multipartFile = new MockMultipartFile("file", "filename.yang", "text/plain", "content".getBytes())
-        when:
+        when: 'resources are extracted from the file'
             def result = MultipartFileUtil.extractYangResourcesMap(multipartFile)
-        then:
-            assert result != null
+        then: 'the expected name and content are extracted as result'
             assert result.size() == 1
             assert result.get("filename.yang") == "content"
     }
 
-    def 'Extract yang resource from  file with invalid filename extension'() {
-        given:
+    def 'Extract yang resources from zip archive.'() {
+        given: 'uploaded zip archive containing 2 yang files and 1 not yang (json) file'
+            def multipartFile = new MockMultipartFile("file", "TEST.ZIP", "application/zip",
+                    getClass().getResource("/yang-files-set.zip").getBytes())
+        when: 'resources are extracted from zip file'
+            def result = MultipartFileUtil.extractYangResourcesMap(multipartFile)
+        then: 'information from yang files is extracted, not yang file (json) is ignored'
+            assert result.size() == 2
+            assert result["assembly.yang"] == "fake assembly content 1\n"
+            assert result["component.yang"] == "fake component content 1\n"
+    }
+
+    @Unroll
+    def 'Extract resources from zip archive having #caseDescriptor.'() {
+        when: 'attempt to extract resources from zip file is performed'
+            MultipartFileUtil.extractYangResourcesMap(multipartFile)
+        then: 'the validation exception is thrown indicating invalid zip file content'
+            thrown(ModelValidationException)
+        where: 'following cases are tested'
+            caseDescriptor                      | multipartFile
+            'text files only'                   | multipartZipFileFromResource("/no-yang-files.zip")
+            'multiple yang file with same name' | multipartZipFileFromResource("/yang-files-multiple-sets.zip")
+    }
+
+    def 'Extract yang resource from a file with invalid filename extension.'() {
+        given: 'uploaded file with unsupported (.doc) exception'
             def multipartFile = new MockMultipartFile("file", "filename.doc", "text/plain", "content".getBytes())
-        when:
+        when: 'attempt to extract resources from the file is performed'
             MultipartFileUtil.extractYangResourcesMap(multipartFile)
-        then:
+        then: 'validation exception is thrown indicating the file type is not supported'
             thrown(ModelValidationException)
     }
 
+    @Unroll
+    def 'IOException thrown during yang resources extraction from #fileType file.'() {
+        when: 'attempt to extract resources from the file is performed'
+            MultipartFileUtil.extractYangResourcesMap(multipartFileForIOException(fileType))
+        then: 'CpsException is thrown indicating the internal error occurrence'
+            thrown(CpsException)
+        where: 'following file types are used'
+            fileType << ['YANG', 'ZIP']
+    }
+
+    def multipartZipFileFromResource(resourcePath) {
+        return new MockMultipartFile("file", "TEST.ZIP", "application/zip",
+                getClass().getResource(resourcePath).getBytes())
+    }
+
+    def multipartFileForIOException(extension) {
+        def multipartFile = Mock(MultipartFile)
+        multipartFile.getOriginalFilename() >> "TEST." + extension
+        multipartFile.getBytes() >> { throw new IOException() }
+        multipartFile.getInputStream() >> { throw new IOException() }
+        return multipartFile
+    }
+
 }
diff --git a/cps-rest/src/test/resources/no-yang-files.zip b/cps-rest/src/test/resources/no-yang-files.zip
new file mode 100644
index 0000000000..83f6963b18
--- /dev/null
+++ b/cps-rest/src/test/resources/no-yang-files.zip
diff --git a/cps-rest/src/test/resources/yang-files-multiple-sets.zip b/cps-rest/src/test/resources/yang-files-multiple-sets.zip
new file mode 100644
index 0000000000..855e87bb27
--- /dev/null
+++ b/cps-rest/src/test/resources/yang-files-multiple-sets.zip
diff --git a/cps-rest/src/test/resources/yang-files-set.zip b/cps-rest/src/test/resources/yang-files-set.zip
new file mode 100644
index 0000000000..09236ce474
--- /dev/null
+++ b/cps-rest/src/test/resources/yang-files-set.zip