Merge "Add tar.gz compression capability"

3 files changed, 160 insertions, 36 deletions

diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
index 28060ef44..fb2daab3a 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
@@ -91,6 +91,10 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-logging</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+        </dependency>
         <!--Testing dependencies-->
         <dependency>
             <groupId>org.jetbrains.kotlin</groupId>
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
index 9ccf856b5..595dbce6b 100755
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
@@ -22,22 +22,39 @@ import com.google.common.base.Predicates
 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
 import org.slf4j.LoggerFactory
 import java.io.BufferedInputStream
+import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
+import java.io.Closeable
 import java.io.File
 import java.io.FileOutputStream
+import java.io.InputStream
+import java.io.InputStreamReader
 import java.io.IOException
 import java.io.OutputStream
-import java.nio.charset.Charset
 import java.nio.file.FileVisitResult
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.SimpleFileVisitor
 import java.nio.file.attribute.BasicFileAttributes
 import java.util.function.Predicate
+import org.apache.commons.compress.archivers.ArchiveEntry
+import org.apache.commons.compress.archivers.ArchiveInputStream
+import org.apache.commons.compress.archivers.ArchiveOutputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipFile
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
+import java.util.Enumeration
 import java.util.zip.Deflater
-import java.util.zip.ZipEntry
-import java.util.zip.ZipFile
-import java.util.zip.ZipOutputStream
+
+enum class ArchiveType {
+    TarGz,
+    Zip
+}
 
 class BluePrintArchiveUtils {
 
@@ -51,7 +68,7 @@ class BluePrintArchiveUtils {
          * @param destination the output filename
          * @return True if OK
          */
-        fun compress(source: File, destination: File): Boolean {
+        fun compress(source: File, destination: File, archiveType: ArchiveType = ArchiveType.Zip): Boolean {
             try {
                 if (!destination.parentFile.exists()) {
                     destination.parentFile.mkdirs()
@@ -59,7 +76,7 @@ class BluePrintArchiveUtils {
                 destination.createNewFile()
                 val ignoreZipFiles = Predicate<Path> { path -> !path.endsWith(".zip") && !path.endsWith(".ZIP") }
                 FileOutputStream(destination).use { out ->
-                    compressFolder(source.toPath(), out, pathFilter = ignoreZipFiles)
+                    compressFolder(source.toPath(), out, archiveType, pathFilter = ignoreZipFiles)
                 }
             } catch (e: Exception) {
                 log.error("Fail to compress folder($source) to path(${destination.path})", e)
@@ -71,8 +88,12 @@ class BluePrintArchiveUtils {
         /**
          * In-memory compress an entire folder.
          */
-        fun compressToBytes(baseDir: Path, compressionLevel: Int = Deflater.NO_COMPRESSION): ByteArray {
-            return compressFolder(baseDir, ByteArrayOutputStream(), compressionLevel = compressionLevel)
+        fun compressToBytes(
+            baseDir: Path,
+            archiveType: ArchiveType = ArchiveType.Zip,
+            compressionLevel: Int = Deflater.NO_COMPRESSION
+        ): ByteArray {
+            return compressFolder(baseDir, ByteArrayOutputStream(), archiveType, compressionLevel = compressionLevel)
                 .toByteArray()
         }
 
@@ -89,38 +110,51 @@ class BluePrintArchiveUtils {
         private fun <T> compressFolder(
             baseDir: Path,
             output: T,
+            archiveType: ArchiveType,
             pathFilter: Predicate<Path> = Predicates.alwaysTrue(),
             compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
             fixedModificationTime: Long? = null
         ): T
                 where T : OutputStream {
-            ZipOutputStream(output)
-                .apply { setLevel(compressionLevel) }
-                .use { zos ->
+            val stream: ArchiveOutputStream = if (archiveType == ArchiveType.Zip)
+                ZipArchiveOutputStream(output).apply { setLevel(compressionLevel) }
+            else
+                TarArchiveOutputStream(GzipCompressorOutputStream(output))
+            stream
+                .use { aos ->
                     Files.walkFileTree(baseDir, object : SimpleFileVisitor<Path>() {
                         @Throws(IOException::class)
                         override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
                             if (pathFilter.test(file)) {
-                                val zipEntry = ZipEntry(baseDir.relativize(file).toString())
-                                fixedModificationTime?.let {
-                                    zipEntry.time = it
+                                var archiveEntry: ArchiveEntry = aos.createArchiveEntry(file.toFile(),
+                                        baseDir.relativize(file).toString())
+                                if (archiveType == ArchiveType.Zip) {
+                                    val entry = archiveEntry as ZipArchiveEntry
+                                    fixedModificationTime?.let {
+                                        entry.time = it
+                                    }
+                                    entry.time = 0
                                 }
-                                zipEntry.time = 0
-                                zos.putNextEntry(zipEntry)
-                                Files.copy(file, zos)
-                                zos.closeEntry()
+                                aos.putArchiveEntry(archiveEntry)
+                                Files.copy(file, aos)
+                                aos.closeArchiveEntry()
                             }
                             return FileVisitResult.CONTINUE
                         }
 
                         @Throws(IOException::class)
                         override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
-                            val zipEntry = ZipEntry(baseDir.relativize(dir).toString() + "/")
-                            fixedModificationTime?.let {
-                                zipEntry.time = it
-                            }
-                            zos.putNextEntry(zipEntry)
-                            zos.closeEntry()
+                            var archiveEntry: ArchiveEntry?
+                            if (archiveType == ArchiveType.Zip) {
+                                val entry = ZipArchiveEntry(baseDir.relativize(dir).toString() + "/")
+                                fixedModificationTime?.let {
+                                    entry.time = it
+                                }
+                                archiveEntry = entry
+                            } else
+                                archiveEntry = TarArchiveEntry(baseDir.relativize(dir).toString() + "/")
+                            aos.putArchiveEntry(archiveEntry)
+                            aos.closeArchiveEntry()
                             return FileVisitResult.CONTINUE
                         }
                     })
@@ -128,31 +162,111 @@ class BluePrintArchiveUtils {
             return output
         }
 
-        fun deCompress(zipFile: File, targetPath: String): File {
-            val zip = ZipFile(zipFile, Charset.defaultCharset())
-            val enumeration = zip.entries()
-            while (enumeration.hasMoreElements()) {
-                val entry = enumeration.nextElement()
-                val destFilePath = File(targetPath, entry.name)
-                destFilePath.parentFile.mkdirs()
+        private fun getDefaultEncoding(): String? {
+            val bytes = byteArrayOf('D'.toByte())
+            val inputStream: InputStream = ByteArrayInputStream(bytes)
+            val reader = InputStreamReader(inputStream)
+            return reader.encoding
+        }
 
-                if (entry.isDirectory)
-                    continue
+        fun deCompress(archiveFile: File, targetPath: String, archiveType: ArchiveType = ArchiveType.Zip): File {
+            var enumeration: ArchiveEnumerator? = null
+            if (archiveType == ArchiveType.Zip) {
+                val zipArchive = ZipFile(archiveFile, getDefaultEncoding())
+                enumeration = ArchiveEnumerator(zipArchive)
+            } else { // Tar Gz
+                var tarGzArchiveIs: InputStream = BufferedInputStream(archiveFile.inputStream())
+                tarGzArchiveIs = GzipCompressorInputStream(tarGzArchiveIs)
+                val tarGzArchive: ArchiveInputStream = TarArchiveInputStream(tarGzArchiveIs)
+                enumeration = ArchiveEnumerator(tarGzArchive)
+            }
+
+            enumeration.use {
+                while (enumeration!!.hasMoreElements()) {
+                    val entry: ArchiveEntry? = enumeration.nextElement()
+                    val destFilePath = File(targetPath, entry!!.name)
+                    destFilePath.parentFile.mkdirs()
 
-                val bufferedIs = BufferedInputStream(zip.getInputStream(entry))
-                bufferedIs.use {
+                    if (entry!!.isDirectory)
+                        continue
+
+                    val bufferedIs = BufferedInputStream(enumeration.getInputStream(entry))
                     destFilePath.outputStream().buffered(1024).use { bos ->
                         bufferedIs.copyTo(bos)
                     }
+
+                    if (!enumeration.getHasSharedEntryInputStream())
+                        bufferedIs.close()
                 }
             }
 
             val destinationDir = File(targetPath)
             check(destinationDir.isDirectory && destinationDir.exists()) {
-                throw BluePrintProcessorException("failed to decompress blueprint(${zipFile.absolutePath}) to ($targetPath) ")
+                throw BluePrintProcessorException("failed to decompress blueprint(${archiveFile.absolutePath}) to ($targetPath) ")
             }
 
             return destinationDir
         }
     }
+
+    class ArchiveEnumerator : Enumeration<ArchiveEntry>, Closeable {
+        private val zipArchive: ZipFile?
+        private val zipEnumeration: Enumeration<ZipArchiveEntry>?
+        private val archiveStream: ArchiveInputStream?
+        private var nextEntry: ArchiveEntry? = null
+        private val hasSharedEntryInputStream: Boolean
+
+        constructor(zipFile: ZipFile) {
+            zipArchive = zipFile
+            zipEnumeration = zipFile.entries
+            archiveStream = null
+            hasSharedEntryInputStream = false
+        }
+
+        constructor(archiveStream: ArchiveInputStream) {
+            this.archiveStream = archiveStream
+            zipArchive = null
+            zipEnumeration = null
+            hasSharedEntryInputStream = true
+        }
+
+        fun getHasSharedEntryInputStream(): Boolean {
+            return hasSharedEntryInputStream
+        }
+
+        fun getInputStream(entry: ArchiveEntry): InputStream? {
+            return if (zipArchive != null)
+                zipArchive?.getInputStream(entry as ZipArchiveEntry?)
+            else
+                archiveStream
+        }
+
+        override fun hasMoreElements(): Boolean {
+            if (zipEnumeration != null)
+                return zipEnumeration?.hasMoreElements()
+            else if (archiveStream != null) {
+                nextEntry = archiveStream.nextEntry
+                if (nextEntry != null && !archiveStream.canReadEntryData(nextEntry))
+                    return hasMoreElements()
+                return nextEntry != null
+            }
+            return false
+        }
+
+        override fun nextElement(): ArchiveEntry? {
+            if (zipEnumeration != null)
+                nextEntry = zipEnumeration.nextElement()
+            else if (archiveStream != null) {
+                if (nextEntry == null)
+                    nextEntry = archiveStream.nextEntry
+            }
+            return nextEntry
+        }
+
+        override fun close() {
+            if (zipArchive != null)
+                zipArchive.close()
+            else archiveStream?.close()
+        }
+    }
 }
diff --git a/ms/blueprintsprocessor/parent/pom.xml b/ms/blueprintsprocessor/parent/pom.xml
index 8301fbccf..d47889a48 100755
--- a/ms/blueprintsprocessor/parent/pom.xml
+++ b/ms/blueprintsprocessor/parent/pom.xml
@@ -60,6 +60,7 @@
         <json-smart.version>2.3</json-smart.version>
 
         <commons-io-version>2.6</commons-io-version>
+        <commons-compress-version>1.20</commons-compress-version>
         <commons-collections-version>3.2.2</commons-collections-version>
     </properties>
 
@@ -130,6 +131,11 @@
                 <version>${commons-io-version}</version>
             </dependency>
             <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-compress</artifactId>
+                <version>${commons-compress-version}</version>
+            </dependency>
+            <dependency>
                 <groupId>com.hubspot.jinjava</groupId>
                 <artifactId>jinjava</artifactId>
                 <version>${jinja.version}</version>