aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBrinda Santh Muthuramalingam <brindasanth@in.ibm.com>2020-02-25 18:48:47 +0000
committerGerrit Code Review <gerrit@onap.org>2020-02-25 18:48:47 +0000
commit319521c8bcc13a37268ec6279d1eba6db90568d5 (patch)
tree64b2602f4c01b931c5742769a6a863eae74a4176
parentfe3f59a016a53f7b19a107f73fb2341677a29651 (diff)
parentcc9f145afbc2f087c4aeb2afb75effcff30099cd (diff)
Merge "Add tar.gz compression capability"
-rw-r--r--ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml4
-rwxr-xr-xms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt186
-rwxr-xr-xms/blueprintsprocessor/parent/pom.xml6
3 files changed, 160 insertions, 36 deletions
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
index 28060ef44..fb2daab3a 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
@@ -91,6 +91,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
<!--Testing dependencies-->
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
index 9ccf856b5..595dbce6b 100755
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
@@ -22,22 +22,39 @@ import com.google.common.base.Predicates
import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
import org.slf4j.LoggerFactory
import java.io.BufferedInputStream
+import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
+import java.io.Closeable
import java.io.File
import java.io.FileOutputStream
+import java.io.InputStream
+import java.io.InputStreamReader
import java.io.IOException
import java.io.OutputStream
-import java.nio.charset.Charset
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import java.util.function.Predicate
+import org.apache.commons.compress.archivers.ArchiveEntry
+import org.apache.commons.compress.archivers.ArchiveInputStream
+import org.apache.commons.compress.archivers.ArchiveOutputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipFile
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
+import java.util.Enumeration
import java.util.zip.Deflater
-import java.util.zip.ZipEntry
-import java.util.zip.ZipFile
-import java.util.zip.ZipOutputStream
+
+enum class ArchiveType {
+ TarGz,
+ Zip
+}
class BluePrintArchiveUtils {
@@ -51,7 +68,7 @@ class BluePrintArchiveUtils {
* @param destination the output filename
* @return True if OK
*/
- fun compress(source: File, destination: File): Boolean {
+ fun compress(source: File, destination: File, archiveType: ArchiveType = ArchiveType.Zip): Boolean {
try {
if (!destination.parentFile.exists()) {
destination.parentFile.mkdirs()
@@ -59,7 +76,7 @@ class BluePrintArchiveUtils {
destination.createNewFile()
val ignoreZipFiles = Predicate<Path> { path -> !path.endsWith(".zip") && !path.endsWith(".ZIP") }
FileOutputStream(destination).use { out ->
- compressFolder(source.toPath(), out, pathFilter = ignoreZipFiles)
+ compressFolder(source.toPath(), out, archiveType, pathFilter = ignoreZipFiles)
}
} catch (e: Exception) {
log.error("Fail to compress folder($source) to path(${destination.path})", e)
@@ -71,8 +88,12 @@ class BluePrintArchiveUtils {
/**
* In-memory compress an entire folder.
*/
- fun compressToBytes(baseDir: Path, compressionLevel: Int = Deflater.NO_COMPRESSION): ByteArray {
- return compressFolder(baseDir, ByteArrayOutputStream(), compressionLevel = compressionLevel)
+ fun compressToBytes(
+ baseDir: Path,
+ archiveType: ArchiveType = ArchiveType.Zip,
+ compressionLevel: Int = Deflater.NO_COMPRESSION
+ ): ByteArray {
+ return compressFolder(baseDir, ByteArrayOutputStream(), archiveType, compressionLevel = compressionLevel)
.toByteArray()
}
@@ -89,38 +110,51 @@ class BluePrintArchiveUtils {
private fun <T> compressFolder(
baseDir: Path,
output: T,
+ archiveType: ArchiveType,
pathFilter: Predicate<Path> = Predicates.alwaysTrue(),
compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
fixedModificationTime: Long? = null
): T
where T : OutputStream {
- ZipOutputStream(output)
- .apply { setLevel(compressionLevel) }
- .use { zos ->
+ val stream: ArchiveOutputStream = if (archiveType == ArchiveType.Zip)
+ ZipArchiveOutputStream(output).apply { setLevel(compressionLevel) }
+ else
+ TarArchiveOutputStream(GzipCompressorOutputStream(output))
+ stream
+ .use { aos ->
Files.walkFileTree(baseDir, object : SimpleFileVisitor<Path>() {
@Throws(IOException::class)
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
if (pathFilter.test(file)) {
- val zipEntry = ZipEntry(baseDir.relativize(file).toString())
- fixedModificationTime?.let {
- zipEntry.time = it
+ var archiveEntry: ArchiveEntry = aos.createArchiveEntry(file.toFile(),
+ baseDir.relativize(file).toString())
+ if (archiveType == ArchiveType.Zip) {
+ val entry = archiveEntry as ZipArchiveEntry
+ fixedModificationTime?.let {
+ entry.time = it
+ }
+ entry.time = 0
}
- zipEntry.time = 0
- zos.putNextEntry(zipEntry)
- Files.copy(file, zos)
- zos.closeEntry()
+ aos.putArchiveEntry(archiveEntry)
+ Files.copy(file, aos)
+ aos.closeArchiveEntry()
}
return FileVisitResult.CONTINUE
}
@Throws(IOException::class)
override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
- val zipEntry = ZipEntry(baseDir.relativize(dir).toString() + "/")
- fixedModificationTime?.let {
- zipEntry.time = it
- }
- zos.putNextEntry(zipEntry)
- zos.closeEntry()
+ var archiveEntry: ArchiveEntry?
+ if (archiveType == ArchiveType.Zip) {
+ val entry = ZipArchiveEntry(baseDir.relativize(dir).toString() + "/")
+ fixedModificationTime?.let {
+ entry.time = it
+ }
+ archiveEntry = entry
+ } else
+ archiveEntry = TarArchiveEntry(baseDir.relativize(dir).toString() + "/")
+ aos.putArchiveEntry(archiveEntry)
+ aos.closeArchiveEntry()
return FileVisitResult.CONTINUE
}
})
@@ -128,31 +162,111 @@ class BluePrintArchiveUtils {
return output
}
- fun deCompress(zipFile: File, targetPath: String): File {
- val zip = ZipFile(zipFile, Charset.defaultCharset())
- val enumeration = zip.entries()
- while (enumeration.hasMoreElements()) {
- val entry = enumeration.nextElement()
- val destFilePath = File(targetPath, entry.name)
- destFilePath.parentFile.mkdirs()
+ private fun getDefaultEncoding(): String? {
+ val bytes = byteArrayOf('D'.toByte())
+ val inputStream: InputStream = ByteArrayInputStream(bytes)
+ val reader = InputStreamReader(inputStream)
+ return reader.encoding
+ }
- if (entry.isDirectory)
- continue
+ fun deCompress(archiveFile: File, targetPath: String, archiveType: ArchiveType = ArchiveType.Zip): File {
+ var enumeration: ArchiveEnumerator? = null
+ if (archiveType == ArchiveType.Zip) {
+ val zipArchive = ZipFile(archiveFile, getDefaultEncoding())
+ enumeration = ArchiveEnumerator(zipArchive)
+ } else { // Tar Gz
+ var tarGzArchiveIs: InputStream = BufferedInputStream(archiveFile.inputStream())
+ tarGzArchiveIs = GzipCompressorInputStream(tarGzArchiveIs)
+ val tarGzArchive: ArchiveInputStream = TarArchiveInputStream(tarGzArchiveIs)
+ enumeration = ArchiveEnumerator(tarGzArchive)
+ }
+
+ enumeration.use {
+ while (enumeration!!.hasMoreElements()) {
+ val entry: ArchiveEntry? = enumeration.nextElement()
+ val destFilePath = File(targetPath, entry!!.name)
+ destFilePath.parentFile.mkdirs()
- val bufferedIs = BufferedInputStream(zip.getInputStream(entry))
- bufferedIs.use {
+ if (entry!!.isDirectory)
+ continue
+
+ val bufferedIs = BufferedInputStream(enumeration.getInputStream(entry))
destFilePath.outputStream().buffered(1024).use { bos ->
bufferedIs.copyTo(bos)
}
+
+ if (!enumeration.getHasSharedEntryInputStream())
+ bufferedIs.close()
}
}
val destinationDir = File(targetPath)
check(destinationDir.isDirectory && destinationDir.exists()) {
- throw BluePrintProcessorException("failed to decompress blueprint(${zipFile.absolutePath}) to ($targetPath) ")
+ throw BluePrintProcessorException("failed to decompress blueprint(${archiveFile.absolutePath}) to ($targetPath) ")
}
return destinationDir
}
}
+
+ class ArchiveEnumerator : Enumeration<ArchiveEntry>, Closeable {
+ private val zipArchive: ZipFile?
+ private val zipEnumeration: Enumeration<ZipArchiveEntry>?
+ private val archiveStream: ArchiveInputStream?
+ private var nextEntry: ArchiveEntry? = null
+ private val hasSharedEntryInputStream: Boolean
+
+ constructor(zipFile: ZipFile) {
+ zipArchive = zipFile
+ zipEnumeration = zipFile.entries
+ archiveStream = null
+ hasSharedEntryInputStream = false
+ }
+
+ constructor(archiveStream: ArchiveInputStream) {
+ this.archiveStream = archiveStream
+ zipArchive = null
+ zipEnumeration = null
+ hasSharedEntryInputStream = true
+ }
+
+ fun getHasSharedEntryInputStream(): Boolean {
+ return hasSharedEntryInputStream
+ }
+
+ fun getInputStream(entry: ArchiveEntry): InputStream? {
+ return if (zipArchive != null)
+ zipArchive?.getInputStream(entry as ZipArchiveEntry?)
+ else
+ archiveStream
+ }
+
+ override fun hasMoreElements(): Boolean {
+ if (zipEnumeration != null)
+ return zipEnumeration?.hasMoreElements()
+ else if (archiveStream != null) {
+ nextEntry = archiveStream.nextEntry
+ if (nextEntry != null && !archiveStream.canReadEntryData(nextEntry))
+ return hasMoreElements()
+ return nextEntry != null
+ }
+ return false
+ }
+
+ override fun nextElement(): ArchiveEntry? {
+ if (zipEnumeration != null)
+ nextEntry = zipEnumeration.nextElement()
+ else if (archiveStream != null) {
+ if (nextEntry == null)
+ nextEntry = archiveStream.nextEntry
+ }
+ return nextEntry
+ }
+
+ override fun close() {
+ if (zipArchive != null)
+ zipArchive.close()
+ else archiveStream?.close()
+ }
+ }
}
diff --git a/ms/blueprintsprocessor/parent/pom.xml b/ms/blueprintsprocessor/parent/pom.xml
index 8301fbccf..d47889a48 100755
--- a/ms/blueprintsprocessor/parent/pom.xml
+++ b/ms/blueprintsprocessor/parent/pom.xml
@@ -60,6 +60,7 @@
<json-smart.version>2.3</json-smart.version>
<commons-io-version>2.6</commons-io-version>
+ <commons-compress-version>1.20</commons-compress-version>
<commons-collections-version>3.2.2</commons-collections-version>
</properties>
@@ -130,6 +131,11 @@
<version>${commons-io-version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>${commons-compress-version}</version>
+ </dependency>
+ <dependency>
<groupId>com.hubspot.jinjava</groupId>
<artifactId>jinjava</artifactId>
<version>${jinja.version}</version>