summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLukasz Rajewski <lukasz.rajewski@orange.com>2020-02-23 19:36:38 +0100
committerLukasz Rajewski <lukasz.rajewski@orange.com>2020-02-24 09:16:04 +0100
commitcc9f145afbc2f087c4aeb2afb75effcff30099cd (patch)
tree7dbeac6e9cd53cd01254cb7c00d933ded4f3ad70
parent488047494da258c07c723fdd378c5092005613c6 (diff)
Add tar.gz compression capability
The change modifies ArchiveUtils by change of the library responsible for compression and decompression of zip files. After change of library for appache compression tar.gz was added as another format which archive utils can support. For ZIP files there is kept backward compatibility because new lib keeps implementation of ZipFile similar to the one used before in the utils. Issue-ID: INT-1458 Signed-off-by: Lukasz Rajewski <lukasz.rajewski@orange.com> Change-Id: I78388ef8c5e7a23ac6664ae49c00638af38d8c8a
-rw-r--r--ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml4
-rwxr-xr-xms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt186
-rwxr-xr-xms/blueprintsprocessor/parent/pom.xml6
3 files changed, 160 insertions, 36 deletions
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
index 28060ef44..fb2daab3a 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
@@ -91,6 +91,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
<!--Testing dependencies-->
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
index 9ccf856b5..595dbce6b 100755
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
@@ -22,22 +22,39 @@ import com.google.common.base.Predicates
import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
import org.slf4j.LoggerFactory
import java.io.BufferedInputStream
+import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
+import java.io.Closeable
import java.io.File
import java.io.FileOutputStream
+import java.io.InputStream
+import java.io.InputStreamReader
import java.io.IOException
import java.io.OutputStream
-import java.nio.charset.Charset
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import java.util.function.Predicate
+import org.apache.commons.compress.archivers.ArchiveEntry
+import org.apache.commons.compress.archivers.ArchiveInputStream
+import org.apache.commons.compress.archivers.ArchiveOutputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipFile
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
+import java.util.Enumeration
import java.util.zip.Deflater
-import java.util.zip.ZipEntry
-import java.util.zip.ZipFile
-import java.util.zip.ZipOutputStream
+
+enum class ArchiveType {
+ TarGz,
+ Zip
+}
class BluePrintArchiveUtils {
@@ -51,7 +68,7 @@ class BluePrintArchiveUtils {
* @param destination the output filename
* @return True if OK
*/
- fun compress(source: File, destination: File): Boolean {
+ fun compress(source: File, destination: File, archiveType: ArchiveType = ArchiveType.Zip): Boolean {
try {
if (!destination.parentFile.exists()) {
destination.parentFile.mkdirs()
@@ -59,7 +76,7 @@ class BluePrintArchiveUtils {
destination.createNewFile()
val ignoreZipFiles = Predicate<Path> { path -> !path.endsWith(".zip") && !path.endsWith(".ZIP") }
FileOutputStream(destination).use { out ->
- compressFolder(source.toPath(), out, pathFilter = ignoreZipFiles)
+ compressFolder(source.toPath(), out, archiveType, pathFilter = ignoreZipFiles)
}
} catch (e: Exception) {
log.error("Fail to compress folder($source) to path(${destination.path})", e)
@@ -71,8 +88,12 @@ class BluePrintArchiveUtils {
/**
* In-memory compress an entire folder.
*/
- fun compressToBytes(baseDir: Path, compressionLevel: Int = Deflater.NO_COMPRESSION): ByteArray {
- return compressFolder(baseDir, ByteArrayOutputStream(), compressionLevel = compressionLevel)
+ fun compressToBytes(
+ baseDir: Path,
+ archiveType: ArchiveType = ArchiveType.Zip,
+ compressionLevel: Int = Deflater.NO_COMPRESSION
+ ): ByteArray {
+ return compressFolder(baseDir, ByteArrayOutputStream(), archiveType, compressionLevel = compressionLevel)
.toByteArray()
}
@@ -89,38 +110,51 @@ class BluePrintArchiveUtils {
private fun <T> compressFolder(
baseDir: Path,
output: T,
+ archiveType: ArchiveType,
pathFilter: Predicate<Path> = Predicates.alwaysTrue(),
compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
fixedModificationTime: Long? = null
): T
where T : OutputStream {
- ZipOutputStream(output)
- .apply { setLevel(compressionLevel) }
- .use { zos ->
+ val stream: ArchiveOutputStream = if (archiveType == ArchiveType.Zip)
+ ZipArchiveOutputStream(output).apply { setLevel(compressionLevel) }
+ else
+ TarArchiveOutputStream(GzipCompressorOutputStream(output))
+ stream
+ .use { aos ->
Files.walkFileTree(baseDir, object : SimpleFileVisitor<Path>() {
@Throws(IOException::class)
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
if (pathFilter.test(file)) {
- val zipEntry = ZipEntry(baseDir.relativize(file).toString())
- fixedModificationTime?.let {
- zipEntry.time = it
+ var archiveEntry: ArchiveEntry = aos.createArchiveEntry(file.toFile(),
+ baseDir.relativize(file).toString())
+ if (archiveType == ArchiveType.Zip) {
+ val entry = archiveEntry as ZipArchiveEntry
+ fixedModificationTime?.let {
+ entry.time = it
+ }
+ entry.time = 0
}
- zipEntry.time = 0
- zos.putNextEntry(zipEntry)
- Files.copy(file, zos)
- zos.closeEntry()
+ aos.putArchiveEntry(archiveEntry)
+ Files.copy(file, aos)
+ aos.closeArchiveEntry()
}
return FileVisitResult.CONTINUE
}
@Throws(IOException::class)
override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
- val zipEntry = ZipEntry(baseDir.relativize(dir).toString() + "/")
- fixedModificationTime?.let {
- zipEntry.time = it
- }
- zos.putNextEntry(zipEntry)
- zos.closeEntry()
+ var archiveEntry: ArchiveEntry?
+ if (archiveType == ArchiveType.Zip) {
+ val entry = ZipArchiveEntry(baseDir.relativize(dir).toString() + "/")
+ fixedModificationTime?.let {
+ entry.time = it
+ }
+ archiveEntry = entry
+ } else
+ archiveEntry = TarArchiveEntry(baseDir.relativize(dir).toString() + "/")
+ aos.putArchiveEntry(archiveEntry)
+ aos.closeArchiveEntry()
return FileVisitResult.CONTINUE
}
})
@@ -128,31 +162,111 @@ class BluePrintArchiveUtils {
return output
}
- fun deCompress(zipFile: File, targetPath: String): File {
- val zip = ZipFile(zipFile, Charset.defaultCharset())
- val enumeration = zip.entries()
- while (enumeration.hasMoreElements()) {
- val entry = enumeration.nextElement()
- val destFilePath = File(targetPath, entry.name)
- destFilePath.parentFile.mkdirs()
+ private fun getDefaultEncoding(): String? {
+ val bytes = byteArrayOf('D'.toByte())
+ val inputStream: InputStream = ByteArrayInputStream(bytes)
+ val reader = InputStreamReader(inputStream)
+ return reader.encoding
+ }
- if (entry.isDirectory)
- continue
+ fun deCompress(archiveFile: File, targetPath: String, archiveType: ArchiveType = ArchiveType.Zip): File {
+ var enumeration: ArchiveEnumerator? = null
+ if (archiveType == ArchiveType.Zip) {
+ val zipArchive = ZipFile(archiveFile, getDefaultEncoding())
+ enumeration = ArchiveEnumerator(zipArchive)
+ } else { // Tar Gz
+ var tarGzArchiveIs: InputStream = BufferedInputStream(archiveFile.inputStream())
+ tarGzArchiveIs = GzipCompressorInputStream(tarGzArchiveIs)
+ val tarGzArchive: ArchiveInputStream = TarArchiveInputStream(tarGzArchiveIs)
+ enumeration = ArchiveEnumerator(tarGzArchive)
+ }
+
+ enumeration.use {
+ while (enumeration!!.hasMoreElements()) {
+ val entry: ArchiveEntry? = enumeration.nextElement()
+ val destFilePath = File(targetPath, entry!!.name)
+ destFilePath.parentFile.mkdirs()
- val bufferedIs = BufferedInputStream(zip.getInputStream(entry))
- bufferedIs.use {
+ if (entry!!.isDirectory)
+ continue
+
+ val bufferedIs = BufferedInputStream(enumeration.getInputStream(entry))
destFilePath.outputStream().buffered(1024).use { bos ->
bufferedIs.copyTo(bos)
}
+
+ if (!enumeration.getHasSharedEntryInputStream())
+ bufferedIs.close()
}
}
val destinationDir = File(targetPath)
check(destinationDir.isDirectory && destinationDir.exists()) {
- throw BluePrintProcessorException("failed to decompress blueprint(${zipFile.absolutePath}) to ($targetPath) ")
+ throw BluePrintProcessorException("failed to decompress blueprint(${archiveFile.absolutePath}) to ($targetPath) ")
}
return destinationDir
}
}
+
+ class ArchiveEnumerator : Enumeration<ArchiveEntry>, Closeable {
+ private val zipArchive: ZipFile?
+ private val zipEnumeration: Enumeration<ZipArchiveEntry>?
+ private val archiveStream: ArchiveInputStream?
+ private var nextEntry: ArchiveEntry? = null
+ private val hasSharedEntryInputStream: Boolean
+
+ constructor(zipFile: ZipFile) {
+ zipArchive = zipFile
+ zipEnumeration = zipFile.entries
+ archiveStream = null
+ hasSharedEntryInputStream = false
+ }
+
+ constructor(archiveStream: ArchiveInputStream) {
+ this.archiveStream = archiveStream
+ zipArchive = null
+ zipEnumeration = null
+ hasSharedEntryInputStream = true
+ }
+
+ fun getHasSharedEntryInputStream(): Boolean {
+ return hasSharedEntryInputStream
+ }
+
+ fun getInputStream(entry: ArchiveEntry): InputStream? {
+ return if (zipArchive != null)
+ zipArchive?.getInputStream(entry as ZipArchiveEntry?)
+ else
+ archiveStream
+ }
+
+ override fun hasMoreElements(): Boolean {
+ if (zipEnumeration != null)
+ return zipEnumeration?.hasMoreElements()
+ else if (archiveStream != null) {
+ nextEntry = archiveStream.nextEntry
+ if (nextEntry != null && !archiveStream.canReadEntryData(nextEntry))
+ return hasMoreElements()
+ return nextEntry != null
+ }
+ return false
+ }
+
+ override fun nextElement(): ArchiveEntry? {
+ if (zipEnumeration != null)
+ nextEntry = zipEnumeration.nextElement()
+ else if (archiveStream != null) {
+ if (nextEntry == null)
+ nextEntry = archiveStream.nextEntry
+ }
+ return nextEntry
+ }
+
+ override fun close() {
+ if (zipArchive != null)
+ zipArchive.close()
+ else archiveStream?.close()
+ }
+ }
}
diff --git a/ms/blueprintsprocessor/parent/pom.xml b/ms/blueprintsprocessor/parent/pom.xml
index 8301fbccf..d47889a48 100755
--- a/ms/blueprintsprocessor/parent/pom.xml
+++ b/ms/blueprintsprocessor/parent/pom.xml
@@ -60,6 +60,7 @@
<json-smart.version>2.3</json-smart.version>
<commons-io-version>2.6</commons-io-version>
+ <commons-compress-version>1.20</commons-compress-version>
<commons-collections-version>3.2.2</commons-collections-version>
</properties>
@@ -130,6 +131,11 @@
<version>${commons-io-version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>${commons-compress-version}</version>
+ </dependency>
+ <dependency>
<groupId>com.hubspot.jinjava</groupId>
<artifactId>jinjava</artifactId>
<version>${jinja.version}</version>