diff options
author | 2020-02-23 19:36:38 +0100 | |
---|---|---|
committer | 2020-02-24 09:16:04 +0100 | |
commit | cc9f145afbc2f087c4aeb2afb75effcff30099cd (patch) | |
tree | 7dbeac6e9cd53cd01254cb7c00d933ded4f3ad70 /ms/blueprintsprocessor/modules/blueprints/blueprint-core | |
parent | 488047494da258c07c723fdd378c5092005613c6 (diff) |
Add tar.gz compression capability
The change modifies ArchiveUtils by change of
the library responsible for compression and
decompression of zip files. After change of
library for appache compression tar.gz was
added as another format which archive utils
can support. For ZIP files there is kept
backward compatibility because new lib
keeps implementation of ZipFile similar
to the one used before in the utils.
Issue-ID: INT-1458
Signed-off-by: Lukasz Rajewski <lukasz.rajewski@orange.com>
Change-Id: I78388ef8c5e7a23ac6664ae49c00638af38d8c8a
Diffstat (limited to 'ms/blueprintsprocessor/modules/blueprints/blueprint-core')
2 files changed, 154 insertions, 36 deletions
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml index 28060ef44..fb2daab3a 100644 --- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml +++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml @@ -91,6 +91,10 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-logging</artifactId> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </dependency> <!--Testing dependencies--> <dependency> <groupId>org.jetbrains.kotlin</groupId> diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt index 9ccf856b5..595dbce6b 100755 --- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt +++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt @@ -22,22 +22,39 @@ import com.google.common.base.Predicates import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException import org.slf4j.LoggerFactory import java.io.BufferedInputStream +import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream +import java.io.Closeable import java.io.File import java.io.FileOutputStream +import java.io.InputStream +import java.io.InputStreamReader import java.io.IOException import java.io.OutputStream -import java.nio.charset.Charset import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.Path import java.nio.file.SimpleFileVisitor import java.nio.file.attribute.BasicFileAttributes import java.util.function.Predicate +import org.apache.commons.compress.archivers.ArchiveEntry +import org.apache.commons.compress.archivers.ArchiveInputStream +import org.apache.commons.compress.archivers.ArchiveOutputStream +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream +import org.apache.commons.compress.archivers.zip.ZipFile +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream +import java.util.Enumeration import java.util.zip.Deflater -import java.util.zip.ZipEntry -import java.util.zip.ZipFile -import java.util.zip.ZipOutputStream + +enum class ArchiveType { + TarGz, + Zip +} class BluePrintArchiveUtils { @@ -51,7 +68,7 @@ class BluePrintArchiveUtils { * @param destination the output filename * @return True if OK */ - fun compress(source: File, destination: File): Boolean { + fun compress(source: File, destination: File, archiveType: ArchiveType = ArchiveType.Zip): Boolean { try { if (!destination.parentFile.exists()) { destination.parentFile.mkdirs() @@ -59,7 +76,7 @@ class BluePrintArchiveUtils { destination.createNewFile() val ignoreZipFiles = Predicate<Path> { path -> !path.endsWith(".zip") && !path.endsWith(".ZIP") } FileOutputStream(destination).use { out -> - compressFolder(source.toPath(), out, pathFilter = ignoreZipFiles) + compressFolder(source.toPath(), out, archiveType, pathFilter = ignoreZipFiles) } } catch (e: Exception) { log.error("Fail to compress folder($source) to path(${destination.path})", e) @@ -71,8 +88,12 @@ class BluePrintArchiveUtils { /** * In-memory compress an entire folder. */ - fun compressToBytes(baseDir: Path, compressionLevel: Int = Deflater.NO_COMPRESSION): ByteArray { - return compressFolder(baseDir, ByteArrayOutputStream(), compressionLevel = compressionLevel) + fun compressToBytes( + baseDir: Path, + archiveType: ArchiveType = ArchiveType.Zip, + compressionLevel: Int = Deflater.NO_COMPRESSION + ): ByteArray { + return compressFolder(baseDir, ByteArrayOutputStream(), archiveType, compressionLevel = compressionLevel) .toByteArray() } @@ -89,38 +110,51 @@ class BluePrintArchiveUtils { private fun <T> compressFolder( baseDir: Path, output: T, + archiveType: ArchiveType, pathFilter: Predicate<Path> = Predicates.alwaysTrue(), compressionLevel: Int = Deflater.DEFAULT_COMPRESSION, fixedModificationTime: Long? = null ): T where T : OutputStream { - ZipOutputStream(output) - .apply { setLevel(compressionLevel) } - .use { zos -> + val stream: ArchiveOutputStream = if (archiveType == ArchiveType.Zip) + ZipArchiveOutputStream(output).apply { setLevel(compressionLevel) } + else + TarArchiveOutputStream(GzipCompressorOutputStream(output)) + stream + .use { aos -> Files.walkFileTree(baseDir, object : SimpleFileVisitor<Path>() { @Throws(IOException::class) override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { if (pathFilter.test(file)) { - val zipEntry = ZipEntry(baseDir.relativize(file).toString()) - fixedModificationTime?.let { - zipEntry.time = it + var archiveEntry: ArchiveEntry = aos.createArchiveEntry(file.toFile(), + baseDir.relativize(file).toString()) + if (archiveType == ArchiveType.Zip) { + val entry = archiveEntry as ZipArchiveEntry + fixedModificationTime?.let { + entry.time = it + } + entry.time = 0 } - zipEntry.time = 0 - zos.putNextEntry(zipEntry) - Files.copy(file, zos) - zos.closeEntry() + aos.putArchiveEntry(archiveEntry) + Files.copy(file, aos) + aos.closeArchiveEntry() } return FileVisitResult.CONTINUE } @Throws(IOException::class) override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { - val zipEntry = ZipEntry(baseDir.relativize(dir).toString() + "/") - fixedModificationTime?.let { - zipEntry.time = it - } - zos.putNextEntry(zipEntry) - zos.closeEntry() + var archiveEntry: ArchiveEntry? + if (archiveType == ArchiveType.Zip) { + val entry = ZipArchiveEntry(baseDir.relativize(dir).toString() + "/") + fixedModificationTime?.let { + entry.time = it + } + archiveEntry = entry + } else + archiveEntry = TarArchiveEntry(baseDir.relativize(dir).toString() + "/") + aos.putArchiveEntry(archiveEntry) + aos.closeArchiveEntry() return FileVisitResult.CONTINUE } }) @@ -128,31 +162,111 @@ class BluePrintArchiveUtils { return output } - fun deCompress(zipFile: File, targetPath: String): File { - val zip = ZipFile(zipFile, Charset.defaultCharset()) - val enumeration = zip.entries() - while (enumeration.hasMoreElements()) { - val entry = enumeration.nextElement() - val destFilePath = File(targetPath, entry.name) - destFilePath.parentFile.mkdirs() + private fun getDefaultEncoding(): String? { + val bytes = byteArrayOf('D'.toByte()) + val inputStream: InputStream = ByteArrayInputStream(bytes) + val reader = InputStreamReader(inputStream) + return reader.encoding + } - if (entry.isDirectory) - continue + fun deCompress(archiveFile: File, targetPath: String, archiveType: ArchiveType = ArchiveType.Zip): File { + var enumeration: ArchiveEnumerator? = null + if (archiveType == ArchiveType.Zip) { + val zipArchive = ZipFile(archiveFile, getDefaultEncoding()) + enumeration = ArchiveEnumerator(zipArchive) + } else { // Tar Gz + var tarGzArchiveIs: InputStream = BufferedInputStream(archiveFile.inputStream()) + tarGzArchiveIs = GzipCompressorInputStream(tarGzArchiveIs) + val tarGzArchive: ArchiveInputStream = TarArchiveInputStream(tarGzArchiveIs) + enumeration = ArchiveEnumerator(tarGzArchive) + } + + enumeration.use { + while (enumeration!!.hasMoreElements()) { + val entry: ArchiveEntry? = enumeration.nextElement() + val destFilePath = File(targetPath, entry!!.name) + destFilePath.parentFile.mkdirs() - val bufferedIs = BufferedInputStream(zip.getInputStream(entry)) - bufferedIs.use { + if (entry!!.isDirectory) + continue + + val bufferedIs = BufferedInputStream(enumeration.getInputStream(entry)) destFilePath.outputStream().buffered(1024).use { bos -> bufferedIs.copyTo(bos) } + + if (!enumeration.getHasSharedEntryInputStream()) + bufferedIs.close() } } val destinationDir = File(targetPath) check(destinationDir.isDirectory && destinationDir.exists()) { - throw BluePrintProcessorException("failed to decompress blueprint(${zipFile.absolutePath}) to ($targetPath) ") + throw BluePrintProcessorException("failed to decompress blueprint(${archiveFile.absolutePath}) to ($targetPath) ") } return destinationDir } } + + class ArchiveEnumerator : Enumeration<ArchiveEntry>, Closeable { + private val zipArchive: ZipFile? + private val zipEnumeration: Enumeration<ZipArchiveEntry>? + private val archiveStream: ArchiveInputStream? + private var nextEntry: ArchiveEntry? = null + private val hasSharedEntryInputStream: Boolean + + constructor(zipFile: ZipFile) { + zipArchive = zipFile + zipEnumeration = zipFile.entries + archiveStream = null + hasSharedEntryInputStream = false + } + + constructor(archiveStream: ArchiveInputStream) { + this.archiveStream = archiveStream + zipArchive = null + zipEnumeration = null + hasSharedEntryInputStream = true + } + + fun getHasSharedEntryInputStream(): Boolean { + return hasSharedEntryInputStream + } + + fun getInputStream(entry: ArchiveEntry): InputStream? { + return if (zipArchive != null) + zipArchive?.getInputStream(entry as ZipArchiveEntry?) + else + archiveStream + } + + override fun hasMoreElements(): Boolean { + if (zipEnumeration != null) + return zipEnumeration?.hasMoreElements() + else if (archiveStream != null) { + nextEntry = archiveStream.nextEntry + if (nextEntry != null && !archiveStream.canReadEntryData(nextEntry)) + return hasMoreElements() + return nextEntry != null + } + return false + } + + override fun nextElement(): ArchiveEntry? { + if (zipEnumeration != null) + nextEntry = zipEnumeration.nextElement() + else if (archiveStream != null) { + if (nextEntry == null) + nextEntry = archiveStream.nextEntry + } + return nextEntry + } + + override fun close() { + if (zipArchive != null) + zipArchive.close() + else archiveStream?.close() + } + } } |