diff options
Diffstat (limited to 'ms')
9 files changed, 190 insertions, 105 deletions
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml index 28060ef44..fb2daab3a 100644 --- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml +++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml @@ -91,6 +91,10 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-logging</artifactId> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </dependency> <!--Testing dependencies--> <dependency> <groupId>org.jetbrains.kotlin</groupId> diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt index 9ccf856b5..595dbce6b 100755 --- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt +++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt @@ -22,22 +22,39 @@ import com.google.common.base.Predicates import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException import org.slf4j.LoggerFactory import java.io.BufferedInputStream +import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream +import java.io.Closeable import java.io.File import java.io.FileOutputStream +import java.io.InputStream +import java.io.InputStreamReader import java.io.IOException import java.io.OutputStream -import java.nio.charset.Charset import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.Path import java.nio.file.SimpleFileVisitor import java.nio.file.attribute.BasicFileAttributes import java.util.function.Predicate +import org.apache.commons.compress.archivers.ArchiveEntry +import org.apache.commons.compress.archivers.ArchiveInputStream +import org.apache.commons.compress.archivers.ArchiveOutputStream +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream +import org.apache.commons.compress.archivers.zip.ZipFile +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream +import java.util.Enumeration import java.util.zip.Deflater -import java.util.zip.ZipEntry -import java.util.zip.ZipFile -import java.util.zip.ZipOutputStream + +enum class ArchiveType { + TarGz, + Zip +} class BluePrintArchiveUtils { @@ -51,7 +68,7 @@ class BluePrintArchiveUtils { * @param destination the output filename * @return True if OK */ - fun compress(source: File, destination: File): Boolean { + fun compress(source: File, destination: File, archiveType: ArchiveType = ArchiveType.Zip): Boolean { try { if (!destination.parentFile.exists()) { destination.parentFile.mkdirs() @@ -59,7 +76,7 @@ class BluePrintArchiveUtils { destination.createNewFile() val ignoreZipFiles = Predicate<Path> { path -> !path.endsWith(".zip") && !path.endsWith(".ZIP") } FileOutputStream(destination).use { out -> - compressFolder(source.toPath(), out, pathFilter = ignoreZipFiles) + compressFolder(source.toPath(), out, archiveType, pathFilter = ignoreZipFiles) } } catch (e: Exception) { log.error("Fail to compress folder($source) to path(${destination.path})", e) @@ -71,8 +88,12 @@ class BluePrintArchiveUtils { /** * In-memory compress an entire folder. */ - fun compressToBytes(baseDir: Path, compressionLevel: Int = Deflater.NO_COMPRESSION): ByteArray { - return compressFolder(baseDir, ByteArrayOutputStream(), compressionLevel = compressionLevel) + fun compressToBytes( + baseDir: Path, + archiveType: ArchiveType = ArchiveType.Zip, + compressionLevel: Int = Deflater.NO_COMPRESSION + ): ByteArray { + return compressFolder(baseDir, ByteArrayOutputStream(), archiveType, compressionLevel = compressionLevel) .toByteArray() } @@ -89,38 +110,51 @@ class BluePrintArchiveUtils { private fun <T> compressFolder( baseDir: Path, output: T, + archiveType: ArchiveType, pathFilter: Predicate<Path> = Predicates.alwaysTrue(), compressionLevel: Int = Deflater.DEFAULT_COMPRESSION, fixedModificationTime: Long? = null ): T where T : OutputStream { - ZipOutputStream(output) - .apply { setLevel(compressionLevel) } - .use { zos -> + val stream: ArchiveOutputStream = if (archiveType == ArchiveType.Zip) + ZipArchiveOutputStream(output).apply { setLevel(compressionLevel) } + else + TarArchiveOutputStream(GzipCompressorOutputStream(output)) + stream + .use { aos -> Files.walkFileTree(baseDir, object : SimpleFileVisitor<Path>() { @Throws(IOException::class) override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { if (pathFilter.test(file)) { - val zipEntry = ZipEntry(baseDir.relativize(file).toString()) - fixedModificationTime?.let { - zipEntry.time = it + var archiveEntry: ArchiveEntry = aos.createArchiveEntry(file.toFile(), + baseDir.relativize(file).toString()) + if (archiveType == ArchiveType.Zip) { + val entry = archiveEntry as ZipArchiveEntry + fixedModificationTime?.let { + entry.time = it + } + entry.time = 0 } - zipEntry.time = 0 - zos.putNextEntry(zipEntry) - Files.copy(file, zos) - zos.closeEntry() + aos.putArchiveEntry(archiveEntry) + Files.copy(file, aos) + aos.closeArchiveEntry() } return FileVisitResult.CONTINUE } @Throws(IOException::class) override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { - val zipEntry = ZipEntry(baseDir.relativize(dir).toString() + "/") - fixedModificationTime?.let { - zipEntry.time = it - } - zos.putNextEntry(zipEntry) - zos.closeEntry() + var archiveEntry: ArchiveEntry? + if (archiveType == ArchiveType.Zip) { + val entry = ZipArchiveEntry(baseDir.relativize(dir).toString() + "/") + fixedModificationTime?.let { + entry.time = it + } + archiveEntry = entry + } else + archiveEntry = TarArchiveEntry(baseDir.relativize(dir).toString() + "/") + aos.putArchiveEntry(archiveEntry) + aos.closeArchiveEntry() return FileVisitResult.CONTINUE } }) @@ -128,31 +162,111 @@ class BluePrintArchiveUtils { return output } - fun deCompress(zipFile: File, targetPath: String): File { - val zip = ZipFile(zipFile, Charset.defaultCharset()) - val enumeration = zip.entries() - while (enumeration.hasMoreElements()) { - val entry = enumeration.nextElement() - val destFilePath = File(targetPath, entry.name) - destFilePath.parentFile.mkdirs() + private fun getDefaultEncoding(): String? { + val bytes = byteArrayOf('D'.toByte()) + val inputStream: InputStream = ByteArrayInputStream(bytes) + val reader = InputStreamReader(inputStream) + return reader.encoding + } - if (entry.isDirectory) - continue + fun deCompress(archiveFile: File, targetPath: String, archiveType: ArchiveType = ArchiveType.Zip): File { + var enumeration: ArchiveEnumerator? = null + if (archiveType == ArchiveType.Zip) { + val zipArchive = ZipFile(archiveFile, getDefaultEncoding()) + enumeration = ArchiveEnumerator(zipArchive) + } else { // Tar Gz + var tarGzArchiveIs: InputStream = BufferedInputStream(archiveFile.inputStream()) + tarGzArchiveIs = GzipCompressorInputStream(tarGzArchiveIs) + val tarGzArchive: ArchiveInputStream = TarArchiveInputStream(tarGzArchiveIs) + enumeration = ArchiveEnumerator(tarGzArchive) + } + + enumeration.use { + while (enumeration!!.hasMoreElements()) { + val entry: ArchiveEntry? = enumeration.nextElement() + val destFilePath = File(targetPath, entry!!.name) + destFilePath.parentFile.mkdirs() - val bufferedIs = BufferedInputStream(zip.getInputStream(entry)) - bufferedIs.use { + if (entry!!.isDirectory) + continue + + val bufferedIs = BufferedInputStream(enumeration.getInputStream(entry)) destFilePath.outputStream().buffered(1024).use { bos -> bufferedIs.copyTo(bos) } + + if (!enumeration.getHasSharedEntryInputStream()) + bufferedIs.close() } } val destinationDir = File(targetPath) check(destinationDir.isDirectory && destinationDir.exists()) { - throw BluePrintProcessorException("failed to decompress blueprint(${zipFile.absolutePath}) to ($targetPath) ") + throw BluePrintProcessorException("failed to decompress blueprint(${archiveFile.absolutePath}) to ($targetPath) ") } return destinationDir } } + + class ArchiveEnumerator : Enumeration<ArchiveEntry>, Closeable { + private val zipArchive: ZipFile? + private val zipEnumeration: Enumeration<ZipArchiveEntry>? + private val archiveStream: ArchiveInputStream? + private var nextEntry: ArchiveEntry? = null + private val hasSharedEntryInputStream: Boolean + + constructor(zipFile: ZipFile) { + zipArchive = zipFile + zipEnumeration = zipFile.entries + archiveStream = null + hasSharedEntryInputStream = false + } + + constructor(archiveStream: ArchiveInputStream) { + this.archiveStream = archiveStream + zipArchive = null + zipEnumeration = null + hasSharedEntryInputStream = true + } + + fun getHasSharedEntryInputStream(): Boolean { + return hasSharedEntryInputStream + } + + fun getInputStream(entry: ArchiveEntry): InputStream? { + return if (zipArchive != null) + zipArchive?.getInputStream(entry as ZipArchiveEntry?) + else + archiveStream + } + + override fun hasMoreElements(): Boolean { + if (zipEnumeration != null) + return zipEnumeration?.hasMoreElements() + else if (archiveStream != null) { + nextEntry = archiveStream.nextEntry + if (nextEntry != null && !archiveStream.canReadEntryData(nextEntry)) + return hasMoreElements() + return nextEntry != null + } + return false + } + + override fun nextElement(): ArchiveEntry? { + if (zipEnumeration != null) + nextEntry = zipEnumeration.nextElement() + else if (archiveStream != null) { + if (nextEntry == null) + nextEntry = archiveStream.nextEntry + } + return nextEntry + } + + override fun close() { + if (zipArchive != null) + zipArchive.close() + else archiveStream?.close() + } + } } diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/cluster/HazlecastClusterServiceTest.kt b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/cluster/HazlecastClusterServiceTest.kt index b298eacae..80cf41558 100644 --- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/cluster/HazlecastClusterServiceTest.kt +++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/cluster/HazlecastClusterServiceTest.kt @@ -113,7 +113,7 @@ class HazlecastClusterServiceTest { } else { ClusterInfo( id = "test-cluster", nodeId = nodeId, joinAsClient = false, - configFile = "./src/test/resources/hazelcast/hazelcast-$port.yaml", + configFile = "./src/test/resources/hazelcast/hazelcast-cluster.yaml", properties = properties ) } @@ -162,7 +162,7 @@ class HazlecastClusterServiceTest { executeLock(bluePrintClusterServices[0], "first", lockName) } val deferred2 = async { - executeLock(bluePrintClusterServices[0], "second", lockName) + executeLock(bluePrintClusterServices[1], "second", lockName) } val deferred3 = async { executeLock(bluePrintClusterServices[2], "third", lockName) diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5679.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5679.yaml deleted file mode 100644 index e7ac273ed..000000000 --- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5679.yaml +++ /dev/null @@ -1,21 +0,0 @@ -hazelcast: - cluster-name: ${CLUSTER_ID} - instance-name: node-5679 - lite-member: - enabled: false - cp-subsystem: - cp-member-count: 3 - group-size: 3 - session-time-to-live-seconds: 60 - session-heartbeat-interval-seconds: 5 - missing-cp-member-auto-removal-seconds: 120 -# network: -# join: -# multicast: -# enabled: false -# kubernetes: -# enabled: true -# namespace: MY-KUBERNETES-NAMESPACE -# service-name: MY-SERVICE-NAME -# service-label-name: MY-SERVICE-LABEL-NAME -# service-label-value: MY-SERVICE-LABEL-VALUE
\ No newline at end of file diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5680.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5680.yaml deleted file mode 100644 index cb493d169..000000000 --- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5680.yaml +++ /dev/null @@ -1,21 +0,0 @@ -hazelcast: - cluster-name: ${CLUSTER_ID} - instance-name: node-5680 - lite-member: - enabled: false - cp-subsystem: - cp-member-count: 3 - group-size: 3 - session-time-to-live-seconds: 60 - session-heartbeat-interval-seconds: 5 - missing-cp-member-auto-removal-seconds: 120 -# network: -# join: -# multicast: -# enabled: false -# kubernetes: -# enabled: true -# namespace: MY-KUBERNETES-NAMESPACE -# service-name: MY-SERVICE-NAME -# service-label-name: MY-SERVICE-LABEL-NAME -# service-label-value: MY-SERVICE-LABEL-VALUE
\ No newline at end of file diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5681.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5681.yaml deleted file mode 100644 index e60b0c506..000000000 --- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5681.yaml +++ /dev/null @@ -1,21 +0,0 @@ -hazelcast: - cluster-name: ${CLUSTER_ID} - instance-name: node-5681 - lite-member: - enabled: false - cp-subsystem: - cp-member-count: 3 - group-size: 3 - session-time-to-live-seconds: 60 - session-heartbeat-interval-seconds: 5 - missing-cp-member-auto-removal-seconds: 120 -# network: -# join: -# multicast: -# enabled: false -# kubernetes: -# enabled: true -# namespace: MY-KUBERNETES-NAMESPACE -# service-name: MY-SERVICE-NAME -# service-label-name: MY-SERVICE-LABEL-NAME -# service-label-value: MY-SERVICE-LABEL-VALUE
\ No newline at end of file diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5682.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5682.yaml index 3cb10a08b..859ea3328 100644 --- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5682.yaml +++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5682.yaml @@ -9,10 +9,11 @@ hazelcast: session-time-to-live-seconds: 60 session-heartbeat-interval-seconds: 5 missing-cp-member-auto-removal-seconds: 120 -# network: -# join: -# multicast: -# enabled: false + network: + join: + multicast: + enabled: true + multicast-group: 224.0.0.1 # kubernetes: # enabled: true # namespace: MY-KUBERNETES-NAMESPACE diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-cluster.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-cluster.yaml new file mode 100644 index 000000000..de6047a90 --- /dev/null +++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-cluster.yaml @@ -0,0 +1,23 @@ +hazelcast: + lite-member: + enabled: false + cp-subsystem: + cp-member-count: 3 + group-size: 3 + session-time-to-live-seconds: 60 + session-heartbeat-interval-seconds: 5 + missing-cp-member-auto-removal-seconds: 120 + network: + join: + multicast: + enabled: true + # Specify 224.0.0.1 instead of default 224.2.2.3 since there's some issue + # on macOs with docker installed and multicast address different than 224.0.0.1 + # https://stackoverflow.com/questions/46341715/hazelcast-multicast-does-not-work-because-of-vboxnet-which-is-used-by-docker-mac + multicast-group: 224.0.0.1 + # kubernetes: + # enabled: true + # namespace: MY-KUBERNETES-NAMESPACE + # service-name: MY-SERVICE-NAME + # service-label-name: MY-SERVICE-LABEL-NAME + # service-label-value: MY-SERVICE-LABEL-VALUE diff --git a/ms/blueprintsprocessor/parent/pom.xml b/ms/blueprintsprocessor/parent/pom.xml index 8301fbccf..d47889a48 100755 --- a/ms/blueprintsprocessor/parent/pom.xml +++ b/ms/blueprintsprocessor/parent/pom.xml @@ -60,6 +60,7 @@ <json-smart.version>2.3</json-smart.version> <commons-io-version>2.6</commons-io-version> + <commons-compress-version>1.20</commons-compress-version> <commons-collections-version>3.2.2</commons-collections-version> </properties> @@ -130,6 +131,11 @@ <version>${commons-io-version}</version> </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + <version>${commons-compress-version}</version> + </dependency> + <dependency> <groupId>com.hubspot.jinjava</groupId> <artifactId>jinjava</artifactId> <version>${jinja.version}</version> |