summaryrefslogtreecommitdiffstats
path: root/ms
diff options
context:
space:
mode:
Diffstat (limited to 'ms')
-rw-r--r--ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml4
-rwxr-xr-xms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt186
-rw-r--r--ms/blueprintsprocessor/modules/commons/processor-core/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/cluster/HazlecastClusterServiceTest.kt4
-rw-r--r--ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5679.yaml21
-rw-r--r--ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5680.yaml21
-rw-r--r--ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5681.yaml21
-rw-r--r--ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5682.yaml9
-rw-r--r--ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-cluster.yaml23
-rwxr-xr-xms/blueprintsprocessor/parent/pom.xml6
9 files changed, 190 insertions, 105 deletions
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
index 28060ef44..fb2daab3a 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
@@ -91,6 +91,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
<!--Testing dependencies-->
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
index 9ccf856b5..595dbce6b 100755
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
@@ -22,22 +22,39 @@ import com.google.common.base.Predicates
import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
import org.slf4j.LoggerFactory
import java.io.BufferedInputStream
+import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
+import java.io.Closeable
import java.io.File
import java.io.FileOutputStream
+import java.io.InputStream
+import java.io.InputStreamReader
import java.io.IOException
import java.io.OutputStream
-import java.nio.charset.Charset
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import java.util.function.Predicate
+import org.apache.commons.compress.archivers.ArchiveEntry
+import org.apache.commons.compress.archivers.ArchiveInputStream
+import org.apache.commons.compress.archivers.ArchiveOutputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipFile
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
+import java.util.Enumeration
import java.util.zip.Deflater
-import java.util.zip.ZipEntry
-import java.util.zip.ZipFile
-import java.util.zip.ZipOutputStream
+
+enum class ArchiveType {
+ TarGz,
+ Zip
+}
class BluePrintArchiveUtils {
@@ -51,7 +68,7 @@ class BluePrintArchiveUtils {
* @param destination the output filename
* @return True if OK
*/
- fun compress(source: File, destination: File): Boolean {
+ fun compress(source: File, destination: File, archiveType: ArchiveType = ArchiveType.Zip): Boolean {
try {
if (!destination.parentFile.exists()) {
destination.parentFile.mkdirs()
@@ -59,7 +76,7 @@ class BluePrintArchiveUtils {
destination.createNewFile()
val ignoreZipFiles = Predicate<Path> { path -> !path.endsWith(".zip") && !path.endsWith(".ZIP") }
FileOutputStream(destination).use { out ->
- compressFolder(source.toPath(), out, pathFilter = ignoreZipFiles)
+ compressFolder(source.toPath(), out, archiveType, pathFilter = ignoreZipFiles)
}
} catch (e: Exception) {
log.error("Fail to compress folder($source) to path(${destination.path})", e)
@@ -71,8 +88,12 @@ class BluePrintArchiveUtils {
/**
* In-memory compress an entire folder.
*/
- fun compressToBytes(baseDir: Path, compressionLevel: Int = Deflater.NO_COMPRESSION): ByteArray {
- return compressFolder(baseDir, ByteArrayOutputStream(), compressionLevel = compressionLevel)
+ fun compressToBytes(
+ baseDir: Path,
+ archiveType: ArchiveType = ArchiveType.Zip,
+ compressionLevel: Int = Deflater.NO_COMPRESSION
+ ): ByteArray {
+ return compressFolder(baseDir, ByteArrayOutputStream(), archiveType, compressionLevel = compressionLevel)
.toByteArray()
}
@@ -89,38 +110,51 @@ class BluePrintArchiveUtils {
private fun <T> compressFolder(
baseDir: Path,
output: T,
+ archiveType: ArchiveType,
pathFilter: Predicate<Path> = Predicates.alwaysTrue(),
compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
fixedModificationTime: Long? = null
): T
where T : OutputStream {
- ZipOutputStream(output)
- .apply { setLevel(compressionLevel) }
- .use { zos ->
+ val stream: ArchiveOutputStream = if (archiveType == ArchiveType.Zip)
+ ZipArchiveOutputStream(output).apply { setLevel(compressionLevel) }
+ else
+ TarArchiveOutputStream(GzipCompressorOutputStream(output))
+ stream
+ .use { aos ->
Files.walkFileTree(baseDir, object : SimpleFileVisitor<Path>() {
@Throws(IOException::class)
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
if (pathFilter.test(file)) {
- val zipEntry = ZipEntry(baseDir.relativize(file).toString())
- fixedModificationTime?.let {
- zipEntry.time = it
+ var archiveEntry: ArchiveEntry = aos.createArchiveEntry(file.toFile(),
+ baseDir.relativize(file).toString())
+ if (archiveType == ArchiveType.Zip) {
+ val entry = archiveEntry as ZipArchiveEntry
+ fixedModificationTime?.let {
+ entry.time = it
+ }
+ entry.time = 0
}
- zipEntry.time = 0
- zos.putNextEntry(zipEntry)
- Files.copy(file, zos)
- zos.closeEntry()
+ aos.putArchiveEntry(archiveEntry)
+ Files.copy(file, aos)
+ aos.closeArchiveEntry()
}
return FileVisitResult.CONTINUE
}
@Throws(IOException::class)
override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
- val zipEntry = ZipEntry(baseDir.relativize(dir).toString() + "/")
- fixedModificationTime?.let {
- zipEntry.time = it
- }
- zos.putNextEntry(zipEntry)
- zos.closeEntry()
+ var archiveEntry: ArchiveEntry?
+ if (archiveType == ArchiveType.Zip) {
+ val entry = ZipArchiveEntry(baseDir.relativize(dir).toString() + "/")
+ fixedModificationTime?.let {
+ entry.time = it
+ }
+ archiveEntry = entry
+ } else
+ archiveEntry = TarArchiveEntry(baseDir.relativize(dir).toString() + "/")
+ aos.putArchiveEntry(archiveEntry)
+ aos.closeArchiveEntry()
return FileVisitResult.CONTINUE
}
})
@@ -128,31 +162,111 @@ class BluePrintArchiveUtils {
return output
}
- fun deCompress(zipFile: File, targetPath: String): File {
- val zip = ZipFile(zipFile, Charset.defaultCharset())
- val enumeration = zip.entries()
- while (enumeration.hasMoreElements()) {
- val entry = enumeration.nextElement()
- val destFilePath = File(targetPath, entry.name)
- destFilePath.parentFile.mkdirs()
+ private fun getDefaultEncoding(): String? {
+ val bytes = byteArrayOf('D'.toByte())
+ val inputStream: InputStream = ByteArrayInputStream(bytes)
+ val reader = InputStreamReader(inputStream)
+ return reader.encoding
+ }
- if (entry.isDirectory)
- continue
+ fun deCompress(archiveFile: File, targetPath: String, archiveType: ArchiveType = ArchiveType.Zip): File {
+ var enumeration: ArchiveEnumerator? = null
+ if (archiveType == ArchiveType.Zip) {
+ val zipArchive = ZipFile(archiveFile, getDefaultEncoding())
+ enumeration = ArchiveEnumerator(zipArchive)
+ } else { // Tar Gz
+ var tarGzArchiveIs: InputStream = BufferedInputStream(archiveFile.inputStream())
+ tarGzArchiveIs = GzipCompressorInputStream(tarGzArchiveIs)
+ val tarGzArchive: ArchiveInputStream = TarArchiveInputStream(tarGzArchiveIs)
+ enumeration = ArchiveEnumerator(tarGzArchive)
+ }
+
+ enumeration.use {
+ while (enumeration!!.hasMoreElements()) {
+ val entry: ArchiveEntry? = enumeration.nextElement()
+ val destFilePath = File(targetPath, entry!!.name)
+ destFilePath.parentFile.mkdirs()
- val bufferedIs = BufferedInputStream(zip.getInputStream(entry))
- bufferedIs.use {
+ if (entry!!.isDirectory)
+ continue
+
+ val bufferedIs = BufferedInputStream(enumeration.getInputStream(entry))
destFilePath.outputStream().buffered(1024).use { bos ->
bufferedIs.copyTo(bos)
}
+
+ if (!enumeration.getHasSharedEntryInputStream())
+ bufferedIs.close()
}
}
val destinationDir = File(targetPath)
check(destinationDir.isDirectory && destinationDir.exists()) {
- throw BluePrintProcessorException("failed to decompress blueprint(${zipFile.absolutePath}) to ($targetPath) ")
+ throw BluePrintProcessorException("failed to decompress blueprint(${archiveFile.absolutePath}) to ($targetPath) ")
}
return destinationDir
}
}
+
+ class ArchiveEnumerator : Enumeration<ArchiveEntry>, Closeable {
+ private val zipArchive: ZipFile?
+ private val zipEnumeration: Enumeration<ZipArchiveEntry>?
+ private val archiveStream: ArchiveInputStream?
+ private var nextEntry: ArchiveEntry? = null
+ private val hasSharedEntryInputStream: Boolean
+
+ constructor(zipFile: ZipFile) {
+ zipArchive = zipFile
+ zipEnumeration = zipFile.entries
+ archiveStream = null
+ hasSharedEntryInputStream = false
+ }
+
+ constructor(archiveStream: ArchiveInputStream) {
+ this.archiveStream = archiveStream
+ zipArchive = null
+ zipEnumeration = null
+ hasSharedEntryInputStream = true
+ }
+
+ fun getHasSharedEntryInputStream(): Boolean {
+ return hasSharedEntryInputStream
+ }
+
+ fun getInputStream(entry: ArchiveEntry): InputStream? {
+ return if (zipArchive != null)
+ zipArchive?.getInputStream(entry as ZipArchiveEntry?)
+ else
+ archiveStream
+ }
+
+ override fun hasMoreElements(): Boolean {
+ if (zipEnumeration != null)
+ return zipEnumeration?.hasMoreElements()
+ else if (archiveStream != null) {
+ nextEntry = archiveStream.nextEntry
+ if (nextEntry != null && !archiveStream.canReadEntryData(nextEntry))
+ return hasMoreElements()
+ return nextEntry != null
+ }
+ return false
+ }
+
+ override fun nextElement(): ArchiveEntry? {
+ if (zipEnumeration != null)
+ nextEntry = zipEnumeration.nextElement()
+ else if (archiveStream != null) {
+ if (nextEntry == null)
+ nextEntry = archiveStream.nextEntry
+ }
+ return nextEntry
+ }
+
+ override fun close() {
+ if (zipArchive != null)
+ zipArchive.close()
+ else archiveStream?.close()
+ }
+ }
}
diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/cluster/HazlecastClusterServiceTest.kt b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/cluster/HazlecastClusterServiceTest.kt
index b298eacae..80cf41558 100644
--- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/cluster/HazlecastClusterServiceTest.kt
+++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/cluster/HazlecastClusterServiceTest.kt
@@ -113,7 +113,7 @@ class HazlecastClusterServiceTest {
} else {
ClusterInfo(
id = "test-cluster", nodeId = nodeId, joinAsClient = false,
- configFile = "./src/test/resources/hazelcast/hazelcast-$port.yaml",
+ configFile = "./src/test/resources/hazelcast/hazelcast-cluster.yaml",
properties = properties
)
}
@@ -162,7 +162,7 @@ class HazlecastClusterServiceTest {
executeLock(bluePrintClusterServices[0], "first", lockName)
}
val deferred2 = async {
- executeLock(bluePrintClusterServices[0], "second", lockName)
+ executeLock(bluePrintClusterServices[1], "second", lockName)
}
val deferred3 = async {
executeLock(bluePrintClusterServices[2], "third", lockName)
diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5679.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5679.yaml
deleted file mode 100644
index e7ac273ed..000000000
--- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5679.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-hazelcast:
- cluster-name: ${CLUSTER_ID}
- instance-name: node-5679
- lite-member:
- enabled: false
- cp-subsystem:
- cp-member-count: 3
- group-size: 3
- session-time-to-live-seconds: 60
- session-heartbeat-interval-seconds: 5
- missing-cp-member-auto-removal-seconds: 120
-# network:
-# join:
-# multicast:
-# enabled: false
-# kubernetes:
-# enabled: true
-# namespace: MY-KUBERNETES-NAMESPACE
-# service-name: MY-SERVICE-NAME
-# service-label-name: MY-SERVICE-LABEL-NAME
-# service-label-value: MY-SERVICE-LABEL-VALUE \ No newline at end of file
diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5680.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5680.yaml
deleted file mode 100644
index cb493d169..000000000
--- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5680.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-hazelcast:
- cluster-name: ${CLUSTER_ID}
- instance-name: node-5680
- lite-member:
- enabled: false
- cp-subsystem:
- cp-member-count: 3
- group-size: 3
- session-time-to-live-seconds: 60
- session-heartbeat-interval-seconds: 5
- missing-cp-member-auto-removal-seconds: 120
-# network:
-# join:
-# multicast:
-# enabled: false
-# kubernetes:
-# enabled: true
-# namespace: MY-KUBERNETES-NAMESPACE
-# service-name: MY-SERVICE-NAME
-# service-label-name: MY-SERVICE-LABEL-NAME
-# service-label-value: MY-SERVICE-LABEL-VALUE \ No newline at end of file
diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5681.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5681.yaml
deleted file mode 100644
index e60b0c506..000000000
--- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5681.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-hazelcast:
- cluster-name: ${CLUSTER_ID}
- instance-name: node-5681
- lite-member:
- enabled: false
- cp-subsystem:
- cp-member-count: 3
- group-size: 3
- session-time-to-live-seconds: 60
- session-heartbeat-interval-seconds: 5
- missing-cp-member-auto-removal-seconds: 120
-# network:
-# join:
-# multicast:
-# enabled: false
-# kubernetes:
-# enabled: true
-# namespace: MY-KUBERNETES-NAMESPACE
-# service-name: MY-SERVICE-NAME
-# service-label-name: MY-SERVICE-LABEL-NAME
-# service-label-value: MY-SERVICE-LABEL-VALUE \ No newline at end of file
diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5682.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5682.yaml
index 3cb10a08b..859ea3328 100644
--- a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5682.yaml
+++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-5682.yaml
@@ -9,10 +9,11 @@ hazelcast:
session-time-to-live-seconds: 60
session-heartbeat-interval-seconds: 5
missing-cp-member-auto-removal-seconds: 120
-# network:
-# join:
-# multicast:
-# enabled: false
+ network:
+ join:
+ multicast:
+ enabled: true
+ multicast-group: 224.0.0.1
# kubernetes:
# enabled: true
# namespace: MY-KUBERNETES-NAMESPACE
diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-cluster.yaml b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-cluster.yaml
new file mode 100644
index 000000000..de6047a90
--- /dev/null
+++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/test/resources/hazelcast/hazelcast-cluster.yaml
@@ -0,0 +1,23 @@
+hazelcast:
+ lite-member:
+ enabled: false
+ cp-subsystem:
+ cp-member-count: 3
+ group-size: 3
+ session-time-to-live-seconds: 60
+ session-heartbeat-interval-seconds: 5
+ missing-cp-member-auto-removal-seconds: 120
+ network:
+ join:
+ multicast:
+ enabled: true
+ # Specify 224.0.0.1 instead of default 224.2.2.3 since there's some issue
+ # on macOs with docker installed and multicast address different than 224.0.0.1
+ # https://stackoverflow.com/questions/46341715/hazelcast-multicast-does-not-work-because-of-vboxnet-which-is-used-by-docker-mac
+ multicast-group: 224.0.0.1
+ # kubernetes:
+ # enabled: true
+ # namespace: MY-KUBERNETES-NAMESPACE
+ # service-name: MY-SERVICE-NAME
+ # service-label-name: MY-SERVICE-LABEL-NAME
+ # service-label-value: MY-SERVICE-LABEL-VALUE
diff --git a/ms/blueprintsprocessor/parent/pom.xml b/ms/blueprintsprocessor/parent/pom.xml
index 8301fbccf..d47889a48 100755
--- a/ms/blueprintsprocessor/parent/pom.xml
+++ b/ms/blueprintsprocessor/parent/pom.xml
@@ -60,6 +60,7 @@
<json-smart.version>2.3</json-smart.version>
<commons-io-version>2.6</commons-io-version>
+ <commons-compress-version>1.20</commons-compress-version>
<commons-collections-version>3.2.2</commons-collections-version>
</properties>
@@ -130,6 +131,11 @@
<version>${commons-io-version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>${commons-compress-version}</version>
+ </dependency>
+ <dependency>
<groupId>com.hubspot.jinjava</groupId>
<artifactId>jinjava</artifactId>
<version>${jinja.version}</version>