aboutsummaryrefslogtreecommitdiffstats
path: root/ms/blueprintsprocessor
diff options
context:
space:
mode:
authorSebastien Premont-Tendland <sebastien.premont@bell.ca>2019-12-05 16:27:15 -0500
committerSebastien Premont-Tendland <sebastien.premont@bell.ca>2019-12-09 10:22:56 -0500
commit8bc22c9b5c0815f2ea56391982e9e8aafcf6fc5e (patch)
treefac38251e49191990f3707c65dd64851ddabce6d /ms/blueprintsprocessor
parentdac9afbc6df66dfe3700b37f12b51bccebf74e83 (diff)
Added option to disable kotlin script cache
using environment variable USE_SCRIPT_COMPILE_CACHE. Disabling the cache allow to scale out the BP pod without having lock file issue when using k8s NFS provisioner for PV. It also forces each pod to read the jar from disk at every request. This way they always create an instance with the latest jar file. Issue-ID: CCSDK-1969 Signed-off-by: Sebastien Premont-Tendland <sebastien.premont@bell.ca> Change-Id: I05afbdb4d49847761142542d1beca78947f032f3
Diffstat (limited to 'ms/blueprintsprocessor')
-rw-r--r--ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt2
-rw-r--r--ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt98
-rw-r--r--ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompilerCache.kt20
-rwxr-xr-xms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintFileUtils.kt31
4 files changed, 89 insertions, 62 deletions
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
index 9520f679c..30162288c 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
@@ -210,4 +210,6 @@ object BluePrintConstants {
const val MODEL_TYPE_ARTIFACT_SCRIPT_KOTLIN = "artifact-script-kotlin"
const val MODEL_TYPE_ARTIFACT_DIRECTED_GRAPH = "artifact-directed-graph"
const val MODEL_TYPE_ARTIFACT_COMPONENT_JAR = "artifact-component-jar"
+
+ val USE_SCRIPT_COMPILE_CACHE: Boolean = (System.getenv("USE_SCRIPT_COMPILE_CACHE") ?: "true").toBoolean()
}
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt
index b2857c6ce..a8c630387 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt
@@ -25,9 +25,11 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
import org.jetbrains.kotlin.config.Services
+import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintException
import org.onap.ccsdk.cds.controllerblueprints.core.checkFileExists
import org.onap.ccsdk.cds.controllerblueprints.core.logger
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintFileUtils
import java.io.File
import java.net.URLClassLoader
import java.util.ArrayList
@@ -50,11 +52,21 @@ open class BluePrintCompileService {
kClassName: String,
args: ArrayList<Any?>?
): T {
- /** Compile the source code */
- compile(bluePrintSourceCode)
- /** Get the class loader with compiled jar */
- val classLoaderWithDependencies = BluePrintCompileCache.classLoader(bluePrintSourceCode.cacheKey)
- /* Create the instance from the class loader */
+ /** Compile the source code if needed */
+ log.debug("Jar Exists : ${bluePrintSourceCode.targetJarFile.exists()}, Regenerate : ${bluePrintSourceCode.regenerate}")
+ if (!bluePrintSourceCode.targetJarFile.exists() || bluePrintSourceCode.regenerate) {
+ compile(bluePrintSourceCode)
+ }
+
+ val classLoaderWithDependencies = if (BluePrintConstants.USE_SCRIPT_COMPILE_CACHE) {
+ /** Get the class loader with compiled jar from cache */
+ BluePrintCompileCache.classLoader(bluePrintSourceCode.cacheKey)
+ } else {
+ /** Get the class loader with compiled jar from disk */
+ BluePrintFileUtils.getURLClassLoaderFromDirectory(bluePrintSourceCode.cacheKey)
+ }
+
+ /** Create the instance from the class loader */
return instance(classLoaderWithDependencies, kClassName, args)
}
@@ -64,51 +76,41 @@ open class BluePrintCompileService {
val sourcePath = bluePrintSourceCode.blueprintKotlinSources.first()
val compiledJarFile = bluePrintSourceCode.targetJarFile
- /** Check cache is present for the blueprint scripts */
- val hasCompiledCache = BluePrintCompileCache.hasClassLoader(bluePrintSourceCode.cacheKey)
-
- log.debug(
- "Jar Exists : ${compiledJarFile.exists()}, Regenerate : ${bluePrintSourceCode.regenerate}," +
- " Compiled hash(${bluePrintSourceCode.cacheKey}) : $hasCompiledCache"
- )
-
- if (!compiledJarFile.exists() || bluePrintSourceCode.regenerate || !hasCompiledCache) {
- log.info("compiling for cache key(${bluePrintSourceCode.cacheKey})")
- coroutineScope {
- val timeTaken = measureTimeMillis {
- /** Create compile arguments */
- val args = mutableListOf<String>().apply {
- add("-no-stdlib")
- add("-no-reflect")
- add("-module-name")
- add(bluePrintSourceCode.moduleName)
- add("-cp")
- add(classPaths!!)
- add(sourcePath)
- add("-d")
- add(compiledJarFile.absolutePath)
- }
- val deferredCompile = async {
- val k2jvmCompiler = K2JVMCompiler()
- /** Construct Arguments */
- val arguments = k2jvmCompiler.createArguments()
- parseCommandLineArguments(args, arguments)
- val messageCollector = CompilationMessageCollector()
- /** Compile with arguments */
- val exitCode: ExitCode = k2jvmCompiler.exec(messageCollector, Services.EMPTY, arguments)
- when (exitCode) {
- ExitCode.OK -> {
- checkFileExists(compiledJarFile) { "couldn't generate compiled jar(${compiledJarFile.absolutePath})" }
- }
- else -> {
- throw BluePrintException("$exitCode :${messageCollector.errors().joinToString("\n")}")
- }
+ log.info("compiling for cache key(${bluePrintSourceCode.cacheKey})")
+ coroutineScope {
+ val timeTaken = measureTimeMillis {
+ /** Create compile arguments */
+ val args = mutableListOf<String>().apply {
+ add("-no-stdlib")
+ add("-no-reflect")
+ add("-module-name")
+ add(bluePrintSourceCode.moduleName)
+ add("-cp")
+ add(classPaths!!)
+ add(sourcePath)
+ add("-d")
+ add(compiledJarFile.absolutePath)
+ }
+ val deferredCompile = async {
+ val k2jvmCompiler = K2JVMCompiler()
+ /** Construct Arguments */
+ val arguments = k2jvmCompiler.createArguments()
+ parseCommandLineArguments(args, arguments)
+ val messageCollector = CompilationMessageCollector()
+ /** Compile with arguments */
+ val exitCode: ExitCode = k2jvmCompiler.exec(messageCollector, Services.EMPTY, arguments)
+ when (exitCode) {
+ ExitCode.OK -> {
+ checkFileExists(compiledJarFile) { "couldn't generate compiled jar(${compiledJarFile.absolutePath})" }
+ }
+ else -> {
+ throw BluePrintException("$exitCode :${messageCollector.errors().joinToString("\n")}")
}
}
- deferredCompile.await()
}
- log.info("compiled in ($timeTaken)mSec for cache key(${bluePrintSourceCode.cacheKey})")
+ deferredCompile.await()
}
+ log.info("compiled in ($timeTaken)mSec for cache key(${bluePrintSourceCode.cacheKey})")
}
}
@@ -123,6 +125,10 @@ open class BluePrintCompileService {
kClazz.constructors
.single().newInstance(*args.toArray())
} ?: throw BluePrintException("failed to create class($kClassName) instance for constructor argument($args).")
+
+ if (!BluePrintConstants.USE_SCRIPT_COMPILE_CACHE) {
+ classLoader.close()
+ }
return instance as T
}
}
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompilerCache.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompilerCache.kt
index fbfcfb94f..cb6616f38 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompilerCache.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompilerCache.kt
@@ -21,8 +21,7 @@ import com.google.common.cache.CacheLoader
import com.google.common.cache.LoadingCache
import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintException
import org.onap.ccsdk.cds.controllerblueprints.core.logger
-import org.onap.ccsdk.cds.controllerblueprints.core.normalizedFile
-import java.net.URL
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintFileUtils
import java.net.URLClassLoader
object BluePrintCompileCache {
@@ -58,19 +57,10 @@ object BluePrintClassLoader : CacheLoader<String, URLClassLoader>() {
val log = logger(BluePrintClassLoader::class)
- override fun load(key: String): URLClassLoader {
+ override fun load(key: String) = try {
log.info("loading compiled cache($key)")
- val keyPath = normalizedFile(key)
- if (!keyPath.exists()) {
- throw BluePrintException("failed to load cache($key), missing files.")
- }
- val urls = arrayListOf<URL>()
- keyPath.walkTopDown()
- .filter { it.name.endsWith("cba-kts.jar") }
- .forEach {
- log.debug("Adding (${it.absolutePath}) to cache($key)")
- urls.add(it.toURI().toURL())
- }
- return URLClassLoader(urls.toTypedArray(), this.javaClass.classLoader)
+ BluePrintFileUtils.getURLClassLoaderFromDirectory(key)
+ } catch (e: Exception) {
+ throw BluePrintException("failed to load cache($key) with Exception($e)")
}
}
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintFileUtils.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintFileUtils.kt
index 6605e8eca..9e1047389 100755
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintFileUtils.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintFileUtils.kt
@@ -32,7 +32,11 @@ import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintContext
import org.slf4j.LoggerFactory
import java.io.File
import java.io.FileFilter
+import java.io.FileNotFoundException
+import java.net.URL
+import java.net.URLClassLoader
import java.nio.file.Files
+import java.nio.file.NotDirectoryException
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.StandardOpenOption
@@ -40,6 +44,8 @@ import java.nio.file.StandardOpenOption
class BluePrintFileUtils {
companion object {
+ const val COMPILED_JAR_SUFFIX = "cba-kts.jar"
+
private val log = LoggerFactory.getLogger(this::class.toString())
fun createEmptyBluePrint(basePath: String) {
@@ -263,7 +269,7 @@ class BluePrintFileUtils {
}
private fun compileJarFileName(artifactName: String, artifactVersion: String): String {
- return "$artifactName-$artifactVersion-cba-kts.jar"
+ return "$artifactName-$artifactVersion-$COMPILED_JAR_SUFFIX"
}
fun compileJarFilePathName(basePath: String, artifactName: String, artifactVersion: String): String {
@@ -288,5 +294,28 @@ class BluePrintFileUtils {
// In case dot is in first position, we are dealing with a hidden file rather than an extension
return if (dotIndexe > 0) fileName.substring(0, dotIndexe) else fileName
}
+
+ fun getURLClassLoaderFromDirectory(directory: File): URLClassLoader {
+ if (!directory.exists()) {
+ throw FileNotFoundException(directory.absolutePath)
+ } else if (!directory.isDirectory) {
+ throw NotDirectoryException(directory.absolutePath)
+ }
+
+ val urls = arrayListOf<URL>()
+ directory.walkTopDown()
+ .filter { it.name.endsWith(COMPILED_JAR_SUFFIX) }
+ .forEach {
+ log.debug("Adding (${it.absolutePath}) to classLoader (${directory.absolutePath})")
+
+ urls.add(it.toURI().toURL())
+ }
+ return URLClassLoader(urls.toTypedArray(), this.javaClass.classLoader)
+ }
+
+ fun getURLClassLoaderFromDirectory(path: String): URLClassLoader {
+ val directory = normalizedFile(path)
+ return getURLClassLoaderFromDirectory(directory)
+ }
}
}