Fixed NoClassDefFoundError when USE_SCRIPT_COMPILE_CACHE is set to false

USE_SCRIPT_COMPILE_CACHE set to false cleans the Class Loader cache after each kotlin script execution.
When several kotlin script are executed in parallel (ie no dependency between them) and USE_SCRIPT_COMPILE_CACHE=false then
the class loader from the cache may be deleted before one of those executed kotlin script get the time to finish which to the NoClassDefFoundError.

Removed cleanupInstance method for kotlin script executors that where causing the class loader to be removed prematurely.
Now the behaviour is to remove the class loader from the cache only when we publish a new CBA which was already the case when CDS run with a single instance.
In cluster mode, a topic has been added to hazelcast to allow the instance publishing the updated CBA to communicate to the other instances by sending a message to clean the class loader
for this CBA from their cache.

Added mutex on kotlin script compilation to fix race condition. For concurrent kotlin script execution each process wanted to compile an executable but it was causing a race condition if a process tries to execute while another still compile. Mutex on the execution path prevent this behaviour

Issue-ID: CCSDK-3052
Signed-off-by: Julien Fontaine <julien.fontaine@bell.ca>
Signed-off-by: Jozsef Csongvai <jozsef.csongvai@bell.ca>
Change-Id: I6ab002352b3272898ad0b183341ee664652c8ae3

4 files changed, 12 insertions, 13 deletions

diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
index 351cf4776..9439c2d98 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
@@ -235,8 +235,6 @@ object BluePrintConstants {
 
     const val TOSCA_SPEC = "TOSCA"
 
-    val USE_SCRIPT_COMPILE_CACHE: Boolean = (System.getenv("USE_SCRIPT_COMPILE_CACHE") ?: "true").toBoolean()
-
     const val LOG_PROTECT: String = "log-protect"
 
     /** Cluster Properties */
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/interfaces/BluePrintScriptsService.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/interfaces/BluePrintScriptsService.kt
index 0f7cb79f2..aa61b0c4d 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/interfaces/BluePrintScriptsService.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/interfaces/BluePrintScriptsService.kt
@@ -36,6 +36,4 @@ interface BluePrintScriptsService {
     suspend fun <T> scriptInstance(cacheKey: String, scriptClassName: String): T
 
     suspend fun <T> scriptInstance(scriptClassName: String): T
-
-    suspend fun cleanupInstance(blueprintBasePath: String)
 }
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt
index d1b42ffe5..230097f3c 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt
@@ -16,8 +16,13 @@
 
 package org.onap.ccsdk.cds.controllerblueprints.core.scripts
 
+import com.google.common.cache.CacheBuilder
+import com.google.common.cache.CacheLoader
+import com.google.common.cache.LoadingCache
 import kotlinx.coroutines.async
 import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
@@ -44,6 +49,8 @@ open class BluePrintCompileService {
         val classPaths = classpathFromClasspathProperty()?.joinToString(File.pathSeparator) {
             it.absolutePath
         }
+        val mutexCache: LoadingCache<String, Mutex> = CacheBuilder.newBuilder()
+            .build(CacheLoader.from { s -> Mutex() })
     }
 
     /** Compile the [bluePrintSourceCode] and get the [kClassName] instance for the constructor [args] */
@@ -54,8 +61,11 @@ open class BluePrintCompileService {
     ): T {
         /** Compile the source code if needed */
         log.debug("Jar Exists : ${bluePrintSourceCode.targetJarFile.exists()}, Regenerate : ${bluePrintSourceCode.regenerate}")
-        if (!bluePrintSourceCode.targetJarFile.exists() || bluePrintSourceCode.regenerate) {
-            compile(bluePrintSourceCode)
+
+        mutexCache.get(bluePrintSourceCode.targetJarFile.absolutePath).withLock {
+            if (!bluePrintSourceCode.targetJarFile.exists() || bluePrintSourceCode.regenerate) {
+                compile(bluePrintSourceCode)
+            }
         }
 
         val classLoaderWithDependencies = BluePrintCompileCache.classLoader(bluePrintSourceCode.cacheKey)
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintScriptsServiceImpl.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintScriptsServiceImpl.kt
index fa8ca2719..f3eb1a2b9 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintScriptsServiceImpl.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintScriptsServiceImpl.kt
@@ -79,11 +79,4 @@ open class BluePrintScriptsServiceImpl : BluePrintScriptsService {
         return Thread.currentThread().contextClassLoader.loadClass(scriptClassName).constructors
             .single().newInstance(*args.toArray()) as T
     }
-
-    override suspend fun cleanupInstance(blueprintBasePath: String) {
-        if (!BluePrintConstants.USE_SCRIPT_COMPILE_CACHE) {
-            log.info("Invalidating compile cache for blueprint ($blueprintBasePath)")
-            BluePrintCompileCache.cleanClassLoader(blueprintBasePath)
-        }
-    }
 }