cmd-exec separate env-prepare and exec timeouts.

Issue-ID: CCSDK-2039

added inputs for cmd-exec "env-prepare-timeout" and "execution-timeout" to segregate the timeouts.

rev1: initial commit
rev2: reverted application-dev.properties
rev3: apply default timeouts if they are not specified + fixed a test

Signed-off-by: Oleg Mitsura <oleg.mitsura@amdocs.com>
Change-Id: I651594932a3fc87e0d853e65879b1548d9dbde8a

3 files changed, 63 insertions, 18 deletions

diff --git a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
index acaea4e7b..8fa8e7c7b 100644
--- a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
+++ b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
@@ -60,6 +60,18 @@
               "entry_schema" : {
                 "type" : "dt-system-packages"
               }
+            },
+            "env-prepare-timeout": {
+              "description": "Time-out for environment preparation.",
+              "required": false,
+              "type": "integer",
+              "default": 120
+            },
+            "execution-timeout": {
+              "description": "Time-out for the execution step.",
+              "required": false,
+              "type": "integer",
+              "default": 180
             }
           }
         }
diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
index 6ad1e1f11..f3169d937 100644
--- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
+++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
@@ -46,14 +46,19 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
         const val INPUT_ENDPOINT_SELECTOR = "endpoint-selector"
         const val INPUT_DYNAMIC_PROPERTIES = "dynamic-properties"
         const val INPUT_ARGUMENT_PROPERTIES = "argument-properties"
+
         const val INPUT_COMMAND = "command"
         const val INPUT_PACKAGES = "packages"
         const val DEFAULT_SELECTOR = "remote-python"
+        const val INPUT_ENV_PREPARE_TIMEOUT = "env-prepare-timeout"
+        const val INPUT_EXECUTE_TIMEOUT = "execution-timeout"
 
         const val ATTRIBUTE_EXEC_CMD_STATUS = "status"
         const val ATTRIBUTE_PREPARE_ENV_LOG = "prepare-environment-logs"
         const val ATTRIBUTE_EXEC_CMD_LOG = "execute-command-logs"
         const val ATTRIBUTE_RESPONSE_DATA = "response-data"
+        const val DEFAULT_ENV_PREPARE_TIMEOUT_IN_SEC = 120
+        const val DEFAULT_EXECUTE_TIMEOUT_IN_SEC = 180
     }
 
     override suspend fun processNB(executionRequest: ExecutionServiceInput) {
@@ -90,6 +95,16 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
             ?.rootFieldsToMap()?.toSortedMap()?.values?.joinToString(" ") { formatNestedJsonNode(it) }
 
         val command = getOperationInput(INPUT_COMMAND).asText()
+
+        /**
+         * Timeouts that are specific to the command executor.
+         * Note: the interface->input->timeout is the component level timeout.
+         */
+        val envPrepTimeout = getOptionalOperationInput(INPUT_ENV_PREPARE_TIMEOUT)?.asInt()
+            ?: DEFAULT_ENV_PREPARE_TIMEOUT_IN_SEC
+        val executionTimeout = getOptionalOperationInput(INPUT_EXECUTE_TIMEOUT)?.asInt()
+            ?: DEFAULT_EXECUTE_TIMEOUT_IN_SEC
+
         var scriptCommand = command.replace(pythonScript.name, pythonScript.absolutePath)
         if (args != null && args.isNotEmpty()) {
             scriptCommand = scriptCommand.plus(" ").plus(args)
@@ -110,7 +125,9 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
                 val prepareEnvInput = PrepareRemoteEnvInput(requestId = processId,
                     remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName,
                         blueprintVersion = blueprintVersion),
-                    packages = packages
+                    packages = packages,
+                    timeOut = envPrepTimeout.toLong()
+
                 )
                 val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput)
                 log.info("$ATTRIBUTE_PREPARE_ENV_LOG - ${prepareEnvOutput.response}")
@@ -125,9 +142,22 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
                     setNodeOutputProperties(prepareEnvOutput.status.name.asJsonPrimitive(), logsEnv, "".asJsonPrimitive())
                 }
             }
-
-            // if Env preparation was successful, then proceed with command execution in this Env
-            if (bluePrintRuntimeService.getBluePrintError().errors.isEmpty()) {
+        } catch (grpcEx: io.grpc.StatusRuntimeException) {
+            val timeoutErrMsg = "Command executor timed out in GRPC call during env. preparation.. timeout($envPrepTimeout) requestId ($processId)."
+            setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, timeoutErrMsg.asJsonPrimitive())
+            setNodeOutputErrors(status = timeoutErrMsg, message = "${grpcEx.status}".asJsonPrimitive())
+            log.error(timeoutErrMsg, grpcEx)
+            addError(timeoutErrMsg)
+        } catch (e: Exception) {
+            val timeoutErrMsg = "Command executor failed during env. preparation.. timeout($envPrepTimeout) requestId ($processId)."
+            setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, e.message.asJsonPrimitive())
+            setNodeOutputErrors(status = timeoutErrMsg, message = "${e.message}".asJsonPrimitive())
+            log.error("Failed to process on remote executor requestId ($processId)", e)
+            addError(timeoutErrMsg)
+        }
+        // if Env preparation was successful, then proceed with command execution in this Env
+        if (bluePrintRuntimeService.getBluePrintError().errors.isEmpty()) {
+            try {
                 // Populate command execution properties and pass it to the remote server
                 val properties = dynamicProperties?.returnNullIfMissing()?.rootFieldsToMap() ?: hashMapOf()
 
@@ -136,14 +166,14 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
                     remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion),
                     command = scriptCommand,
                     properties = properties,
-                    timeOut = timeout.toLong())
+                    timeOut = executionTimeout.toLong())
 
 
                 val remoteExecutionOutputDeferred = GlobalScope.async {
                     remoteScriptExecutionService.executeCommand(remoteExecutionInput)
                 }
 
-                val remoteExecutionOutput = withTimeout(timeout * 1000L) {
+                val remoteExecutionOutput = withTimeout(executionTimeout * 1000L) {
                     remoteExecutionOutputDeferred.await()
                 }
 
@@ -159,21 +189,23 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
                     setNodeOutputProperties(remoteExecutionOutput.status.name.asJsonPrimitive(), logs,
                         remoteExecutionOutput.payload)
                 }
+            } catch (timeoutEx: TimeoutCancellationException) {
+                val timeoutErrMsg = "Command executor timed out executing after $executionTimeout seconds requestId ($processId)"
+                setNodeOutputErrors(status = timeoutErrMsg, message = "".asJsonPrimitive())
+                log.error(timeoutErrMsg, timeoutEx)
+            } catch (grpcEx: io.grpc.StatusRuntimeException) {
+                val timeoutErrMsg = "Command executor timed out executing after $executionTimeout seconds requestId ($processId)"
+                setNodeOutputErrors(status = timeoutErrMsg, message = "".asJsonPrimitive())
+                log.error("Command executor time out during GRPC call", grpcEx)
+            } catch (e: Exception) {
+                log.error("Failed to process on remote executor requestId ($processId)", e)
             }
-
-        } catch (timeoutEx: TimeoutCancellationException) {
-            setNodeOutputErrors(status = "Command executor timed out after $timeout seconds", message = "".asJsonPrimitive())
-            log.error("Command executor timed out after $timeout seconds", timeoutEx)
-        } catch (grpcEx: io.grpc.StatusRuntimeException) {
-            setNodeOutputErrors(status = "Command executor timed out in GRPC call", message = "${grpcEx.status}".asJsonPrimitive())
-            log.error("Command executor time out during GRPC call", grpcEx)
-        } catch (e: Exception) {
-            log.error("Failed to process on remote executor", e)
-        } finally {
-            remoteScriptExecutionService.close()
         }
+        log.debug("Trying to close GRPC channel. request ($processId)")
+        remoteScriptExecutionService.close()
     }
 
+
     override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
         bluePrintRuntimeService.getBluePrintError()
             .addError("Failed in ComponentRemotePythonExecutor : ${runtimeException.message}")
@@ -211,7 +243,6 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
         log.info("Executor message  : $message")
         setAttribute(ATTRIBUTE_RESPONSE_DATA, artifacts)
         log.info("Executor artifacts: $artifacts")
-
         addError(status, ATTRIBUTE_EXEC_CMD_LOG, message.toString())
     }
 }
diff --git a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
index 89af42579..11ced624c 100644
--- a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
+++ b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
@@ -25,6 +25,7 @@ import org.junit.Test
 import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.*
 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService
 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
+import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintError
 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonPrimitive
 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonType
 import org.onap.ccsdk.cds.controllerblueprints.core.putJsonElement
@@ -81,6 +82,7 @@ class ComponentRemotePythonExecutorTest {
             val componentRemotePythonExecutor = ComponentRemotePythonExecutor(remoteScriptExecutionService)
             val bluePrintRuntime = mockk<DefaultBluePrintRuntimeService>("123456-1000")
 
+            every { bluePrintRuntime.getBluePrintError() } answers { BluePrintError() } //successful case.
             every { bluePrintRuntime.setNodeTemplateAttributeValue(any(), any(), any()) } answers {}
 
             val input = getMockedOutput(bluePrintRuntime)