From 9f92c18e8cccc5f15a7c65ef3e95252316360077 Mon Sep 17 00:00:00 2001 From: Oleg Mitsura Date: Fri, 24 Jan 2020 17:56:36 -0500 Subject: cmd-exec separate env-prepare and exec timeouts. Issue-ID: CCSDK-2039 added inputs for cmd-exec "env-prepare-timeout" and "execution-timeout" to segregate the timeouts. rev1: initial commit rev2: reverted application-dev.properties rev3: apply default timeouts if they are not specified + fixed a test Signed-off-by: Oleg Mitsura Change-Id: I651594932a3fc87e0d853e65879b1548d9dbde8a --- .../component-remote-python-executor.json | 12 ++++ .../executor/ComponentRemotePythonExecutor.kt | 67 ++++++++++++++++------ .../executor/ComponentRemotePythonExecutorTest.kt | 2 + 3 files changed, 63 insertions(+), 18 deletions(-) diff --git a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json index acaea4e7b..8fa8e7c7b 100644 --- a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json +++ b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json @@ -60,6 +60,18 @@ "entry_schema" : { "type" : "dt-system-packages" } + }, + "env-prepare-timeout": { + "description": "Time-out for environment preparation.", + "required": false, + "type": "integer", + "default": 120 + }, + "execution-timeout": { + "description": "Time-out for the execution step.", + "required": false, + "type": "integer", + "default": 180 } } } diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt index 6ad1e1f11..f3169d937 100644 --- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt +++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt @@ -46,14 +46,19 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic const val INPUT_ENDPOINT_SELECTOR = "endpoint-selector" const val INPUT_DYNAMIC_PROPERTIES = "dynamic-properties" const val INPUT_ARGUMENT_PROPERTIES = "argument-properties" + const val INPUT_COMMAND = "command" const val INPUT_PACKAGES = "packages" const val DEFAULT_SELECTOR = "remote-python" + const val INPUT_ENV_PREPARE_TIMEOUT = "env-prepare-timeout" + const val INPUT_EXECUTE_TIMEOUT = "execution-timeout" const val ATTRIBUTE_EXEC_CMD_STATUS = "status" const val ATTRIBUTE_PREPARE_ENV_LOG = "prepare-environment-logs" const val ATTRIBUTE_EXEC_CMD_LOG = "execute-command-logs" const val ATTRIBUTE_RESPONSE_DATA = "response-data" + const val DEFAULT_ENV_PREPARE_TIMEOUT_IN_SEC = 120 + const val DEFAULT_EXECUTE_TIMEOUT_IN_SEC = 180 } override suspend fun processNB(executionRequest: ExecutionServiceInput) { @@ -90,6 +95,16 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic ?.rootFieldsToMap()?.toSortedMap()?.values?.joinToString(" ") { formatNestedJsonNode(it) } val command = getOperationInput(INPUT_COMMAND).asText() + + /** + * Timeouts that are specific to the command executor. + * Note: the interface->input->timeout is the component level timeout. + */ + val envPrepTimeout = getOptionalOperationInput(INPUT_ENV_PREPARE_TIMEOUT)?.asInt() + ?: DEFAULT_ENV_PREPARE_TIMEOUT_IN_SEC + val executionTimeout = getOptionalOperationInput(INPUT_EXECUTE_TIMEOUT)?.asInt() + ?: DEFAULT_EXECUTE_TIMEOUT_IN_SEC + var scriptCommand = command.replace(pythonScript.name, pythonScript.absolutePath) if (args != null && args.isNotEmpty()) { scriptCommand = scriptCommand.plus(" ").plus(args) @@ -110,7 +125,9 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic val prepareEnvInput = PrepareRemoteEnvInput(requestId = processId, remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion), - packages = packages + packages = packages, + timeOut = envPrepTimeout.toLong() + ) val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput) log.info("$ATTRIBUTE_PREPARE_ENV_LOG - ${prepareEnvOutput.response}") @@ -125,9 +142,22 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic setNodeOutputProperties(prepareEnvOutput.status.name.asJsonPrimitive(), logsEnv, "".asJsonPrimitive()) } } - - // if Env preparation was successful, then proceed with command execution in this Env - if (bluePrintRuntimeService.getBluePrintError().errors.isEmpty()) { + } catch (grpcEx: io.grpc.StatusRuntimeException) { + val timeoutErrMsg = "Command executor timed out in GRPC call during env. preparation.. timeout($envPrepTimeout) requestId ($processId)." + setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, timeoutErrMsg.asJsonPrimitive()) + setNodeOutputErrors(status = timeoutErrMsg, message = "${grpcEx.status}".asJsonPrimitive()) + log.error(timeoutErrMsg, grpcEx) + addError(timeoutErrMsg) + } catch (e: Exception) { + val timeoutErrMsg = "Command executor failed during env. preparation.. timeout($envPrepTimeout) requestId ($processId)." + setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, e.message.asJsonPrimitive()) + setNodeOutputErrors(status = timeoutErrMsg, message = "${e.message}".asJsonPrimitive()) + log.error("Failed to process on remote executor requestId ($processId)", e) + addError(timeoutErrMsg) + } + // if Env preparation was successful, then proceed with command execution in this Env + if (bluePrintRuntimeService.getBluePrintError().errors.isEmpty()) { + try { // Populate command execution properties and pass it to the remote server val properties = dynamicProperties?.returnNullIfMissing()?.rootFieldsToMap() ?: hashMapOf() @@ -136,14 +166,14 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion), command = scriptCommand, properties = properties, - timeOut = timeout.toLong()) + timeOut = executionTimeout.toLong()) val remoteExecutionOutputDeferred = GlobalScope.async { remoteScriptExecutionService.executeCommand(remoteExecutionInput) } - val remoteExecutionOutput = withTimeout(timeout * 1000L) { + val remoteExecutionOutput = withTimeout(executionTimeout * 1000L) { remoteExecutionOutputDeferred.await() } @@ -159,21 +189,23 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic setNodeOutputProperties(remoteExecutionOutput.status.name.asJsonPrimitive(), logs, remoteExecutionOutput.payload) } + } catch (timeoutEx: TimeoutCancellationException) { + val timeoutErrMsg = "Command executor timed out executing after $executionTimeout seconds requestId ($processId)" + setNodeOutputErrors(status = timeoutErrMsg, message = "".asJsonPrimitive()) + log.error(timeoutErrMsg, timeoutEx) + } catch (grpcEx: io.grpc.StatusRuntimeException) { + val timeoutErrMsg = "Command executor timed out executing after $executionTimeout seconds requestId ($processId)" + setNodeOutputErrors(status = timeoutErrMsg, message = "".asJsonPrimitive()) + log.error("Command executor time out during GRPC call", grpcEx) + } catch (e: Exception) { + log.error("Failed to process on remote executor requestId ($processId)", e) } - - } catch (timeoutEx: TimeoutCancellationException) { - setNodeOutputErrors(status = "Command executor timed out after $timeout seconds", message = "".asJsonPrimitive()) - log.error("Command executor timed out after $timeout seconds", timeoutEx) - } catch (grpcEx: io.grpc.StatusRuntimeException) { - setNodeOutputErrors(status = "Command executor timed out in GRPC call", message = "${grpcEx.status}".asJsonPrimitive()) - log.error("Command executor time out during GRPC call", grpcEx) - } catch (e: Exception) { - log.error("Failed to process on remote executor", e) - } finally { - remoteScriptExecutionService.close() } + log.debug("Trying to close GRPC channel. request ($processId)") + remoteScriptExecutionService.close() } + override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) { bluePrintRuntimeService.getBluePrintError() .addError("Failed in ComponentRemotePythonExecutor : ${runtimeException.message}") @@ -211,7 +243,6 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic log.info("Executor message : $message") setAttribute(ATTRIBUTE_RESPONSE_DATA, artifacts) log.info("Executor artifacts: $artifacts") - addError(status, ATTRIBUTE_EXEC_CMD_LOG, message.toString()) } } diff --git a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt index 89af42579..11ced624c 100644 --- a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt +++ b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt @@ -25,6 +25,7 @@ import org.junit.Test import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.* import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants +import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintError import org.onap.ccsdk.cds.controllerblueprints.core.asJsonPrimitive import org.onap.ccsdk.cds.controllerblueprints.core.asJsonType import org.onap.ccsdk.cds.controllerblueprints.core.putJsonElement @@ -81,6 +82,7 @@ class ComponentRemotePythonExecutorTest { val componentRemotePythonExecutor = ComponentRemotePythonExecutor(remoteScriptExecutionService) val bluePrintRuntime = mockk("123456-1000") + every { bluePrintRuntime.getBluePrintError() } answers { BluePrintError() } //successful case. every { bluePrintRuntime.setNodeTemplateAttributeValue(any(), any(), any()) } answers {} val input = getMockedOutput(bluePrintRuntime) -- cgit 1.2.3-korg