diff options
author | 2020-01-24 17:56:36 -0500 | |
---|---|---|
committer | 2020-01-27 11:56:31 -0500 | |
commit | 9f92c18e8cccc5f15a7c65ef3e95252316360077 (patch) | |
tree | eabf4b603706c5a3acdfffb669ce0ddc1db4791b /ms | |
parent | 87b00f3a63482fe21ab468541d6f89b8348422a0 (diff) |
cmd-exec separate env-prepare and exec timeouts.
Issue-ID: CCSDK-2039
added inputs for cmd-exec "env-prepare-timeout" and "execution-timeout" to segregate the timeouts.
rev1: initial commit
rev2: reverted application-dev.properties
rev3: apply default timeouts if they are not specified + fixed a test
Signed-off-by: Oleg Mitsura <oleg.mitsura@amdocs.com>
Change-Id: I651594932a3fc87e0d853e65879b1548d9dbde8a
Diffstat (limited to 'ms')
2 files changed, 51 insertions, 18 deletions
diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt index 6ad1e1f11..f3169d937 100644 --- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt +++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt @@ -46,14 +46,19 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic const val INPUT_ENDPOINT_SELECTOR = "endpoint-selector" const val INPUT_DYNAMIC_PROPERTIES = "dynamic-properties" const val INPUT_ARGUMENT_PROPERTIES = "argument-properties" + const val INPUT_COMMAND = "command" const val INPUT_PACKAGES = "packages" const val DEFAULT_SELECTOR = "remote-python" + const val INPUT_ENV_PREPARE_TIMEOUT = "env-prepare-timeout" + const val INPUT_EXECUTE_TIMEOUT = "execution-timeout" const val ATTRIBUTE_EXEC_CMD_STATUS = "status" const val ATTRIBUTE_PREPARE_ENV_LOG = "prepare-environment-logs" const val ATTRIBUTE_EXEC_CMD_LOG = "execute-command-logs" const val ATTRIBUTE_RESPONSE_DATA = "response-data" + const val DEFAULT_ENV_PREPARE_TIMEOUT_IN_SEC = 120 + const val DEFAULT_EXECUTE_TIMEOUT_IN_SEC = 180 } override suspend fun processNB(executionRequest: ExecutionServiceInput) { @@ -90,6 +95,16 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic ?.rootFieldsToMap()?.toSortedMap()?.values?.joinToString(" ") { formatNestedJsonNode(it) } val command = getOperationInput(INPUT_COMMAND).asText() + + /** + * Timeouts that are specific to the command executor. + * Note: the interface->input->timeout is the component level timeout. + */ + val envPrepTimeout = getOptionalOperationInput(INPUT_ENV_PREPARE_TIMEOUT)?.asInt() + ?: DEFAULT_ENV_PREPARE_TIMEOUT_IN_SEC + val executionTimeout = getOptionalOperationInput(INPUT_EXECUTE_TIMEOUT)?.asInt() + ?: DEFAULT_EXECUTE_TIMEOUT_IN_SEC + var scriptCommand = command.replace(pythonScript.name, pythonScript.absolutePath) if (args != null && args.isNotEmpty()) { scriptCommand = scriptCommand.plus(" ").plus(args) @@ -110,7 +125,9 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic val prepareEnvInput = PrepareRemoteEnvInput(requestId = processId, remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion), - packages = packages + packages = packages, + timeOut = envPrepTimeout.toLong() + ) val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput) log.info("$ATTRIBUTE_PREPARE_ENV_LOG - ${prepareEnvOutput.response}") @@ -125,9 +142,22 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic setNodeOutputProperties(prepareEnvOutput.status.name.asJsonPrimitive(), logsEnv, "".asJsonPrimitive()) } } - - // if Env preparation was successful, then proceed with command execution in this Env - if (bluePrintRuntimeService.getBluePrintError().errors.isEmpty()) { + } catch (grpcEx: io.grpc.StatusRuntimeException) { + val timeoutErrMsg = "Command executor timed out in GRPC call during env. preparation.. timeout($envPrepTimeout) requestId ($processId)." + setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, timeoutErrMsg.asJsonPrimitive()) + setNodeOutputErrors(status = timeoutErrMsg, message = "${grpcEx.status}".asJsonPrimitive()) + log.error(timeoutErrMsg, grpcEx) + addError(timeoutErrMsg) + } catch (e: Exception) { + val timeoutErrMsg = "Command executor failed during env. preparation.. timeout($envPrepTimeout) requestId ($processId)." + setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, e.message.asJsonPrimitive()) + setNodeOutputErrors(status = timeoutErrMsg, message = "${e.message}".asJsonPrimitive()) + log.error("Failed to process on remote executor requestId ($processId)", e) + addError(timeoutErrMsg) + } + // if Env preparation was successful, then proceed with command execution in this Env + if (bluePrintRuntimeService.getBluePrintError().errors.isEmpty()) { + try { // Populate command execution properties and pass it to the remote server val properties = dynamicProperties?.returnNullIfMissing()?.rootFieldsToMap() ?: hashMapOf() @@ -136,14 +166,14 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion), command = scriptCommand, properties = properties, - timeOut = timeout.toLong()) + timeOut = executionTimeout.toLong()) val remoteExecutionOutputDeferred = GlobalScope.async { remoteScriptExecutionService.executeCommand(remoteExecutionInput) } - val remoteExecutionOutput = withTimeout(timeout * 1000L) { + val remoteExecutionOutput = withTimeout(executionTimeout * 1000L) { remoteExecutionOutputDeferred.await() } @@ -159,21 +189,23 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic setNodeOutputProperties(remoteExecutionOutput.status.name.asJsonPrimitive(), logs, remoteExecutionOutput.payload) } + } catch (timeoutEx: TimeoutCancellationException) { + val timeoutErrMsg = "Command executor timed out executing after $executionTimeout seconds requestId ($processId)" + setNodeOutputErrors(status = timeoutErrMsg, message = "".asJsonPrimitive()) + log.error(timeoutErrMsg, timeoutEx) + } catch (grpcEx: io.grpc.StatusRuntimeException) { + val timeoutErrMsg = "Command executor timed out executing after $executionTimeout seconds requestId ($processId)" + setNodeOutputErrors(status = timeoutErrMsg, message = "".asJsonPrimitive()) + log.error("Command executor time out during GRPC call", grpcEx) + } catch (e: Exception) { + log.error("Failed to process on remote executor requestId ($processId)", e) } - - } catch (timeoutEx: TimeoutCancellationException) { - setNodeOutputErrors(status = "Command executor timed out after $timeout seconds", message = "".asJsonPrimitive()) - log.error("Command executor timed out after $timeout seconds", timeoutEx) - } catch (grpcEx: io.grpc.StatusRuntimeException) { - setNodeOutputErrors(status = "Command executor timed out in GRPC call", message = "${grpcEx.status}".asJsonPrimitive()) - log.error("Command executor time out during GRPC call", grpcEx) - } catch (e: Exception) { - log.error("Failed to process on remote executor", e) - } finally { - remoteScriptExecutionService.close() } + log.debug("Trying to close GRPC channel. request ($processId)") + remoteScriptExecutionService.close() } + override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) { bluePrintRuntimeService.getBluePrintError() .addError("Failed in ComponentRemotePythonExecutor : ${runtimeException.message}") @@ -211,7 +243,6 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic log.info("Executor message : $message") setAttribute(ATTRIBUTE_RESPONSE_DATA, artifacts) log.info("Executor artifacts: $artifacts") - addError(status, ATTRIBUTE_EXEC_CMD_LOG, message.toString()) } } diff --git a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt index 89af42579..11ced624c 100644 --- a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt +++ b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt @@ -25,6 +25,7 @@ import org.junit.Test import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.* import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants +import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintError import org.onap.ccsdk.cds.controllerblueprints.core.asJsonPrimitive import org.onap.ccsdk.cds.controllerblueprints.core.asJsonType import org.onap.ccsdk.cds.controllerblueprints.core.putJsonElement @@ -81,6 +82,7 @@ class ComponentRemotePythonExecutorTest { val componentRemotePythonExecutor = ComponentRemotePythonExecutor(remoteScriptExecutionService) val bluePrintRuntime = mockk<DefaultBluePrintRuntimeService>("123456-1000") + every { bluePrintRuntime.getBluePrintError() } answers { BluePrintError() } //successful case. every { bluePrintRuntime.setNodeTemplateAttributeValue(any(), any(), any()) } answers {} val input = getMockedOutput(bluePrintRuntime) |