summaryrefslogtreecommitdiffstats
path: root/ms
diff options
context:
space:
mode:
authorOleg Mitsura <oleg.mitsura@amdocs.com>2020-01-24 17:56:36 -0500
committerOleg Mitsura <oleg.mitsura@amdocs.com>2020-01-27 11:56:31 -0500
commit9f92c18e8cccc5f15a7c65ef3e95252316360077 (patch)
treeeabf4b603706c5a3acdfffb669ce0ddc1db4791b /ms
parent87b00f3a63482fe21ab468541d6f89b8348422a0 (diff)
cmd-exec separate env-prepare and exec timeouts.
Issue-ID: CCSDK-2039 added inputs for cmd-exec "env-prepare-timeout" and "execution-timeout" to segregate the timeouts. rev1: initial commit rev2: reverted application-dev.properties rev3: apply default timeouts if they are not specified + fixed a test Signed-off-by: Oleg Mitsura <oleg.mitsura@amdocs.com> Change-Id: I651594932a3fc87e0d853e65879b1548d9dbde8a
Diffstat (limited to 'ms')
-rw-r--r--ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt67
-rw-r--r--ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt2
2 files changed, 51 insertions, 18 deletions
diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
index 6ad1e1f11..f3169d937 100644
--- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
+++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
@@ -46,14 +46,19 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
const val INPUT_ENDPOINT_SELECTOR = "endpoint-selector"
const val INPUT_DYNAMIC_PROPERTIES = "dynamic-properties"
const val INPUT_ARGUMENT_PROPERTIES = "argument-properties"
+
const val INPUT_COMMAND = "command"
const val INPUT_PACKAGES = "packages"
const val DEFAULT_SELECTOR = "remote-python"
+ const val INPUT_ENV_PREPARE_TIMEOUT = "env-prepare-timeout"
+ const val INPUT_EXECUTE_TIMEOUT = "execution-timeout"
const val ATTRIBUTE_EXEC_CMD_STATUS = "status"
const val ATTRIBUTE_PREPARE_ENV_LOG = "prepare-environment-logs"
const val ATTRIBUTE_EXEC_CMD_LOG = "execute-command-logs"
const val ATTRIBUTE_RESPONSE_DATA = "response-data"
+ const val DEFAULT_ENV_PREPARE_TIMEOUT_IN_SEC = 120
+ const val DEFAULT_EXECUTE_TIMEOUT_IN_SEC = 180
}
override suspend fun processNB(executionRequest: ExecutionServiceInput) {
@@ -90,6 +95,16 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
?.rootFieldsToMap()?.toSortedMap()?.values?.joinToString(" ") { formatNestedJsonNode(it) }
val command = getOperationInput(INPUT_COMMAND).asText()
+
+ /**
+ * Timeouts that are specific to the command executor.
+ * Note: the interface->input->timeout is the component level timeout.
+ */
+ val envPrepTimeout = getOptionalOperationInput(INPUT_ENV_PREPARE_TIMEOUT)?.asInt()
+ ?: DEFAULT_ENV_PREPARE_TIMEOUT_IN_SEC
+ val executionTimeout = getOptionalOperationInput(INPUT_EXECUTE_TIMEOUT)?.asInt()
+ ?: DEFAULT_EXECUTE_TIMEOUT_IN_SEC
+
var scriptCommand = command.replace(pythonScript.name, pythonScript.absolutePath)
if (args != null && args.isNotEmpty()) {
scriptCommand = scriptCommand.plus(" ").plus(args)
@@ -110,7 +125,9 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
val prepareEnvInput = PrepareRemoteEnvInput(requestId = processId,
remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName,
blueprintVersion = blueprintVersion),
- packages = packages
+ packages = packages,
+ timeOut = envPrepTimeout.toLong()
+
)
val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput)
log.info("$ATTRIBUTE_PREPARE_ENV_LOG - ${prepareEnvOutput.response}")
@@ -125,9 +142,22 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
setNodeOutputProperties(prepareEnvOutput.status.name.asJsonPrimitive(), logsEnv, "".asJsonPrimitive())
}
}
-
- // if Env preparation was successful, then proceed with command execution in this Env
- if (bluePrintRuntimeService.getBluePrintError().errors.isEmpty()) {
+ } catch (grpcEx: io.grpc.StatusRuntimeException) {
+ val timeoutErrMsg = "Command executor timed out in GRPC call during env. preparation.. timeout($envPrepTimeout) requestId ($processId)."
+ setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, timeoutErrMsg.asJsonPrimitive())
+ setNodeOutputErrors(status = timeoutErrMsg, message = "${grpcEx.status}".asJsonPrimitive())
+ log.error(timeoutErrMsg, grpcEx)
+ addError(timeoutErrMsg)
+ } catch (e: Exception) {
+ val timeoutErrMsg = "Command executor failed during env. preparation.. timeout($envPrepTimeout) requestId ($processId)."
+ setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, e.message.asJsonPrimitive())
+ setNodeOutputErrors(status = timeoutErrMsg, message = "${e.message}".asJsonPrimitive())
+ log.error("Failed to process on remote executor requestId ($processId)", e)
+ addError(timeoutErrMsg)
+ }
+ // if Env preparation was successful, then proceed with command execution in this Env
+ if (bluePrintRuntimeService.getBluePrintError().errors.isEmpty()) {
+ try {
// Populate command execution properties and pass it to the remote server
val properties = dynamicProperties?.returnNullIfMissing()?.rootFieldsToMap() ?: hashMapOf()
@@ -136,14 +166,14 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion),
command = scriptCommand,
properties = properties,
- timeOut = timeout.toLong())
+ timeOut = executionTimeout.toLong())
val remoteExecutionOutputDeferred = GlobalScope.async {
remoteScriptExecutionService.executeCommand(remoteExecutionInput)
}
- val remoteExecutionOutput = withTimeout(timeout * 1000L) {
+ val remoteExecutionOutput = withTimeout(executionTimeout * 1000L) {
remoteExecutionOutputDeferred.await()
}
@@ -159,21 +189,23 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
setNodeOutputProperties(remoteExecutionOutput.status.name.asJsonPrimitive(), logs,
remoteExecutionOutput.payload)
}
+ } catch (timeoutEx: TimeoutCancellationException) {
+ val timeoutErrMsg = "Command executor timed out executing after $executionTimeout seconds requestId ($processId)"
+ setNodeOutputErrors(status = timeoutErrMsg, message = "".asJsonPrimitive())
+ log.error(timeoutErrMsg, timeoutEx)
+ } catch (grpcEx: io.grpc.StatusRuntimeException) {
+ val timeoutErrMsg = "Command executor timed out executing after $executionTimeout seconds requestId ($processId)"
+ setNodeOutputErrors(status = timeoutErrMsg, message = "".asJsonPrimitive())
+ log.error("Command executor time out during GRPC call", grpcEx)
+ } catch (e: Exception) {
+ log.error("Failed to process on remote executor requestId ($processId)", e)
}
-
- } catch (timeoutEx: TimeoutCancellationException) {
- setNodeOutputErrors(status = "Command executor timed out after $timeout seconds", message = "".asJsonPrimitive())
- log.error("Command executor timed out after $timeout seconds", timeoutEx)
- } catch (grpcEx: io.grpc.StatusRuntimeException) {
- setNodeOutputErrors(status = "Command executor timed out in GRPC call", message = "${grpcEx.status}".asJsonPrimitive())
- log.error("Command executor time out during GRPC call", grpcEx)
- } catch (e: Exception) {
- log.error("Failed to process on remote executor", e)
- } finally {
- remoteScriptExecutionService.close()
}
+ log.debug("Trying to close GRPC channel. request ($processId)")
+ remoteScriptExecutionService.close()
}
+
override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
bluePrintRuntimeService.getBluePrintError()
.addError("Failed in ComponentRemotePythonExecutor : ${runtimeException.message}")
@@ -211,7 +243,6 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
log.info("Executor message : $message")
setAttribute(ATTRIBUTE_RESPONSE_DATA, artifacts)
log.info("Executor artifacts: $artifacts")
-
addError(status, ATTRIBUTE_EXEC_CMD_LOG, message.toString())
}
}
diff --git a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
index 89af42579..11ced624c 100644
--- a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
+++ b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
@@ -25,6 +25,7 @@ import org.junit.Test
import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.*
import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService
import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
+import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintError
import org.onap.ccsdk.cds.controllerblueprints.core.asJsonPrimitive
import org.onap.ccsdk.cds.controllerblueprints.core.asJsonType
import org.onap.ccsdk.cds.controllerblueprints.core.putJsonElement
@@ -81,6 +82,7 @@ class ComponentRemotePythonExecutorTest {
val componentRemotePythonExecutor = ComponentRemotePythonExecutor(remoteScriptExecutionService)
val bluePrintRuntime = mockk<DefaultBluePrintRuntimeService>("123456-1000")
+ every { bluePrintRuntime.getBluePrintError() } answers { BluePrintError() } //successful case.
every { bluePrintRuntime.setNodeTemplateAttributeValue(any(), any(), any()) } answers {}
val input = getMockedOutput(bluePrintRuntime)