aboutsummaryrefslogtreecommitdiffstats
path: root/ms
diff options
context:
space:
mode:
authorOleg Mitsura <oleg.mitsura@amdocs.com>2019-06-21 07:44:56 -0400
committerOleg Mitsura <omitsura@gmail.com>2019-06-25 17:35:57 +0000
commitc974b0293789c2b74d73dfb19e08fd233270f407 (patch)
treeb31c84989230c9681042917403643ce5e1a0efa3 /ms
parentb3eb3fc950e6cda05d85d8422863b81bd0e766dd (diff)
python-executor will provide output line-by-line
* python-executor STDOUT contains cleaner logs * return status has been reworked to return a list of lines Issue-ID: CCSDK-1404 Signed-off-by: Oleg Mitsura <oleg.mitsura@amdocs.com> Change-Id: Ib17495e92a136ff4f950bb6e049246a9e100eb7d
Diffstat (limited to 'ms')
-rw-r--r--ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt7
-rw-r--r--ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt6
-rw-r--r--ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt4
-rw-r--r--ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt4
-rw-r--r--ms/command-executor/src/main/python/command_executor_handler.py17
-rw-r--r--ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py7
-rw-r--r--ms/command-executor/src/main/python/utils.py4
7 files changed, 28 insertions, 21 deletions
diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
index 4ef1cfbb3..2de22424e 100644
--- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
+++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
@@ -22,6 +22,7 @@ import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.ExecutionServic
import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService
import org.onap.ccsdk.cds.controllerblueprints.core.*
import org.onap.ccsdk.cds.controllerblueprints.core.data.OperationAssignment
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.config.ConfigurableBeanFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean
@@ -96,7 +97,7 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
)
val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput)
log.info("$ATTRIBUTE_PREPARE_ENV_LOG - ${prepareEnvOutput.response}")
- setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, prepareEnvOutput.response.asJsonPrimitive())
+ setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, JacksonUtils.jsonNodeFromObject(prepareEnvOutput.response))
setAttribute(ATTRIBUTE_EXEC_CMD_LOG, "N/A".asJsonPrimitive())
check(prepareEnvOutput.status == StatusType.SUCCESS) {
"failed to get prepare remote env response status for requestId(${prepareEnvInput.requestId})"
@@ -112,7 +113,7 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
properties = properties)
val remoteExecutionOutput = remoteScriptExecutionService.executeCommand(remoteExecutionInput)
log.info("$ATTRIBUTE_EXEC_CMD_LOG - ${remoteExecutionOutput.response}")
- setAttribute(ATTRIBUTE_EXEC_CMD_LOG, remoteExecutionOutput.response.asJsonPrimitive())
+ setAttribute(ATTRIBUTE_EXEC_CMD_LOG, JacksonUtils.jsonNodeFromObject(remoteExecutionOutput.response))
check(remoteExecutionOutput.status == StatusType.SUCCESS) {
"failed to get prepare remote command response status for requestId(${remoteExecutionOutput.requestId})"
}
@@ -128,4 +129,4 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
bluePrintRuntimeService.getBluePrintError()
.addError("Failed in ComponentJythonExecutor : ${runtimeException.message}")
}
-} \ No newline at end of file
+}
diff --git a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
index 896d9a648..166d7b136 100644
--- a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
+++ b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
@@ -189,7 +189,7 @@ class MockRemoteScriptExecutionService : RemoteScriptExecutionService {
assertNotNull(prepareEnvInput.packages, "failed to get packages")
val remoteScriptExecutionOutput = mockk<RemoteScriptExecutionOutput>()
- every { remoteScriptExecutionOutput.response } returns "prepared successfully"
+ every { remoteScriptExecutionOutput.response } returns listOf("prepared successfully")
every { remoteScriptExecutionOutput.status } returns StatusType.SUCCESS
return remoteScriptExecutionOutput
}
@@ -198,7 +198,7 @@ class MockRemoteScriptExecutionService : RemoteScriptExecutionService {
assertEquals(remoteExecutionInput.requestId, "123456-1000", "failed to match request id")
val remoteScriptExecutionOutput = mockk<RemoteScriptExecutionOutput>()
- every { remoteScriptExecutionOutput.response } returns "processed successfully"
+ every { remoteScriptExecutionOutput.response } returns listOf("processed successfully")
every { remoteScriptExecutionOutput.status } returns StatusType.SUCCESS
return remoteScriptExecutionOutput
}
@@ -206,4 +206,4 @@ class MockRemoteScriptExecutionService : RemoteScriptExecutionService {
override suspend fun close() {
}
-} \ No newline at end of file
+}
diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt b/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt
index 83254cecc..2f9ea4a25 100644
--- a/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt
+++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt
@@ -37,7 +37,7 @@ data class RemoteScriptExecutionInput(var requestId: String,
data class RemoteScriptExecutionOutput(var requestId: String,
- var response: String,
+ var response: List<String>,
var status: StatusType = StatusType.SUCCESS,
var timestamp: Date = Date())
@@ -46,4 +46,4 @@ data class PrepareRemoteEnvInput(var requestId: String,
var remoteIdentifier: RemoteIdentifier? = null,
var packages: JsonNode,
var timeOut: Long = 120,
- var properties: MutableMap<String, JsonNode> = hashMapOf()) \ No newline at end of file
+ var properties: MutableMap<String, JsonNode> = hashMapOf())
diff --git a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
index 7aee95e11..3af57a22b 100644
--- a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
+++ b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
@@ -148,9 +148,9 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi
fun ExecutionOutput.asJavaData(): RemoteScriptExecutionOutput {
return RemoteScriptExecutionOutput(
requestId = this.requestId,
- response = this.response,
+ response = this.responseList,
status = StatusType.valueOf(this.status.name)
)
}
-} \ No newline at end of file
+}
diff --git a/ms/command-executor/src/main/python/command_executor_handler.py b/ms/command-executor/src/main/python/command_executor_handler.py
index a5951fdb3..365c00188 100644
--- a/ms/command-executor/src/main/python/command_executor_handler.py
+++ b/ms/command-executor/src/main/python/command_executor_handler.py
@@ -19,6 +19,7 @@ from subprocess import CalledProcessError, PIPE
import logging
import os
import subprocess
+import sys
import virtualenv
import venv
import utils
@@ -37,10 +38,7 @@ class CommandExecutorHandler():
self.installed = self.venv_home + '/.installed'
def is_installed(self):
- if os.path.exists(self.installed):
- return True
- else:
- return False
+ return os.path.exists(self.installed)
def prepare_env(self, request, results):
if not self.is_installed():
@@ -78,7 +76,16 @@ class CommandExecutorHandler():
self.logger.info("Command: {}".format(cmd))
try:
- results.append(os.popen(cmd).read())
+ with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ shell=True, bufsize=1, universal_newlines=True) as newProcess:
+ while True:
+ output = newProcess.stdout.readline()
+ if output == '' and newProcess.poll() is not None:
+ break
+ if output:
+ self.logger.info(output.strip())
+ results.append(output.strip())
+ rc = newProcess.poll()
except Exception as e:
self.logger.info("{} - Failed to execute command. Error: {}".format(self.blueprint_id, e))
results.append(e)
diff --git a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
index 3afeb35fc..478e00959 100644
--- a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
+++ b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: CommandExecutor.proto
@@ -22,7 +23,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
package='org.onap.ccsdk.cds.controllerblueprints.command.api',
syntax='proto3',
serialized_options=_b('P\001'),
- serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xba\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x01(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01**\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3')
+ serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xba\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x03(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01**\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3')
,
dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,])
@@ -279,8 +280,8 @@ _EXECUTIONOUTPUT = _descriptor.Descriptor(
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='response', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput.response', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
+ number=2, type=9, cpp_type=9, label=3,
+ has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
diff --git a/ms/command-executor/src/main/python/utils.py b/ms/command-executor/src/main/python/utils.py
index 6260997f3..dc5d0089f 100644
--- a/ms/command-executor/src/main/python/utils.py
+++ b/ms/command-executor/src/main/python/utils.py
@@ -17,7 +17,6 @@ from google.protobuf.timestamp_pb2 import Timestamp
import proto.CommandExecutor_pb2 as CommandExecutor_pb2
-
def get_blueprint_id(request):
blueprint_name = request.identifiers.blueprintName
blueprint_version = request.identifiers.blueprintVersion
@@ -32,6 +31,5 @@ def build_response(request, results, is_success=True):
timestamp = Timestamp()
timestamp.GetCurrentTime()
-
- return CommandExecutor_pb2.ExecutionOutput(requestId=request.requestId, response="".join(results), status=status,
+ return CommandExecutor_pb2.ExecutionOutput(requestId=request.requestId, response=results, status=status,
timestamp=timestamp)