python-executor will provide output line-by-line

* python-executor STDOUT contains cleaner logs
* return status has been reworked to return a list
of lines

Issue-ID: CCSDK-1404

Signed-off-by: Oleg Mitsura <oleg.mitsura@amdocs.com>
Change-Id: Ib17495e92a136ff4f950bb6e049246a9e100eb7d

10 files changed, 41 insertions, 28 deletions

diff --git a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json
index ddbccac93..4945da889 100644
--- a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json
+++ b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json
@@ -8,9 +8,12 @@
           "required" : false,
           "type" : "string"
         },
-        "execute-command-logs" : {
-          "required" : false,
-          "type" : "string"
+        "execute-command-logs": {
+          "required": false,
+          "type": "list",
+          "entry_schema": {
+            "type": "string"
+          }
         }
       },
       "capabilities" : {
@@ -179,4 +182,4 @@
       "derived_from" : "tosca.nodes.Root"
     }
   }
-}
\ No newline at end of file
+}
diff --git a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
index e14b63e58..caab5f7ac 100644
--- a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
+++ b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
@@ -8,7 +8,10 @@
     },
     "execute-command-logs": {
       "required": false,
-      "type": "string"
+      "type": "list",
+      "entry_schema": {
+        "type": "string"
+      }
     }
   },
   "capabilities": {
@@ -56,4 +59,4 @@
     }
   },
   "derived_from": "tosca.nodes.Component"
-}
\ No newline at end of file
+}
diff --git a/components/model-catalog/proto-definition/proto/CommandExecutor.proto b/components/model-catalog/proto-definition/proto/CommandExecutor.proto
index bc175dbc1..fd2d4f305 100644
--- a/components/model-catalog/proto-definition/proto/CommandExecutor.proto
+++ b/components/model-catalog/proto-definition/proto/CommandExecutor.proto
@@ -37,7 +37,7 @@ message Identifiers {
 
 message ExecutionOutput {
     string requestId = 1;
-    string response = 2;
+    repeated string response = 2;
     ResponseStatus status = 3;
     google.protobuf.Timestamp timestamp = 4;
 }
diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
index 4ef1cfbb3..2de22424e 100644
--- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
+++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
@@ -22,6 +22,7 @@ import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.ExecutionServic
 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService
 import org.onap.ccsdk.cds.controllerblueprints.core.*
 import org.onap.ccsdk.cds.controllerblueprints.core.data.OperationAssignment
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.config.ConfigurableBeanFactory
 import org.springframework.boot.autoconfigure.condition.ConditionalOnBean
@@ -96,7 +97,7 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
                 )
                 val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput)
                 log.info("$ATTRIBUTE_PREPARE_ENV_LOG - ${prepareEnvOutput.response}")
-                setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, prepareEnvOutput.response.asJsonPrimitive())
+                setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, JacksonUtils.jsonNodeFromObject(prepareEnvOutput.response))
                 setAttribute(ATTRIBUTE_EXEC_CMD_LOG, "N/A".asJsonPrimitive())
                 check(prepareEnvOutput.status == StatusType.SUCCESS) {
                     "failed to get prepare remote env response status for requestId(${prepareEnvInput.requestId})"
@@ -112,7 +113,7 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
                     properties = properties)
             val remoteExecutionOutput = remoteScriptExecutionService.executeCommand(remoteExecutionInput)
             log.info("$ATTRIBUTE_EXEC_CMD_LOG  - ${remoteExecutionOutput.response}")
-            setAttribute(ATTRIBUTE_EXEC_CMD_LOG, remoteExecutionOutput.response.asJsonPrimitive())
+            setAttribute(ATTRIBUTE_EXEC_CMD_LOG, JacksonUtils.jsonNodeFromObject(remoteExecutionOutput.response))
             check(remoteExecutionOutput.status == StatusType.SUCCESS) {
                 "failed to get prepare remote command response status for requestId(${remoteExecutionOutput.requestId})"
             }
@@ -128,4 +129,4 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
         bluePrintRuntimeService.getBluePrintError()
                 .addError("Failed in ComponentJythonExecutor : ${runtimeException.message}")
     }
-}
\ No newline at end of file
+}
diff --git a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
index 896d9a648..166d7b136 100644
--- a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
+++ b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt
@@ -189,7 +189,7 @@ class MockRemoteScriptExecutionService : RemoteScriptExecutionService {
         assertNotNull(prepareEnvInput.packages, "failed to get packages")
 
         val remoteScriptExecutionOutput = mockk<RemoteScriptExecutionOutput>()
-        every { remoteScriptExecutionOutput.response } returns "prepared successfully"
+        every { remoteScriptExecutionOutput.response } returns listOf("prepared successfully")
         every { remoteScriptExecutionOutput.status } returns StatusType.SUCCESS
         return remoteScriptExecutionOutput
     }
@@ -198,7 +198,7 @@ class MockRemoteScriptExecutionService : RemoteScriptExecutionService {
         assertEquals(remoteExecutionInput.requestId, "123456-1000", "failed to match request id")
 
         val remoteScriptExecutionOutput = mockk<RemoteScriptExecutionOutput>()
-        every { remoteScriptExecutionOutput.response } returns "processed successfully"
+        every { remoteScriptExecutionOutput.response } returns listOf("processed successfully")
         every { remoteScriptExecutionOutput.status } returns StatusType.SUCCESS
         return remoteScriptExecutionOutput
     }
@@ -206,4 +206,4 @@ class MockRemoteScriptExecutionService : RemoteScriptExecutionService {
     override suspend fun close() {
 
     }
-}
\ No newline at end of file
+}
diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt b/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt
index 83254cecc..2f9ea4a25 100644
--- a/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt
+++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt
@@ -37,7 +37,7 @@ data class RemoteScriptExecutionInput(var requestId: String,
 
 
 data class RemoteScriptExecutionOutput(var requestId: String,
-                                       var response: String,
+                                       var response: List<String>,
                                        var status: StatusType = StatusType.SUCCESS,
                                        var timestamp: Date = Date())
 
@@ -46,4 +46,4 @@ data class PrepareRemoteEnvInput(var requestId: String,
                                  var remoteIdentifier: RemoteIdentifier? = null,
                                  var packages: JsonNode,
                                  var timeOut: Long = 120,
-                                 var properties: MutableMap<String, JsonNode> = hashMapOf())
\ No newline at end of file
+                                 var properties: MutableMap<String, JsonNode> = hashMapOf())
diff --git a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
index 7aee95e11..3af57a22b 100644
--- a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
+++ b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
@@ -148,9 +148,9 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi
     fun ExecutionOutput.asJavaData(): RemoteScriptExecutionOutput {
         return RemoteScriptExecutionOutput(
             requestId = this.requestId,
-            response = this.response,
+            response = this.responseList,
             status = StatusType.valueOf(this.status.name)
         )
     }
 
-}
\ No newline at end of file
+}
diff --git a/ms/command-executor/src/main/python/command_executor_handler.py b/ms/command-executor/src/main/python/command_executor_handler.py
index a5951fdb3..365c00188 100644
--- a/ms/command-executor/src/main/python/command_executor_handler.py
+++ b/ms/command-executor/src/main/python/command_executor_handler.py
@@ -19,6 +19,7 @@ from subprocess import CalledProcessError, PIPE
 import logging
 import os
 import subprocess
+import sys
 import virtualenv
 import venv
 import utils
@@ -37,10 +38,7 @@ class CommandExecutorHandler():
         self.installed = self.venv_home + '/.installed'
 
     def is_installed(self):
-        if os.path.exists(self.installed):
-            return True
-        else:
-            return False
+        return os.path.exists(self.installed)
 
     def prepare_env(self, request, results):
         if not self.is_installed():
@@ -78,7 +76,16 @@ class CommandExecutorHandler():
 
         self.logger.info("Command: {}".format(cmd))
         try:
-            results.append(os.popen(cmd).read())
+            with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                  shell=True, bufsize=1, universal_newlines=True) as newProcess:
+                while True:
+                    output = newProcess.stdout.readline()
+                    if output == '' and newProcess.poll() is not None:
+                        break
+                    if output:
+                        self.logger.info(output.strip())
+                        results.append(output.strip())
+                    rc = newProcess.poll()
         except Exception as e:
             self.logger.info("{} - Failed to execute command. Error: {}".format(self.blueprint_id, e))
             results.append(e)
diff --git a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
index 3afeb35fc..478e00959 100644
--- a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
+++ b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: CommandExecutor.proto
 
@@ -22,7 +23,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   package='org.onap.ccsdk.cds.controllerblueprints.command.api',
   syntax='proto3',
   serialized_options=_b('P\001'),
-  serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xba\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x01(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01**\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3')
+  serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xba\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x03(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01**\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3')
   ,
   dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,])
 
@@ -279,8 +280,8 @@ _EXECUTIONOUTPUT = _descriptor.Descriptor(
       serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='response', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput.response', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
+      number=2, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
diff --git a/ms/command-executor/src/main/python/utils.py b/ms/command-executor/src/main/python/utils.py
index 6260997f3..dc5d0089f 100644
--- a/ms/command-executor/src/main/python/utils.py
+++ b/ms/command-executor/src/main/python/utils.py
@@ -17,7 +17,6 @@ from google.protobuf.timestamp_pb2 import Timestamp
 
 import proto.CommandExecutor_pb2 as CommandExecutor_pb2
 
-
 def get_blueprint_id(request):
     blueprint_name = request.identifiers.blueprintName
     blueprint_version = request.identifiers.blueprintVersion
@@ -32,6 +31,5 @@ def build_response(request, results, is_success=True):
 
     timestamp = Timestamp()
     timestamp.GetCurrentTime()
-
-    return CommandExecutor_pb2.ExecutionOutput(requestId=request.requestId, response="".join(results), status=status,
+    return CommandExecutor_pb2.ExecutionOutput(requestId=request.requestId, response=results, status=status,
                                                timestamp=timestamp)