Improve Remote Python Executor error handling and allow for structured response

Issue-ID: CCSDK-1855

Signed-off-by: Serge Simard <serge@agilitae.com>
Change-Id: I924918fb06a5a0f118772dedaa41c536c7eca9e7
Signed-off-by: Serge Simard <serge@agilitae.com>

6 files changed, 109 insertions, 48 deletions

diff --git a/ms/command-executor/src/main/python/cds_utils/__init__.py b/ms/command-executor/src/main/python/cds_utils/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/ms/command-executor/src/main/python/cds_utils/__init__.py
diff --git a/ms/command-executor/src/main/python/cds_utils/payload_coder.py b/ms/command-executor/src/main/python/cds_utils/payload_coder.py
new file mode 100644
index 000000000..951262172
--- /dev/null
+++ b/ms/command-executor/src/main/python/cds_utils/payload_coder.py
@@ -0,0 +1,13 @@
+import json

+from email.mime import multipart

+from email.mime import text

+import email.parser

+

+def send_response_data_payload(json_payload):

+    m = multipart.MIMEMultipart("form-data")

+    data = text.MIMEText("response_payload", "json", "utf8")

+    data.set_payload(json.JSONEncoder().encode(json_payload))

+    m.attach(data)

+    print("BEGIN_EXTRA_PAYLOAD")

+    print(m.as_string())

+    print("END_EXTRA_PAYLOAD")
\ No newline at end of file
diff --git a/ms/command-executor/src/main/python/command_executor_handler.py b/ms/command-executor/src/main/python/command_executor_handler.py
index 972dad627..c920dda89 100644
--- a/ms/command-executor/src/main/python/command_executor_handler.py
+++ b/ms/command-executor/src/main/python/command_executor_handler.py
@@ -25,6 +25,8 @@ import virtualenv
 import venv
 import utils
 import proto.CommandExecutor_pb2 as CommandExecutor_pb2
+import email.parser
+import json
 
 REQUIREMENTS_TXT = "requirements.txt"
 
@@ -75,6 +77,10 @@ class CommandExecutorHandler():
         else:
             cmd = cmd + "; " + request.command + " " + re.escape(MessageToJson(request.properties))
 
+        payload_result = {}
+        payload_section = []
+        is_payload_section = False
+
         try:
             with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                                   shell=True, bufsize=1, universal_newlines=True) as newProcess:
@@ -82,19 +88,36 @@ class CommandExecutorHandler():
                     output = newProcess.stdout.readline()
                     if output == '' and newProcess.poll() is not None:
                         break
-                    if output:
+                    if output.startswith('BEGIN_EXTRA_PAYLOAD'):
+                        is_payload_section = True
+                        output = newProcess.stdout.readline()
+                    if output.startswith('END_EXTRA_PAYLOAD'):
+                        is_payload_section = False
+                        output = ''
+                        payload = '\n'.join(payload_section)
+                        msg = email.parser.Parser().parsestr(payload)
+                        for part in msg.get_payload():
+                            payload_result = json.loads(part.get_payload())
+                    if output and not is_payload_section:
                         self.logger.info(output.strip())
                         results.append(output.strip())
-                    rc = newProcess.poll()
+                    else:
+                        payload_section.append(output.strip())
+                rc = newProcess.poll()
         except Exception as e:
             self.logger.info("{} - Failed to execute command. Error: {}".format(self.blueprint_id, e))
             results.append(e)
-            return False
+            payload_result["cds_return_code"] = False
+            return payload_result
 
         # deactivate_venv(blueprint_id)
-        return True
+
+        payload_result["cds_return_code"] = rc
+        return payload_result
 
     def install_packages(self, request, type, f, results):
+        success = self.install_python_packages('UTILITY', results)
+
         for package in request.packages:
             if package.type == type:
                 f.write("Installed %s packages:\r\n" % CommandExecutor_pb2.PackageType.Name(type))
@@ -116,6 +139,8 @@ class CommandExecutorHandler():
 
         if REQUIREMENTS_TXT == package:
             command = ["pip", "install", "-r", self.venv_home + "/Environments/" + REQUIREMENTS_TXT]
+        elif package == 'UTILITY':
+            command = ["cp", "-r", "./cds_utils", self.venv_home + "/lib/python3.6/site-packages/"]
         else:
             command = ["pip", "install", package]
 
diff --git a/ms/command-executor/src/main/python/command_executor_server.py b/ms/command-executor/src/main/python/command_executor_server.py
index 6266141d9..577c8a0ca 100644
--- a/ms/command-executor/src/main/python/command_executor_server.py
+++ b/ms/command-executor/src/main/python/command_executor_server.py
@@ -16,7 +16,7 @@
 # limitations under the License.
 #
 import logging
-
+import os, sys
 import proto.CommandExecutor_pb2_grpc as CommandExecutor_pb2_grpc
 
 from command_executor_handler import CommandExecutorHandler
@@ -39,19 +39,26 @@ class CommandExecutorServer(CommandExecutor_pb2_grpc.CommandExecutorServiceServi
         handler = CommandExecutorHandler(request)
         if not handler.prepare_env(request, results):
             self.logger.info("{} - Failed to prepare python environment. {}".format(blueprint_id, results))
-            return utils.build_response(request, results, False)
+            return utils.build_response(request, results, {}, False)
         self.logger.info("{} - Package installation logs {}".format(blueprint_id, results))
-        return utils.build_response(request, results)
+        return utils.build_response(request, results, {}, True)
 
     def executeCommand(self, request, context):
         blueprint_id = utils.get_blueprint_id(request)
         self.logger.info("{} - Received executeCommand request".format(blueprint_id))
-        self.logger.info(request)
+        if os.environ.get('CE_DEBUG','false') == "true":
+            self.logger.info(request)
 
-        results = []
+        log_results = []
+        payload_result = {}
         handler = CommandExecutorHandler(request)
-        if not handler.execute_command(request, results):
-            self.logger.info("{} - Failed to executeCommand. {}".format(blueprint_id, results))
-            return utils.build_response(request, results, False)
-        self.logger.info("{} - Execution finished successfully.".format(blueprint_id))
-        return utils.build_response(request, results)
+        payload_result = handler.execute_command(request, log_results)
+        if not payload_result["cds_return_code"]:
+            self.logger.info("{} - Failed to executeCommand. {}".format(blueprint_id, log_results))
+        else:
+            self.logger.info("{} - Execution finished successfully.".format(blueprint_id))
+
+        ret = utils.build_response(request, log_results, payload_result, payload_result["cds_return_code"])
+        self.logger.info("Payload returned %s" % payload_result)
+
+        return ret
\ No newline at end of file
diff --git a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
index 478e00959..4edfc6cd5 100644
--- a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
+++ b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
@@ -23,7 +23,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   package='org.onap.ccsdk.cds.controllerblueprints.command.api',
   syntax='proto3',
   serialized_options=_b('P\001'),
-  serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xba\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x03(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01**\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3')
+  serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xcb\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x03(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07payload\x18\x05 \x01(\t\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01*9\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x12\r\n\tutilities\x10\x02\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3')
   ,
   dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,])
 
@@ -44,8 +44,8 @@ _RESPONSESTATUS = _descriptor.EnumDescriptor(
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=1116,
-  serialized_end=1158,
+  serialized_start=1133,
+  serialized_end=1175,
 )
 _sym_db.RegisterEnumDescriptor(_RESPONSESTATUS)
 
@@ -64,11 +64,15 @@ _PACKAGETYPE = _descriptor.EnumDescriptor(
       name='ansible_galaxy', index=1, number=1,
       serialized_options=None,
       type=None),
+    _descriptor.EnumValueDescriptor(
+      name='utilities', index=2, number=2,
+      serialized_options=None,
+      type=None),
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=1160,
-  serialized_end=1202,
+  serialized_start=1177,
+  serialized_end=1234,
 )
 _sym_db.RegisterEnumDescriptor(_PACKAGETYPE)
 
@@ -77,6 +81,7 @@ SUCCESS = 0
 FAILURE = 1
 pip = 0
 ansible_galaxy = 1
+utilities = 2
 
 
 
@@ -299,6 +304,13 @@ _EXECUTIONOUTPUT = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='payload', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput.payload', index=4,
+      number=5, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -312,7 +324,7 @@ _EXECUTIONOUTPUT = _descriptor.Descriptor(
   oneofs=[
   ],
   serialized_start=819,
-  serialized_end=1005,
+  serialized_end=1022,
 )
 
 
@@ -349,8 +361,8 @@ _PACKAGES = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=1007,
-  serialized_end=1114,
+  serialized_start=1024,
+  serialized_end=1131,
 )
 
 _EXECUTIONINPUT.fields_by_name['identifiers'].message_type = _IDENTIFIERS
@@ -372,39 +384,39 @@ DESCRIPTOR.enum_types_by_name['ResponseStatus'] = _RESPONSESTATUS
 DESCRIPTOR.enum_types_by_name['PackageType'] = _PACKAGETYPE
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
-ExecutionInput = _reflection.GeneratedProtocolMessageType('ExecutionInput', (_message.Message,), dict(
-  DESCRIPTOR = _EXECUTIONINPUT,
-  __module__ = 'CommandExecutor_pb2'
+ExecutionInput = _reflection.GeneratedProtocolMessageType('ExecutionInput', (_message.Message,), {
+  'DESCRIPTOR' : _EXECUTIONINPUT,
+  '__module__' : 'CommandExecutor_pb2'
   # @@protoc_insertion_point(class_scope:org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput)
-  ))
+  })
 _sym_db.RegisterMessage(ExecutionInput)
 
-PrepareEnvInput = _reflection.GeneratedProtocolMessageType('PrepareEnvInput', (_message.Message,), dict(
-  DESCRIPTOR = _PREPAREENVINPUT,
-  __module__ = 'CommandExecutor_pb2'
+PrepareEnvInput = _reflection.GeneratedProtocolMessageType('PrepareEnvInput', (_message.Message,), {
+  'DESCRIPTOR' : _PREPAREENVINPUT,
+  '__module__' : 'CommandExecutor_pb2'
   # @@protoc_insertion_point(class_scope:org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput)
-  ))
+  })
 _sym_db.RegisterMessage(PrepareEnvInput)
 
-Identifiers = _reflection.GeneratedProtocolMessageType('Identifiers', (_message.Message,), dict(
-  DESCRIPTOR = _IDENTIFIERS,
-  __module__ = 'CommandExecutor_pb2'
+Identifiers = _reflection.GeneratedProtocolMessageType('Identifiers', (_message.Message,), {
+  'DESCRIPTOR' : _IDENTIFIERS,
+  '__module__' : 'CommandExecutor_pb2'
   # @@protoc_insertion_point(class_scope:org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers)
-  ))
+  })
 _sym_db.RegisterMessage(Identifiers)
 
-ExecutionOutput = _reflection.GeneratedProtocolMessageType('ExecutionOutput', (_message.Message,), dict(
-  DESCRIPTOR = _EXECUTIONOUTPUT,
-  __module__ = 'CommandExecutor_pb2'
+ExecutionOutput = _reflection.GeneratedProtocolMessageType('ExecutionOutput', (_message.Message,), {
+  'DESCRIPTOR' : _EXECUTIONOUTPUT,
+  '__module__' : 'CommandExecutor_pb2'
   # @@protoc_insertion_point(class_scope:org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput)
-  ))
+  })
 _sym_db.RegisterMessage(ExecutionOutput)
 
-Packages = _reflection.GeneratedProtocolMessageType('Packages', (_message.Message,), dict(
-  DESCRIPTOR = _PACKAGES,
-  __module__ = 'CommandExecutor_pb2'
+Packages = _reflection.GeneratedProtocolMessageType('Packages', (_message.Message,), {
+  'DESCRIPTOR' : _PACKAGES,
+  '__module__' : 'CommandExecutor_pb2'
   # @@protoc_insertion_point(class_scope:org.onap.ccsdk.cds.controllerblueprints.command.api.Packages)
-  ))
+  })
 _sym_db.RegisterMessage(Packages)
 
 
@@ -416,8 +428,8 @@ _COMMANDEXECUTORSERVICE = _descriptor.ServiceDescriptor(
   file=DESCRIPTOR,
   index=0,
   serialized_options=None,
-  serialized_start=1205,
-  serialized_end=1542,
+  serialized_start=1237,
+  serialized_end=1574,
   methods=[
   _descriptor.MethodDescriptor(
     name='prepareEnv',
diff --git a/ms/command-executor/src/main/python/utils.py b/ms/command-executor/src/main/python/utils.py
index 4314b287d..a3748eb17 100644
--- a/ms/command-executor/src/main/python/utils.py
+++ b/ms/command-executor/src/main/python/utils.py
@@ -16,7 +16,7 @@
 from google.protobuf.timestamp_pb2 import Timestamp
 
 import proto.CommandExecutor_pb2 as CommandExecutor_pb2
-
+import json
 
 def get_blueprint_id(request):
     blueprint_name = request.identifiers.blueprintName
@@ -24,7 +24,7 @@ def get_blueprint_id(request):
     return blueprint_name + '/' + blueprint_version
 
 
-def build_response(request, results, is_success=True):
+def build_response(request, log_results, payload_return, is_success=False):
     if is_success:
         status = CommandExecutor_pb2.SUCCESS
     else:
@@ -32,5 +32,9 @@ def build_response(request, results, is_success=True):
 
     timestamp = Timestamp()
     timestamp.GetCurrentTime()
-    return CommandExecutor_pb2.ExecutionOutput(requestId=request.requestId, response=results, status=status,
-                                               timestamp=timestamp)
+
+    if 'cds_return_code' in payload_return:
+        payload_return.pop('cds_return_code')
+    payload_str = json.dumps(payload_return)
+    return CommandExecutor_pb2.ExecutionOutput(requestId=request.requestId, response=log_results, status=status,
+                                               payload=payload_str, timestamp=timestamp)