diff options
12 files changed, 86 insertions, 62 deletions
diff --git a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/data_types.json b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/data_types.json index df594ab9e..25efb3656 100644 --- a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/data_types.json +++ b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/data_types.json @@ -10,7 +10,7 @@ "constraints": [ { "valid_values": [ - "ANSIBLE", "PYTHON" + "ansible_galaxy", "pip" ] } ] diff --git a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json index 0ee00b3ba..640a4ebee 100644 --- a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json +++ b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json @@ -4,8 +4,12 @@ "description": "This is Remote Python Execution Component.", "version": "1.0.0", "attributes": { - "execution-logs": { - "required": true, + "prepare-environment-logs": { + "required": false, + "type": "string" + }, + "execute-command-logs": { + "required": false, "type": "string" } }, diff --git a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/remote_scripts.json b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/remote_scripts.json index 80ef02f4f..da09e4f39 100644 --- a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/remote_scripts.json +++ b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/remote_scripts.json @@ -41,12 +41,21 @@ }, "inputs": {}, "outputs": { - "logs": { - "type": "json", + "prepare-environment-logs": { + "type": "string", "value": { "get_attribute": [ "execute-remote-python", - "execution-logs" + "prepare-environment-logs" + ] + } + }, + "execute-command-logs": { + "type": "string", + "value": { + "get_attribute": [ + "execute-remote-python", + "execute-command-logs" ] } } @@ -66,12 +75,21 @@ }, "inputs": {}, "outputs": { - "logs": { - "type": "json", + "prepare-environment-logs": { + "type": "string", + "value": { + "get_attribute": [ + "execute-remote-ansible", + "prepare-environment-logs" + ] + } + }, + "execute-command-logs": { + "type": "string", "value": { "get_attribute": [ "execute-remote-ansible", - "execution-logs" + "execute-command-logs" ] } } @@ -86,13 +104,18 @@ "operations": { "process": { "implementation": { - "primary": "component-script", - "dependencies": [ - "pyaml" - ] + "primary": "component-script" }, "inputs": { - "command": "python SamplePython.py blah" + "command": "python SamplePython.py blah", + "packages": [ + { + "type": "pip", + "package": [ + "pyaml" + ] + } + ] } } } @@ -118,13 +141,13 @@ "command": "ansible-playbook first.yaml", "packages": [ { - "type": "PYTHON", + "type": "pip", "package": [ "ansible" ] }, { - "type": "ANSIBLE", + "type": "ansible_galaxy", "package": [ "juniper.junos" ] diff --git a/components/model-catalog/definition-type/starter-type/data_type/dt-system-packages.json b/components/model-catalog/definition-type/starter-type/data_type/dt-system-packages.json index 81bb030b4..e70aa5ec8 100644 --- a/components/model-catalog/definition-type/starter-type/data_type/dt-system-packages.json +++ b/components/model-catalog/definition-type/starter-type/data_type/dt-system-packages.json @@ -8,8 +8,8 @@ "constraints": [ { "valid_values": [ - "ANSIBLE", - "PYTHON" + "ansible_galaxy", + "pip" ] } ] diff --git a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json index 3eaee761f..5c9503fc3 100644 --- a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json +++ b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json @@ -2,8 +2,12 @@ "description": "This is Remote Python Execution Component.", "version": "1.0.0", "attributes": { - "execution-logs": { - "required": true, + "prepare-environment-logs": { + "required": false, + "type": "string" + }, + "execute-command-logs": { + "required": false, "type": "string" } }, diff --git a/components/model-catalog/proto-definition/proto/CommandExecutor.proto b/components/model-catalog/proto-definition/proto/CommandExecutor.proto index 8f02b8aea..bc175dbc1 100644 --- a/components/model-catalog/proto-definition/proto/CommandExecutor.proto +++ b/components/model-catalog/proto-definition/proto/CommandExecutor.proto @@ -53,8 +53,8 @@ message Packages { } enum PackageType { - PYTHON = 0; - ANSIBLE = 1; + pip = 0; + ansible_galaxy = 1; } service CommandExecutorService { diff --git a/components/model-catalog/proto-definition/proto/README b/components/model-catalog/proto-definition/proto/README index 47bbf3f34..8cc9c485f 100644 --- a/components/model-catalog/proto-definition/proto/README +++ b/components/model-catalog/proto-definition/proto/README @@ -2,11 +2,10 @@ To create python bindings, Prerequisites: https://developers.google.com/protocol-buffers/docs/downloads - https://github.com/dropbox/mypy-protobuf - + or + https://grpc.io/docs/quickstart/python.html Command: - protoc -I=. --python_out=. --mypy_out=. CommandExecutor.proto - - - + protoc -I=. --python_out=. --grpc_python_out=. CommandExecutor.proto + or + python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. CommandExecutor.proto diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt index 17d5fc76d..488e0d054 100644 --- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt +++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt @@ -22,9 +22,8 @@ import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.* import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.ExecutionServiceConstant import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService -import org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException -import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode +import org.onap.ccsdk.cds.controllerblueprints.core.asJsonPrimitive import org.onap.ccsdk.cds.controllerblueprints.core.checkFileExists import org.onap.ccsdk.cds.controllerblueprints.core.checkNotBlank import org.onap.ccsdk.cds.controllerblueprints.core.data.OperationAssignment @@ -50,6 +49,9 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic const val INPUT_DYNAMIC_PROPERTIES = "dynamic-properties" const val INPUT_COMMAND = "command" const val INPUT_PACKAGES = "packages" + + const val ATTRIBUTE_PREPARE_ENV_LOG = "prepare-environment-logs" + const val ATTRIBUTE_EXEC_CMD_LOG = "execute-command-logs" } override suspend fun processNB(executionRequest: ExecutionServiceInput) { @@ -83,14 +85,10 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic // TODO("Python execution command and Resolve some expressions with dynamic properties") val scriptCommand = command.replace(pythonScript.name, pythonScript.absolutePath) -// val dependencies = operationAssignment.implementation?.dependencies - try { // Open GRPC Connection remoteScriptExecutionService.init(endPointSelector.asText()) - var executionLogs = "" - // If packages are defined, then install in remote server if (packages !is MissingNode && packages !is NullNode) { val prepareEnvInput = PrepareRemoteEnvInput(requestId = processId, @@ -99,8 +97,7 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic packages = packages ) val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput) - executionLogs = prepareEnvOutput.response - setOutput(executionLogs) + setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, prepareEnvOutput.response.asJsonPrimitive()) check(prepareEnvOutput.status == StatusType.SUCCESS) { "failed to get prepare remote env response status for requestId(${prepareEnvInput.requestId})" } @@ -111,24 +108,18 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion), command = scriptCommand) val remoteExecutionOutput = remoteScriptExecutionService.executeCommand(remoteExecutionInput) - executionLogs += remoteExecutionOutput.response - setOutput(executionLogs) + setAttribute(ATTRIBUTE_EXEC_CMD_LOG, remoteExecutionOutput.response.asJsonPrimitive()) check(remoteExecutionOutput.status == StatusType.SUCCESS) { "failed to get prepare remote command response status for requestId(${remoteExecutionOutput.requestId})" } } catch (e: Exception) { - log.error("", e) + log.error("Failed to process on remote executor", e) } finally { remoteScriptExecutionService.close() } } - private fun setOutput(executionLogs: String) { - bluePrintRuntimeService.setNodeTemplateAttributeValue(nodeTemplateName, - "execution-logs", JacksonUtils.jsonNodeFromObject(executionLogs)) - } - override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) { bluePrintRuntimeService.getBluePrintError() .addError("Failed in ComponentJythonExecutor : ${runtimeException.message}") diff --git a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt index 99d4f8c24..7aee95e11 100644 --- a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt +++ b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt @@ -107,7 +107,7 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi val pckage = Packages.newBuilder() JsonFormat.parser().merge(it.toString(), pckage) packageList.add(pckage.build()) - } + } return PrepareEnvInput.newBuilder() .setIdentifiers(this.remoteIdentifier!!.asGrpcData()) diff --git a/ms/command-executor/src/main/python/command_executor_handler.py b/ms/command-executor/src/main/python/command_executor_handler.py index 248e44308..9eb36fb2d 100644 --- a/ms/command-executor/src/main/python/command_executor_handler.py +++ b/ms/command-executor/src/main/python/command_executor_handler.py @@ -47,11 +47,11 @@ class CommandExecutorHandler(): return False f = open(self.installed, "w+") - if not self.install_packages(request, CommandExecutor_pb2.PYTHON, f, results): + if not self.install_packages(request, CommandExecutor_pb2.pip, f, results): return False f.write("\r\n") results.append("\n") - if not self.install_packages(request, CommandExecutor_pb2.ANSIBLE, f, results): + if not self.install_packages(request, CommandExecutor_pb2.ansible_galaxy, f, results): return False f.close() else: @@ -63,8 +63,8 @@ class CommandExecutorHandler(): return True def execute_command(self, request, results): - if not self.activate_venv(): - return False + # if not self.activate_venv(): + # return False try: results.append(os.popen(request.command).read()) @@ -82,7 +82,7 @@ class CommandExecutorHandler(): f.write("Installed %s packages:\r\n" % CommandExecutor_pb2.PackageType.Name(type)) for python_package in package.package: f.write(" %s\r\n" % python_package) - if package.type == CommandExecutor_pb2.PYTHON: + if package.type == CommandExecutor_pb2.pip: success = self.install_python_packages(python_package, results) else: success = self.install_ansible_packages(python_package, results) @@ -98,7 +98,8 @@ class CommandExecutorHandler(): command = ["pip", "install", package] env = dict(os.environ) - env['https_proxy'] = os.environ['https_proxy'] + if "https_proxy" in os.environ: + env['https_proxy'] = os.environ['https_proxy'] try: results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode()) @@ -111,11 +112,12 @@ class CommandExecutorHandler(): def install_ansible_packages(self, package, results): self.logger.info( "{} - Install Ansible Role package({}) in Python Virtual Environment".format(self.blueprint_id, package)) - command = ["ansible-galaxy", "install", package, "-p", "Scripts/ansible/roles"] + command = ["ansible-galaxy", "install", package, "-p", self.venv_home + "/Scripts/ansible/roles"] env = dict(os.environ) - # ansible galaxy uses https_proxy environment variable, but requires it to be set with http proxy value. - env['https_proxy'] = os.environ['http_proxy'] + if "http_proxy" in os.environ: + # ansible galaxy uses https_proxy environment variable, but requires it to be set with http proxy value. + env['https_proxy'] = os.environ['http_proxy'] try: results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode()) diff --git a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py index 76fc19d02..3afeb35fc 100644 --- a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py +++ b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py @@ -22,7 +22,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( package='org.onap.ccsdk.cds.controllerblueprints.command.api', syntax='proto3', serialized_options=_b('P\001'), - serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xba\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x01(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01*&\n\x0bPackageType\x12\n\n\x06PYTHON\x10\x00\x12\x0b\n\x07\x41NSIBLE\x10\x01\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3') + serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xba\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x01(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01**\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3') , dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,]) @@ -56,26 +56,26 @@ _PACKAGETYPE = _descriptor.EnumDescriptor( file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( - name='PYTHON', index=0, number=0, + name='pip', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ANSIBLE', index=1, number=1, + name='ansible_galaxy', index=1, number=1, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=1160, - serialized_end=1198, + serialized_end=1202, ) _sym_db.RegisterEnumDescriptor(_PACKAGETYPE) PackageType = enum_type_wrapper.EnumTypeWrapper(_PACKAGETYPE) SUCCESS = 0 FAILURE = 1 -PYTHON = 0 -ANSIBLE = 1 +pip = 0 +ansible_galaxy = 1 @@ -415,8 +415,8 @@ _COMMANDEXECUTORSERVICE = _descriptor.ServiceDescriptor( file=DESCRIPTOR, index=0, serialized_options=None, - serialized_start=1201, - serialized_end=1538, + serialized_start=1205, + serialized_end=1542, methods=[ _descriptor.MethodDescriptor( name='prepareEnv', diff --git a/ms/command-executor/src/main/python/server.py b/ms/command-executor/src/main/python/server.py index de620474b..453d751b2 100644 --- a/ms/command-executor/src/main/python/server.py +++ b/ms/command-executor/src/main/python/server.py @@ -1,3 +1,4 @@ + #!/usr/bin/python # |