diff options
12 files changed, 627 insertions, 110 deletions
diff --git a/components/model-catalog/proto-definition/proto/CommandExecutor.proto b/components/model-catalog/proto-definition/proto/CommandExecutor.proto index bad6a01c3..752dd8031 100644 --- a/components/model-catalog/proto-definition/proto/CommandExecutor.proto +++ b/components/model-catalog/proto-definition/proto/CommandExecutor.proto @@ -21,6 +21,29 @@ message ExecutionInput { string originatorId = 9; } +// If a new version of blueprint (new UUID in DB) needs to be uploaded, then pass in the raw bytes data +// properties should specify 'file_format' as either 'gzip' or 'zip' +// TODO: archiveTYPE: should be enum {"CBA_ZIP", "CBA_GZIP"} +message UploadBlueprintInput { + Identifiers identifiers = 1; + string requestId = 2; + string subRequestId = 3; + string originatorId = 4; + string correlationId = 5; + int32 timeOut = 6; + string archiveType = 7; + google.protobuf.Timestamp timestamp = 8; + bytes binData = 9; +} + +message UploadBlueprintOutput { + string requestId = 1; + string subRequestId = 2; + ResponseStatus status = 3; + google.protobuf.Timestamp timestamp = 4; + string payload = 5; +} + message PrepareEnvInput { Identifiers identifiers = 1; string requestId = 2; @@ -37,8 +60,12 @@ message PrepareEnvInput { message Identifiers { string blueprintName = 1; string blueprintVersion = 2; + string blueprintUUID = 3; } +// TODO: need to rething whether we want to include subrequest/correlationID/etc to be consistent +// or drop requestId as it would be returned back to the context where these values are already known. +// and we may just be concerned with the response/status message ExecutionOutput { string requestId = 1; repeated string response = 2; @@ -64,6 +91,10 @@ enum PackageType { } service CommandExecutorService { + // rpc to upload the CBA + rpc uploadBlueprint (UploadBlueprintInput) returns (UploadBlueprintOutput); + // prepare Python environment rpc prepareEnv (PrepareEnvInput) returns (ExecutionOutput); + // execute the actual command. rpc executeCommand (ExecutionInput) returns (ExecutionOutput); } diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt index c20e63667..a809d8f18 100644 --- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt +++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt @@ -18,6 +18,7 @@ package org.onap.ccsdk.cds.blueprintsprocessor.functions.python.executor import com.fasterxml.jackson.databind.JsonNode +import com.google.protobuf.ByteString import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.async @@ -27,12 +28,16 @@ import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInpu import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.PrepareRemoteEnvInput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteIdentifier import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptExecutionInput +import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptExecutionOutput +import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptUploadBlueprintInput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.StatusType +import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.repository.BlueprintModelRepository import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.ExecutionServiceConstant import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService import org.onap.ccsdk.cds.controllerblueprints.core.BlueprintProcessorException import org.onap.ccsdk.cds.controllerblueprints.core.asJsonPrimitive +import org.onap.ccsdk.cds.controllerblueprints.core.asJsonType import org.onap.ccsdk.cds.controllerblueprints.core.checkFileExists import org.onap.ccsdk.cds.controllerblueprints.core.checkNotBlank import org.onap.ccsdk.cds.controllerblueprints.core.data.OperationAssignment @@ -51,7 +56,8 @@ import org.springframework.stereotype.Component @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) open class ComponentRemotePythonExecutor( private val remoteScriptExecutionService: RemoteScriptExecutionService, - private var bluePrintPropertiesService: BlueprintPropertiesService + private val bluePrintPropertiesService: BlueprintPropertiesService, + private val blueprintModelRepository: BlueprintModelRepository ) : AbstractComponentFunction() { private val log = LoggerFactory.getLogger(ComponentRemotePythonExecutor::class.java)!! @@ -78,6 +84,7 @@ open class ComponentRemotePythonExecutor( const val DEFAULT_ENV_PREPARE_TIMEOUT_IN_SEC = 120 const val DEFAULT_EXECUTE_TIMEOUT_IN_SEC = 180 const val TIMEOUT_DELTA = 100L + const val DEFAULT_CBA_UPLOAD_TIMEOUT_IN_SEC = 30 } override suspend fun processNB(executionRequest: ExecutionServiceInput) { @@ -90,6 +97,16 @@ open class ComponentRemotePythonExecutor( val blueprintName = bluePrintContext.name() val blueprintVersion = bluePrintContext.version() + // fetch the template (plus cba bindata) from repository + val cbaModel = blueprintModelRepository.findByArtifactNameAndArtifactVersion(blueprintName, blueprintVersion) + val blueprintUUID = cbaModel?.id!! + val cbaBinData = ByteString.copyFrom(cbaModel?.blueprintModelContent?.content) + val archiveType = cbaModel?.blueprintModelContent?.contentType // TODO: should be enum + val remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion, blueprintUUID = blueprintUUID) + val originatorId = executionServiceInput.commonHeader.originatorId + val subRequestId = executionServiceInput.commonHeader.subRequestId + val requestId = processId + val operationAssignment: OperationAssignment = bluePrintContext .nodeTemplateInterfaceOperation(nodeTemplateName, interfaceName, operationName) @@ -116,6 +133,7 @@ open class ComponentRemotePythonExecutor( ?.rootFieldsToMap()?.toSortedMap()?.values?.joinToString(" ") { formatNestedJsonNode(it) } val command = getOperationInput(INPUT_COMMAND).asText() + val cbaNameVerUuid = "blueprintName($blueprintName) blueprintVersion($blueprintVersion) blueprintUUID($blueprintUUID)" /** * Timeouts that are specific to the command executor. @@ -129,7 +147,7 @@ open class ComponentRemotePythonExecutor( // component level timeout should be => env_prep_timeout + execution_timeout val timeout = implementation.timeout - var scriptCommand = command.replace(pythonScript.name, pythonScript.absolutePath) + var scriptCommand = command.replace(pythonScript.name, artifactDefinition.file) if (args != null && args.isNotEmpty()) { scriptCommand = scriptCommand.plus(" ").plus(args) } @@ -150,13 +168,9 @@ open class ComponentRemotePythonExecutor( originatorId = executionServiceInput.commonHeader.originatorId, requestId = processId, subRequestId = executionServiceInput.commonHeader.subRequestId, - remoteIdentifier = RemoteIdentifier( - blueprintName = blueprintName, - blueprintVersion = blueprintVersion - ), + remoteIdentifier = remoteIdentifier, packages = packages, timeOut = envPrepTimeout.toLong() - ) val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput) log.info("$ATTRIBUTE_PREPARE_ENV_LOG - ${prepareEnvOutput.response}") @@ -171,11 +185,15 @@ open class ComponentRemotePythonExecutor( setNodeOutputProperties(prepareEnvOutput.status, STEP_PREPARE_ENV, logs, prepareEnvOutput.payload, isLogResponseEnabled) } } else { - // set env preparation log to empty... - setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, "".asJsonPrimitive()) + if (packages == null) { + // set env preparation log to empty... + setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, "".asJsonPrimitive()) + } else { + prepareEnv(originatorId, requestId, subRequestId, remoteIdentifier, packages, envPrepTimeout, cbaNameVerUuid, archiveType, cbaBinData, isLogResponseEnabled) + } + // in cases where the exception is caught in BP side due to timeout, we do not have `err_msg` returned by cmd-exec (inside `payload`), + // hence `artifact` field will be empty } - // in cases where the exception is caught in BP side due to timeout, we do not have `err_msg` returned by cmd-exec (inside `payload`), - // hence `artifact` field will be empty } catch (grpcEx: io.grpc.StatusRuntimeException) { val componentLevelWarningMsg = if (timeout < envPrepTimeout) "Note: component-level timeout ($timeout) is shorter than env-prepare timeout ($envPrepTimeout). " else "" @@ -197,7 +215,7 @@ open class ComponentRemotePythonExecutor( log.error(catchallErrMsg, e) } // if Env preparation was successful, then proceed with command execution in this Env - if (bluePrintRuntimeService.getBlueprintError().errors.isEmpty()) { + if (noBlueprintErrors()) { try { // Populate command execution properties and pass it to the remote server val properties = dynamicProperties?.returnNullIfMissing()?.rootFieldsToMap() ?: hashMapOf() @@ -206,7 +224,7 @@ open class ComponentRemotePythonExecutor( originatorId = executionServiceInput.commonHeader.originatorId, requestId = processId, subRequestId = executionServiceInput.commonHeader.subRequestId, - remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion), + remoteIdentifier = remoteIdentifier, command = scriptCommand, properties = properties, timeOut = executionTimeout.toLong() @@ -238,19 +256,19 @@ open class ComponentRemotePythonExecutor( if (timeout < executionTimeout) "Note: component-level timeout ($timeout) is shorter than execution timeout ($executionTimeout). " else "" val timeoutErrMsg = "Command executor execution timeout. DetailedMessage: (${timeoutEx.message}) requestId ($processId). $componentLevelWarningMsg" - setNodeOutputErrors(STEP_EXEC_CMD, listOf(timeoutErrMsg).asJsonPrimitive(), logging = isLogResponseEnabled) + setNodeOutputErrors(STEP_EXEC_CMD, listOf(timeoutErrMsg).asJsonType(), logging = isLogResponseEnabled) addError(StatusType.FAILURE.name, STEP_EXEC_CMD, timeoutErrMsg) log.error(timeoutErrMsg, timeoutEx) } catch (grpcEx: io.grpc.StatusRuntimeException) { val timeoutErrMsg = "Command executor timed out executing after $executionTimeout seconds requestId ($processId) grpcErr: ${grpcEx.status}" - setNodeOutputErrors(STEP_EXEC_CMD, listOf(timeoutErrMsg).asJsonPrimitive(), logging = isLogResponseEnabled) + setNodeOutputErrors(STEP_EXEC_CMD, listOf(timeoutErrMsg).asJsonType(), logging = isLogResponseEnabled) addError(StatusType.FAILURE.name, STEP_EXEC_CMD, timeoutErrMsg) log.error(timeoutErrMsg, grpcEx) } catch (e: Exception) { val catchAllErrMsg = "Command executor failed during process catch-all case requestId ($processId) timeout($envPrepTimeout) exception msg: ${e.message}" - setNodeOutputErrors(STEP_PREPARE_ENV, listOf(catchAllErrMsg).asJsonPrimitive(), logging = isLogResponseEnabled) + setNodeOutputErrors(STEP_PREPARE_ENV, listOf(catchAllErrMsg).asJsonType(), logging = isLogResponseEnabled) addError(StatusType.FAILURE.name, STEP_EXEC_CMD, catchAllErrMsg) log.error(catchAllErrMsg, e) } @@ -259,6 +277,70 @@ open class ComponentRemotePythonExecutor( remoteScriptExecutionService.close() } + // wrapper for call to prepare_env step on cmd-exec - reupload CBA and call prepare env again if cmd-exec reported CBA uuid mismatch + private suspend fun prepareEnv(originatorId: String, requestId: String, subRequestId: String, remoteIdentifier: RemoteIdentifier, packages: JsonNode, envPrepTimeout: Int, cbaNameVerUuid: String, archiveType: String?, cbaBinData: ByteString?, isLogResponseEnabled: Boolean, innerCall: Boolean = false) { + val prepareEnvInput = PrepareRemoteEnvInput( + originatorId = originatorId, + requestId = requestId, + subRequestId = subRequestId, + remoteIdentifier = remoteIdentifier, + packages = packages, + timeOut = envPrepTimeout.toLong() + ) + val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput) + log.info("$ATTRIBUTE_PREPARE_ENV_LOG - ${prepareEnvOutput.response}") + val logs = JacksonUtils.jsonNodeFromObject(prepareEnvOutput.response) + setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, logs) + + // there are no artifacts for env. prepare, but we reuse it for err_log... + if (prepareEnvOutput.status != StatusType.SUCCESS) { + // Check for the flag that blueprint is mismatched first, if so, reupload the blueprint + if (prepareEnvOutput.payload.has("reupload_cba")) { + log.info("Cmd-exec is missing the CBA $cbaNameVerUuid, it will be reuploaded.") + uploadCba(remoteIdentifier, requestId, subRequestId, originatorId, archiveType, cbaBinData, cbaNameVerUuid, prepareEnvOutput, isLogResponseEnabled, logs) + // call prepare_env again. + if (!innerCall) { + log.info("Calling prepare environment again") + prepareEnv(originatorId, requestId, subRequestId, remoteIdentifier, packages, envPrepTimeout, cbaNameVerUuid, archiveType, cbaBinData, isLogResponseEnabled) + } else { + val errMsg = "Something is wrong: prepare_env step attempted to call itself too many times after upload CBA step!" + log.error(errMsg) + setNodeOutputErrors(STEP_PREPARE_ENV, "[]".asJsonPrimitive(), prepareEnvOutput.payload, isLogResponseEnabled) + addError(StatusType.FAILURE.name, STEP_PREPARE_ENV, errMsg) + } + } else { + setNodeOutputErrors(STEP_PREPARE_ENV, "[]".asJsonPrimitive(), prepareEnvOutput.payload, isLogResponseEnabled) + addError(StatusType.FAILURE.name, STEP_PREPARE_ENV, logs.toString()) + } + } else { + setNodeOutputProperties(prepareEnvOutput.status, STEP_PREPARE_ENV, logs, prepareEnvOutput.payload, isLogResponseEnabled) + } + } + + private suspend fun uploadCba(remoteIdentifier: RemoteIdentifier, requestId: String, subRequestId: String, originatorId: String, archiveType: String?, cbaBinData: ByteString?, cbaNameVerUuid: String, prepareEnvOutput: RemoteScriptExecutionOutput, isLogResponseEnabled: Boolean, logs: JsonNode) { + + val uploadCbaInput = RemoteScriptUploadBlueprintInput( + remoteIdentifier = remoteIdentifier, + requestId = requestId, + subRequestId = subRequestId, + originatorId = originatorId, + timeOut = DEFAULT_CBA_UPLOAD_TIMEOUT_IN_SEC.toLong(), + archiveType = archiveType!!, + binData = cbaBinData!! + ) + + val cbaUploadOutput = remoteScriptExecutionService.uploadBlueprint(uploadCbaInput) + if (cbaUploadOutput.status != StatusType.SUCCESS) { + log.error("Error uploading CBA $cbaNameVerUuid error(${cbaUploadOutput.payload})") + setNodeOutputErrors(STEP_PREPARE_ENV, "[]".asJsonPrimitive(), prepareEnvOutput.payload, isLogResponseEnabled) + addError(StatusType.FAILURE.name, STEP_PREPARE_ENV, logs.toString()) + } else { + log.info("Finished uploading CBA $cbaNameVerUuid") + } + } + + private fun noBlueprintErrors() = bluePrintRuntimeService.getBlueprintError().errors.isEmpty() + override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) { bluePrintRuntimeService.getBlueprintError() .addError("Failed in ComponentRemotePythonExecutor : ${runtimeException.message}") diff --git a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt index cad1f8df2..2fd33f2a6 100644 --- a/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt +++ b/ms/blueprintsprocessor/functions/python-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutorTest.kt @@ -27,8 +27,11 @@ import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInpu import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.PrepareRemoteEnvInput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptExecutionInput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptExecutionOutput +import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptUploadBlueprintInput +import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptUploadBlueprintOutput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.StatusType import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.StepData +import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.repository.BlueprintModelRepository import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService import org.onap.ccsdk.cds.controllerblueprints.core.BlueprintConstants import org.onap.ccsdk.cds.controllerblueprints.core.BlueprintError @@ -50,7 +53,8 @@ class ComponentRemotePythonExecutorTest { val componentRemotePythonExecutor = ComponentRemotePythonExecutor( remoteScriptExecutionService, - mockk<BlueprintPropertiesService>() + mockk<BlueprintPropertiesService>(), + mockk<BlueprintModelRepository>() ) val executionServiceInput = @@ -94,7 +98,8 @@ class ComponentRemotePythonExecutorTest { val remoteScriptExecutionService = MockRemoteScriptExecutionService() val componentRemotePythonExecutor = ComponentRemotePythonExecutor( remoteScriptExecutionService, - mockk<BlueprintPropertiesService>() + mockk<BlueprintPropertiesService>(), + mockk<BlueprintModelRepository>() ) val bluePrintRuntime = mockk<DefaultBlueprintRuntimeService>("123456-1000") @@ -223,6 +228,15 @@ class MockRemoteScriptExecutionService : RemoteScriptExecutionService { override suspend fun init(selector: Any) { } + override suspend fun uploadBlueprint(uploadBpInput: RemoteScriptUploadBlueprintInput): RemoteScriptUploadBlueprintOutput { + val uploadBpOutput = mockk<RemoteScriptUploadBlueprintOutput>() + every { uploadBpOutput.payload } returns "[]".asJsonPrimitive() + every { uploadBpOutput.status } returns StatusType.SUCCESS + every { uploadBpOutput.requestId } returns "123456-1000" + every { uploadBpOutput.subRequestId } returns "1234" + return uploadBpOutput + } + override suspend fun prepareEnv(prepareEnvInput: PrepareRemoteEnvInput): RemoteScriptExecutionOutput { assertEquals(prepareEnvInput.requestId, "123456-1000", "failed to match request id") assertNotNull(prepareEnvInput.packages, "failed to get packages") diff --git a/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/primary/domain/BlueprintModelSearch.kt b/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/primary/domain/BlueprintModelSearch.kt index 734bf3689..ba9051898 100644 --- a/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/primary/domain/BlueprintModelSearch.kt +++ b/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/primary/domain/BlueprintModelSearch.kt @@ -104,6 +104,6 @@ class BlueprintModelSearch : Serializable { companion object { - const val serialversionuid = 1L + const val serialVersionUID = 1L } } diff --git a/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/primary/repository/BlueprintModelRepository.kt b/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/primary/repository/BlueprintModelRepository.kt index a7891f6a7..a6f0da1cc 100644 --- a/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/primary/repository/BlueprintModelRepository.kt +++ b/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/primary/repository/BlueprintModelRepository.kt @@ -20,6 +20,8 @@ package org.onap.ccsdk.cds.blueprintsprocessor.db.primary.repository import org.jetbrains.annotations.NotNull import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.domain.BlueprintModel import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.data.jpa.repository.Query +import org.springframework.data.repository.query.Param import org.springframework.stereotype.Repository import java.util.Optional import javax.transaction.Transactional @@ -48,6 +50,16 @@ interface BlueprintModelRepository : JpaRepository<BlueprintModel, String> { fun findByArtifactNameAndArtifactVersion(artifactName: String, artifactVersion: String): BlueprintModel? /** + * Find the Blueprint UUID (blueprint_model_id) for a given artifactName/Version + * + * @param artifactName artifactName + * @param artifactVersion artifactVersion + * @return String? + */ + @Query("SELECT m.id FROM BlueprintModel m WHERE m.artifactName = :artifactName AND m.artifactVersion = :artifactVersion") + fun findIdByArtifactNameAndArtifactVersion(@Param("artifactName") artifactName: String, @Param("artifactVersion") artifactVersion: String): String? + + /** * This is a findTopByArtifactNameOrderByArtifactIdDesc method * * @param artifactName artifactName diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt b/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt index 6baf261fb..d8baa8eaf 100644 --- a/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt +++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/api/data/BlueprintRemoteProcessorData.kt @@ -18,15 +18,39 @@ package org.onap.ccsdk.cds.blueprintsprocessor.core.api.data import com.fasterxml.jackson.databind.JsonNode +import com.google.protobuf.ByteString import java.util.Date enum class StatusType { SUCCESS, FAILURE } +/* TODO: Group fields into another struct containing originatorId, requestId, subRequestId, correlationId generally go together */ +// timeOuts are in seconds + data class RemoteIdentifier( var blueprintName: String, - var blueprintVersion: String + var blueprintVersion: String, + var blueprintUUID: String +) + +data class RemoteScriptUploadBlueprintInput( + val remoteIdentifier: RemoteIdentifier? = null, + val requestId: String, + val subRequestId: String, + val originatorId: String, + val correlationId: String? = null, + val timeOut: Long = 30, + val archiveType: String = "CBA_ZIP", + val binData: ByteString +) + +data class RemoteScriptUploadBlueprintOutput( + val requestId: String, + val subRequestId: String, + val status: StatusType = StatusType.SUCCESS, + val timestamp: Date = Date(), + val payload: JsonNode ) data class RemoteScriptExecutionInput( diff --git a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt index 4f3f60110..7e3f16f39 100644 --- a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt +++ b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt @@ -24,8 +24,10 @@ import com.google.protobuf.util.JsonFormat import io.grpc.ManagedChannel import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.PrepareRemoteEnvInput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteIdentifier +import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptUploadBlueprintInput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptExecutionInput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptExecutionOutput +import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.RemoteScriptUploadBlueprintOutput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.StatusType import org.onap.ccsdk.cds.blueprintsprocessor.grpc.service.BlueprintGrpcClientService import org.onap.ccsdk.cds.blueprintsprocessor.grpc.service.BlueprintGrpcLibPropertyService @@ -35,6 +37,8 @@ import org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput import org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers import org.onap.ccsdk.cds.controllerblueprints.command.api.Packages import org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput +import org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput +import org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintOutput import org.onap.ccsdk.cds.controllerblueprints.core.jsonAsJsonType import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils import org.slf4j.LoggerFactory @@ -47,6 +51,7 @@ import java.util.concurrent.TimeUnit interface RemoteScriptExecutionService { suspend fun init(selector: Any) + suspend fun uploadBlueprint(uploadBpInput: RemoteScriptUploadBlueprintInput): RemoteScriptUploadBlueprintOutput suspend fun prepareEnv(prepareEnvInput: PrepareRemoteEnvInput): RemoteScriptExecutionOutput suspend fun executeCommand(remoteExecutionInput: RemoteScriptExecutionInput): RemoteScriptExecutionOutput suspend fun close() @@ -84,6 +89,19 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi } } + override suspend fun uploadBlueprint(uploadBPInput: RemoteScriptUploadBlueprintInput): RemoteScriptUploadBlueprintOutput { + val logPart = "requestId(${uploadBPInput.requestId}) subRequestId(${uploadBPInput.subRequestId}) blueprintName(${uploadBPInput.remoteIdentifier?.blueprintName}) blueprintVersion(${uploadBPInput.remoteIdentifier?.blueprintVersion}) blueprintUUID(${uploadBPInput.remoteIdentifier?.blueprintUUID})" + val grpcResponse = commandExecutorServiceGrpc + .withDeadlineAfter(uploadBPInput.timeOut * 1000, TimeUnit.MILLISECONDS) + .uploadBlueprint(uploadBPInput.asGrpcData()) + checkNotNull(grpcResponse.status) { + "failed to get GRPC upload CBA response status for $logPart" + } + val uploadBlueprinOutput = grpcResponse.asJavaData() + log.info("Received Upload CBA response status(${uploadBlueprinOutput.status}) for $logPart payload(${uploadBlueprinOutput.payload})") + return uploadBlueprinOutput + } + override suspend fun prepareEnv(prepareEnvInput: PrepareRemoteEnvInput): RemoteScriptExecutionOutput { val grpResponse = commandExecutorServiceGrpc .withDeadlineAfter(prepareEnvInput.timeOut * 1000, TimeUnit.MILLISECONDS) @@ -117,6 +135,21 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi channel?.shutdownNow() } + fun RemoteScriptUploadBlueprintInput.asGrpcData(): UploadBlueprintInput { + val correlationId = this.correlationId ?: this.requestId + return UploadBlueprintInput.newBuilder() + .setIdentifiers(this.remoteIdentifier!!.asGrpcData()) + .setRequestId(this.requestId) + .setSubRequestId(this.subRequestId) + .setOriginatorId(this.originatorId) + .setCorrelationId(correlationId) + .setTimestamp(Timestamp.getDefaultInstance()) + .setBinData(this.binData) + .setArchiveType(this.archiveType) + .setTimeOut(this.timeOut.toInt()) + .build() + } + fun PrepareRemoteEnvInput.asGrpcData(): PrepareEnvInput { val correlationId = this.correlationId ?: this.requestId @@ -159,6 +192,7 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi return Identifiers.newBuilder() .setBlueprintName(this.blueprintName) .setBlueprintVersion(this.blueprintVersion) + .setBlueprintUUID(this.blueprintUUID) .build() } @@ -176,4 +210,13 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi payload = payload.jsonAsJsonType() ) } + + fun UploadBlueprintOutput.asJavaData(): RemoteScriptUploadBlueprintOutput { + return RemoteScriptUploadBlueprintOutput( + requestId = this.requestId, + subRequestId = this.subRequestId, + status = StatusType.valueOf(this.status.name), + payload = payload.jsonAsJsonType() + ) + } } diff --git a/ms/command-executor/src/main/python/command_executor_handler.py b/ms/command-executor/src/main/python/command_executor_handler.py index b7d3eec0d..df06ba5be 100644 --- a/ms/command-executor/src/main/python/command_executor_handler.py +++ b/ms/command-executor/src/main/python/command_executor_handler.py @@ -19,56 +19,125 @@ from google.protobuf.json_format import MessageToJson import tempfile import logging import os +import sys import re import subprocess import virtualenv import venv import utils import proto.CommandExecutor_pb2 as CommandExecutor_pb2 +from zipfile import ZipFile +import io REQUIREMENTS_TXT = "requirements.txt" class CommandExecutorHandler(): + BLUEPRINTS_DEPLOY_DIR = '/opt/app/onap/blueprints/deploy/' + TOSCA_META_FILE = 'TOSCA-Metadata/TOSCA.meta' def __init__(self, request): self.request = request self.logger = logging.getLogger(self.__class__.__name__) - self.blueprint_id = utils.get_blueprint_id(request) + self.blueprint_name = utils.get_blueprint_name(request) + self.blueprint_version = utils.get_blueprint_version(request) + self.uuid = utils.get_blueprint_uuid(request) + self.request_id = utils.get_blueprint_requestid(request) + self.sub_request_id = utils.get_blueprint_subRequestId(request) + self.blueprint_name_version_uuid = utils.blueprint_name_version_uuid(request) self.execution_timeout = utils.get_blueprint_timeout(request) - self.venv_home = '/opt/app/onap/blueprints/deploy/' + self.blueprint_id - self.installed = self.venv_home + '/.installed' + # onap/blueprints/deploy will be ephemeral now + self.blueprint_dir = self.BLUEPRINTS_DEPLOY_DIR + self.blueprint_name_version_uuid + self.blueprint_tosca_meta_file = self.blueprint_dir + '/' + self.TOSCA_META_FILE self.extra = utils.getExtraLogData(request) + self.installed = self.blueprint_dir + '/.installed' def is_installed(self): return os.path.exists(self.installed) + def blueprint_dir_exists(self): + return os.path.exists(self.blueprint_dir) + + # used to validate if the blueprint actually had a chace of getting uploaded + def blueprint_tosca_meta_file_exists(self): + return os.path.exists(self.blueprint_tosca_meta_file) + + def err_exit(self, msg): + self.logger.error(msg, extra=self.extra) + return utils.build_ret_data(False, error=msg) + + def is_valid_archive_type(self, archiveType): + return archiveType=="CBA_ZIP" or archiveType=="CBA_GZIP" + + # Handle uploading blueprint request + # accept UploadBlueprintInput (CommandExecutor.proto) struct + # create dir blueprintName/BlueprintVersion/BlueprintUUID, and extract binData as either ZIP file or GZIP + # based on archiveType field... + def uploadBlueprint(self, request): + archive_type = request.archiveType + compressed_cba_stream = io.BytesIO(request.binData) + + if not self.is_valid_archive_type(archive_type): + return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, False, ["Archive type {} is not valid.".format(archive_type)]) + + # create the BP dir self.blueprint_dir + try: + os.makedirs(name=self.blueprint_dir, mode=0o755, exist_ok=True) + except OSError as ex: + err_msg = "Failed to create blueprint dir: {} exception message: {}".format(self.blueprint_dir, ex.strerror) + self.logger.error(err_msg, extra=self.extra) + return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, False, [err_msg]) + if archive_type=="CBA_ZIP": + self.logger.info("Extracting ZIP data to dir {}".format(self.blueprint_dir), extra=self.extra) + try: + with ZipFile(compressed_cba_stream,'r') as zipfile: + zipfile.extractall(self.blueprint_dir) + self.logger.info("Done extracting ZIP data to dir {}".format(self.blueprint_dir), extra=self.extra) + except (IOError, zipfile.error) as e: + err_msg = "Error extracting ZIP data to dir {} exception: {}".format(self.blueprint_dir, e) + self.logger.error(err_msg, extra=self.extra) + return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, False, [err_msg]) + # TODO with an actual test gzip cba... + elif archive_type=="CBA_GZIP": + self.logger.error("CBA_GZIP TODO", extra=self.extra) + return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, False, ["Error extracting GZIP data to {} GZIP todo!".format(self.blueprint_dir)]) + # Finally, everything is ok! + return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, True, []) + def prepare_env(self, request): results_log = [] + + # validate that the blueprint name in the request exists, if not, notify the caller + if not self.blueprint_dir_exists(): + err_msg = "CBA directory {} not found on cmd-exec. CBA will be uploaded by BP proc.".format(self.blueprint_name_version_uuid) + self.logger.info(err_msg, extra=self.extra) + return utils.build_ret_data(False, results_log=results_log, error=err_msg, reupload_cba=True) + if not self.blueprint_tosca_meta_file_exists(): + err_msg = "CBA directory {} exists on cmd-exec, but TOSCA meta file is not found!!! Returning (null) as UUID. CBA will be uploaded by BP proc.".format(self.blueprint_name_version_uuid) + self.logger.info(err_msg, extra=self.extra) + return utils.build_ret_data(False, results_log=results_log, error=err_msg, reupload_cba=True) + self.logger.info("CBA directory {} exists on cmd-exec.".format(self.blueprint_name_version_uuid), extra=self.extra) + if not self.is_installed(): create_venv_status = self.create_venv() if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]: - err_msg = "ERROR: failed to prepare environment for request {} due to error in creating virtual Python env. Original error {}".format(self.blueprint_id, create_venv_status[utils.ERR_MSG_KEY]) - self.logger.error(err_msg, extra=self.extra) - return utils.build_ret_data(False, error=err_msg) + return self.err_exit("ERROR: failed to prepare environment for request {} due to error in creating virtual Python env. Original error {}".format(self.blueprint_name_version_uuid, create_venv_status[utils.ERR_MSG_KEY])) activate_venv_status = self.activate_venv() if not activate_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]: - err_msg = "ERROR: failed to prepare environment for request {} due Python venv_activation. Original error {}".format(self.blueprint_id, activate_venv_status[utils.ERR_MSG_KEY]) - self.logger.error(err_msg, extra=self.extra) - return utils.build_ret_data(False, error=err_msg) + return self.err_exit("ERROR: failed to prepare environment for request {} due Python venv_activation. Original error {}".format(self.blueprint_name_version_uuid, activate_venv_status[utils.ERR_MSG_KEY])) try: with open(self.installed, "w+") as f: if not self.install_packages(request, CommandExecutor_pb2.pip, f, results_log): - err_msg = "ERROR: failed to prepare environment for request {} during pip package install.".format(self.blueprint_id) + err_msg = "ERROR: failed to prepare environment for request {} during pip package install.".format(self.blueprint_name_version_uuid) return utils.build_ret_data(False, results_log=results_log, error=err_msg) f.write("\r\n") # TODO: is \r needed? results_log.append("\n") if not self.install_packages(request, CommandExecutor_pb2.ansible_galaxy, f, results_log): - err_msg = "ERROR: failed to prepare environment for request {} during Ansible install.".format(self.blueprint_id) + err_msg = "ERROR: failed to prepare environment for request {} during Ansible install.".format(self.blueprint_name_version_uuid) return utils.build_ret_data(False, results_log=results_log, error=err_msg) except Exception as ex: - err_msg = "ERROR: failed to prepare environment for request {} during installing packages. Exception: {}".format(self.blueprint_id, ex) + err_msg = "ERROR: failed to prepare environment for request {} during installing packages. Exception: {}".format(self.blueprint_name_version_uuid, ex) self.logger.error(err_msg, extra=self.extra) return utils.build_ret_data(False, error=err_msg) else: @@ -87,6 +156,7 @@ class CommandExecutorHandler(): results_log = [] # encoded payload returned by the process result = {} + # workaround for when packages are not specified, we may not want to go through the install step # can just call create_venv from here. if not self.is_installed(): @@ -95,15 +165,17 @@ class CommandExecutorHandler(): if not self.is_installed(): create_venv_status = self.create_venv if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]: - err_msg = "{} - Failed to execute command during venv creation. Original error: {}".format(self.blueprint_id, create_venv_status[utils.ERR_MSG_KEY]) + err_msg = "{} - Failed to execute command during venv creation. Original error: {}".format(self.blueprint_name_version_uuid, create_venv_status[utils.ERR_MSG_KEY]) return utils.build_ret_data(False, error=err_msg) activate_response = self.activate_venv() if not activate_response[utils.CDS_IS_SUCCESSFUL_KEY]: orig_error = activate_response[utils.ERR_MSG_KEY] - err_msg = "{} - Failed to execute command during environment activation. Original error: {}".format(self.blueprint_id, orig_error) + err_msg = "{} - Failed to execute command during environment activation. Original error: {}".format(self.blueprint_name_version_uuid, orig_error) return utils.build_ret_data(False, error=err_msg) + # touch blueprint dir to indicate this CBA was used recently + os.utime(self.blueprint_dir) - cmd = "cd " + self.venv_home + cmd = "cd " + self.blueprint_dir ### if properties are defined we add them to the command properties = "" @@ -112,29 +184,25 @@ class CommandExecutorHandler(): ### TODO: replace with os.environ['VIRTUAL_ENV']? if "ansible-playbook" in request.command: - cmd = cmd + "; " + request.command + " -e 'ansible_python_interpreter=" + self.venv_home + "/bin/python'" + cmd = cmd + "; " + request.command + " -e 'ansible_python_interpreter=" + self.blueprint_dir + "/bin/python'" else: cmd = cmd + "; " + request.command + properties ### extract the original header request into sys-env variables - ### RequestID - request_id = request.requestId - ### Sub-requestID - subrequest_id = request.subRequestId ### OriginatorID originator_id = request.originatorId ### CorrelationID correlation_id = request.correlationId - request_id_map = {'CDS_REQUEST_ID':request_id, 'CDS_SUBREQUEST_ID':subrequest_id, 'CDS_ORIGINATOR_ID': originator_id, 'CDS_CORRELATION_ID': correlation_id} + request_id_map = {'CDS_REQUEST_ID':self.request_id, 'CDS_SUBREQUEST_ID':self.sub_request_id, 'CDS_ORIGINATOR_ID': originator_id, 'CDS_CORRELATION_ID': correlation_id} updated_env = { **os.environ, **request_id_map } - self.logger.info("Running blueprint {} with timeout: {}".format(self.blueprint_id, self.execution_timeout), extra=self.extra) + self.logger.info("Running blueprint {} with timeout: {}".format(self.blueprint_name_version_uuid, self.execution_timeout), extra=self.extra) with tempfile.TemporaryFile(mode="w+") as tmp: try: completed_subprocess = subprocess.run(cmd, stdout=tmp, stderr=subprocess.STDOUT, shell=True, env=updated_env, timeout=self.execution_timeout) except TimeoutExpired: - timeout_err_msg = "Running command {} failed due to timeout of {} seconds.".format(self.blueprint_id, self.execution_timeout) + timeout_err_msg = "Running command {} failed due to timeout of {} seconds.".format(self.blueprint_name_version_uuid, self.execution_timeout) self.logger.error(timeout_err_msg, extra=self.extra) utils.parse_cmd_exec_output(tmp, self.logger, result, results_log, self.extra) return utils.build_ret_data(False, results_log=results_log, error=timeout_err_msg) @@ -142,7 +210,7 @@ class CommandExecutorHandler(): utils.parse_cmd_exec_output(tmp, self.logger, result, results_log, self.extra) rc = completed_subprocess.returncode except Exception as e: - err_msg = "{} - Failed to execute command. Error: {}".format(self.blueprint_id, e) + err_msg = "{} - Failed to execute command. Error: {}".format(self.blueprint_name_version_uuid, e) result.update(utils.build_ret_data(False, results_log=results_log, error=err_msg)) return result @@ -155,6 +223,9 @@ class CommandExecutorHandler(): def install_packages(self, request, type, f, results): success = self.install_python_packages('UTILITY', results) + if not success: + self.logger.error("Error installing 'UTILITY (cds_utils) package to CBA python environment!!!", extra=self.extra) + return False for package in request.packages: if package.type == type: @@ -173,21 +244,21 @@ class CommandExecutorHandler(): def install_python_packages(self, package, results): self.logger.info( - "{} - Install Python package({}) in Python Virtual Environment".format(self.blueprint_id, package), extra=self.extra) + "{} - Install Python package({}) in Python Virtual Environment".format(self.blueprint_name_version_uuid, package), extra=self.extra) if REQUIREMENTS_TXT == package: - command = ["pip", "install", "-r", self.venv_home + "/Environments/" + REQUIREMENTS_TXT] + command = ["pip", "install", "--user", "-r", self.blueprint_dir + "/Environments/" + REQUIREMENTS_TXT] elif package == 'UTILITY': - # TODO: fix python version that is hardcoded here, may fail if python image is upgraded - command = ["cp", "-r", "./cds_utils", self.venv_home + "/lib/python3.6/site-packages/"] + py_ver_maj = sys.version_info.major + py_ver_min = sys.version_info.minor + command = ["cp", "-r", "./cds_utils", "{}/lib/python{}.{}/site-packages/".format(self.blueprint_dir, py_ver_maj,py_ver_min)] else: - command = ["pip", "install", package] + command = ["pip", "install", "--user", package] env = dict(os.environ) if "https_proxy" in os.environ: env['https_proxy'] = os.environ['https_proxy'] - self.logger.info("Using https_proxy: ", env['https_proxy'], extra=self.extra) - + self.logger.info("Using https_proxy: {}".format(env['https_proxy']), extra=self.extra) try: results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode()) results.append("\n") @@ -200,14 +271,13 @@ class CommandExecutorHandler(): def install_ansible_packages(self, package, results): self.logger.info( - "{} - Install Ansible Role package({}) in Python Virtual Environment".format(self.blueprint_id, package), extra=self.extra) - command = ["ansible-galaxy", "install", package, "-p", self.venv_home + "/Scripts/ansible/roles"] + "{} - Install Ansible Role package({}) in Python Virtual Environment".format(self.blueprint_name_version_uuid, package), extra=self.extra) + command = ["ansible-galaxy", "install", package, "-p", self.blueprint_dir + "/Scripts/ansible/roles"] env = dict(os.environ) if "http_proxy" in os.environ: # ansible galaxy uses https_proxy environment variable, but requires it to be set with http proxy value. env['https_proxy'] = os.environ['http_proxy'] - try: results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode()) results.append("\n") @@ -221,30 +291,30 @@ class CommandExecutorHandler(): # 'err_msg' indicates an error occurred. The presence of err_msg may not be fatal, # status should be set to False for fatal errors. def create_venv(self): - self.logger.info("{} - Create Python Virtual Environment".format(self.blueprint_id), extra=self.extra) + self.logger.info("{} - Create Python Virtual Environment".format(self.blueprint_name_version_uuid), extra=self.extra) try: - bin_dir = self.venv_home + "/bin" + bin_dir = self.blueprint_dir + "/bin" # venv doesn't populate the activate_this.py script, hence we use from virtualenv - venv.create(self.venv_home, with_pip=True, system_site_packages=True) + venv.create(self.blueprint_dir, with_pip=True, system_site_packages=True) virtualenv.writefile(os.path.join(bin_dir, "activate_this.py"), virtualenv.ACTIVATE_THIS) - self.logger.info("{} - Creation of Python Virtual Environment finished.".format(self.blueprint_id), extra=self.extra) + self.logger.info("{} - Creation of Python Virtual Environment finished.".format(self.blueprint_name_version_uuid), extra=self.extra) return utils.build_ret_data(True) except Exception as err: - err_msg = "{} - Failed to provision Python Virtual Environment. Error: {}".format(self.blueprint_id, err) + err_msg = "{} - Failed to provision Python Virtual Environment. Error: {}".format(self.blueprint_name_version_uuid, err) self.logger.info(err_msg, extra=self.extra) return utils.build_ret_data(False, error=err_msg) # return map cds_is_successful and err_msg. Status is True on success. err_msg may existence doesn't necessarily indicate fatal condition. # the 'status' should be set to False to indicate error. def activate_venv(self): - self.logger.info("{} - Activate Python Virtual Environment".format(self.blueprint_id), extra=self.extra) + self.logger.info("{} - Activate Python Virtual Environment".format(self.blueprint_name_version_uuid), extra=self.extra) # Fix: The python generated activate_this.py script concatenates the env bin dir to PATH on every call # eventually this process PATH variable was so big (128Kb) that no child process could be spawn # This script will remove all duplicates; while keeping the order of the PATH folders fixpathenvvar = "os.environ['PATH']=os.pathsep.join(list(dict.fromkeys(os.environ['PATH'].split(':'))))" - path = "%s/bin/activate_this.py" % self.venv_home + path = "%s/bin/activate_this.py" % self.blueprint_dir try: with open(path) as activate_this_script: exec (activate_this_script.read(), {'__file__': path}) @@ -252,17 +322,17 @@ class CommandExecutorHandler(): self.logger.info("Running with PATH : {}".format(os.environ['PATH']), extra=self.extra) return utils.build_ret_data(True) except Exception as err: - err_msg ="{} - Failed to activate Python Virtual Environment. Error: {}".format(self.blueprint_id, err) + err_msg ="{} - Failed to activate Python Virtual Environment. Error: {}".format(self.blueprint_name_version_uuid, err) self.logger.info( err_msg, extra=self.extra) return utils.build_ret_data(False, error=err_msg) def deactivate_venv(self): - self.logger.info("{} - Deactivate Python Virtual Environment".format(self.blueprint_id), extra=self.extra) + self.logger.info("{} - Deactivate Python Virtual Environment".format(self.blueprint_name_version_uuid), extra=self.extra) command = ["deactivate"] try: subprocess.run(command, check=True) except Exception as err: self.logger.info( - "{} - Failed to deactivate Python Virtual Environment. Error: {}".format(self.blueprint_id, err), extra=self.extra) + "{} - Failed to deactivate Python Virtual Environment. Error: {}".format(self.blueprint_name_version_uuid, err), extra=self.extra) diff --git a/ms/command-executor/src/main/python/command_executor_server.py b/ms/command-executor/src/main/python/command_executor_server.py index 175ddc7d3..057214306 100644 --- a/ms/command-executor/src/main/python/command_executor_server.py +++ b/ms/command-executor/src/main/python/command_executor_server.py @@ -27,8 +27,16 @@ class CommandExecutorServer(CommandExecutor_pb2_grpc.CommandExecutorServiceServi def __init__(self): self.logger = logging.getLogger(self.__class__.__name__) + def uploadBlueprint(self, request, context): + # handler for 'uploadBluleprint' call - extracts compressed cbaData to a bpname/bpver/bpuuid dir. + blueprint_name_version_uuid = utils.blueprint_name_version_uuid(request) + extra = utils.getExtraLogData(request) + self.logger.info("{} - Received uploadBlueprint request".format(blueprint_name_version_uuid), extra=extra) + handler = CommandExecutorHandler(request) + return handler.uploadBlueprint(request) + def prepareEnv(self, request, context): - blueprint_id = utils.get_blueprint_id(request) + blueprint_id = utils.blueprint_name_version_uuid(request) extra = utils.getExtraLogData(request) self.logger.info("{} - Received prepareEnv request".format(blueprint_id), extra=extra) self.logger.info(request, extra=extra) @@ -43,7 +51,7 @@ class CommandExecutorServer(CommandExecutor_pb2_grpc.CommandExecutorServiceServi return utils.build_grpc_response(request.requestId, prepare_env_response) def executeCommand(self, request, context): - blueprint_id = utils.get_blueprint_id(request) + blueprint_id = utils.blueprint_name_version_uuid(request) extra = utils.getExtraLogData(request) self.logger.info("{} - Received executeCommand request".format(blueprint_id), extra=extra) if os.environ.get('CE_DEBUG','false') == "true": @@ -59,4 +67,4 @@ class CommandExecutorServer(CommandExecutor_pb2_grpc.CommandExecutorServiceServi ret = utils.build_grpc_response(request.requestId, exec_cmd_response) self.logger.info("Payload returned : {}".format(exec_cmd_response), extra=extra) - return ret
\ No newline at end of file + return ret diff --git a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py index 5ee04927e..3e9773c6c 100644 --- a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py +++ b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py @@ -23,7 +23,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( package='org.onap.ccsdk.cds.controllerblueprints.command.api', syntax='proto3', serialized_options=_b('P\001'), - serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xbb\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0csubRequestId\x18\x08 \x01(\t\x12\x14\n\x0coriginatorId\x18\t \x01(\t\"\xfc\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0csubRequestId\x18\x08 \x01(\t\x12\x14\n\x0coriginatorId\x18\t \x01(\t\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xcb\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x03(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07payload\x18\x05 \x01(\t\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01*9\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x12\r\n\tutilities\x10\x02\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3') + serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xbb\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0csubRequestId\x18\x08 \x01(\t\x12\x14\n\x0coriginatorId\x18\t \x01(\t\"\xa9\x02\n\x14UploadBlueprintInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x14\n\x0csubRequestId\x18\x03 \x01(\t\x12\x14\n\x0coriginatorId\x18\x04 \x01(\t\x12\x15\n\rcorrelationId\x18\x05 \x01(\t\x12\x0f\n\x07timeOut\x18\x06 \x01(\x05\x12\x13\n\x0b\x61rchiveType\x18\x07 \x01(\t\x12-\n\ttimestamp\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x62inData\x18\t \x01(\x0c\"\xd5\x01\n\x15UploadBlueprintOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x14\n\x0csubRequestId\x18\x02 \x01(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07payload\x18\x05 \x01(\t\"\xfc\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0csubRequestId\x18\x08 \x01(\t\x12\x14\n\x0coriginatorId\x18\t \x01(\t\"U\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\x12\x15\n\rblueprintUUID\x18\x03 \x01(\t\"\xcb\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x03(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07payload\x18\x05 \x01(\t\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01*9\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x12\r\n\tutilities\x10\x02\x32\xfc\x03\n\x16\x43ommandExecutorService\x12\xa8\x01\n\x0fuploadBlueprint\x12I.org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput\x1aJ.org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintOutput\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3') , dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,]) @@ -44,8 +44,8 @@ _RESPONSESTATUS = _descriptor.EnumDescriptor( ], containing_type=None, serialized_options=None, - serialized_start=1221, - serialized_end=1263, + serialized_start=1760, + serialized_end=1802, ) _sym_db.RegisterEnumDescriptor(_RESPONSESTATUS) @@ -71,8 +71,8 @@ _PACKAGETYPE = _descriptor.EnumDescriptor( ], containing_type=None, serialized_options=None, - serialized_start=1265, - serialized_end=1322, + serialized_start=1804, + serialized_end=1861, ) _sym_db.RegisterEnumDescriptor(_PACKAGETYPE) @@ -172,6 +172,152 @@ _EXECUTIONINPUT = _descriptor.Descriptor( ) +_UPLOADBLUEPRINTINPUT = _descriptor.Descriptor( + name='UploadBlueprintInput', + full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='identifiers', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput.identifiers', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='requestId', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput.requestId', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='subRequestId', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput.subRequestId', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='originatorId', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput.originatorId', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='correlationId', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput.correlationId', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timeOut', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput.timeOut', index=5, + number=6, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='archiveType', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput.archiveType', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput.timestamp', index=7, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='binData', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput.binData', index=8, + number=9, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=460, + serialized_end=757, +) + + +_UPLOADBLUEPRINTOUTPUT = _descriptor.Descriptor( + name='UploadBlueprintOutput', + full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintOutput', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='requestId', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintOutput.requestId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='subRequestId', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintOutput.subRequestId', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='status', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintOutput.status', index=2, + number=3, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintOutput.timestamp', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='payload', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintOutput.payload', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=760, + serialized_end=973, +) + + _PREPAREENVINPUT = _descriptor.Descriptor( name='PrepareEnvInput', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput', @@ -254,8 +400,8 @@ _PREPAREENVINPUT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=460, - serialized_end=840, + serialized_start=976, + serialized_end=1356, ) @@ -280,6 +426,13 @@ _IDENTIFIERS = _descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='blueprintUUID', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers.blueprintUUID', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -292,8 +445,8 @@ _IDENTIFIERS = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=842, - serialized_end=904, + serialized_start=1358, + serialized_end=1443, ) @@ -351,8 +504,8 @@ _EXECUTIONOUTPUT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=907, - serialized_end=1110, + serialized_start=1446, + serialized_end=1649, ) @@ -389,13 +542,17 @@ _PACKAGES = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1112, - serialized_end=1219, + serialized_start=1651, + serialized_end=1758, ) _EXECUTIONINPUT.fields_by_name['identifiers'].message_type = _IDENTIFIERS _EXECUTIONINPUT.fields_by_name['properties'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT _EXECUTIONINPUT.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +_UPLOADBLUEPRINTINPUT.fields_by_name['identifiers'].message_type = _IDENTIFIERS +_UPLOADBLUEPRINTINPUT.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +_UPLOADBLUEPRINTOUTPUT.fields_by_name['status'].enum_type = _RESPONSESTATUS +_UPLOADBLUEPRINTOUTPUT.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP _PREPAREENVINPUT.fields_by_name['identifiers'].message_type = _IDENTIFIERS _PREPAREENVINPUT.fields_by_name['packages'].message_type = _PACKAGES _PREPAREENVINPUT.fields_by_name['properties'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT @@ -404,6 +561,8 @@ _EXECUTIONOUTPUT.fields_by_name['status'].enum_type = _RESPONSESTATUS _EXECUTIONOUTPUT.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP _PACKAGES.fields_by_name['type'].enum_type = _PACKAGETYPE DESCRIPTOR.message_types_by_name['ExecutionInput'] = _EXECUTIONINPUT +DESCRIPTOR.message_types_by_name['UploadBlueprintInput'] = _UPLOADBLUEPRINTINPUT +DESCRIPTOR.message_types_by_name['UploadBlueprintOutput'] = _UPLOADBLUEPRINTOUTPUT DESCRIPTOR.message_types_by_name['PrepareEnvInput'] = _PREPAREENVINPUT DESCRIPTOR.message_types_by_name['Identifiers'] = _IDENTIFIERS DESCRIPTOR.message_types_by_name['ExecutionOutput'] = _EXECUTIONOUTPUT @@ -419,6 +578,20 @@ ExecutionInput = _reflection.GeneratedProtocolMessageType('ExecutionInput', (_me )) _sym_db.RegisterMessage(ExecutionInput) +UploadBlueprintInput = _reflection.GeneratedProtocolMessageType('UploadBlueprintInput', (_message.Message,), dict( + DESCRIPTOR = _UPLOADBLUEPRINTINPUT, + __module__ = 'CommandExecutor_pb2' + # @@protoc_insertion_point(class_scope:org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintInput) + )) +_sym_db.RegisterMessage(UploadBlueprintInput) + +UploadBlueprintOutput = _reflection.GeneratedProtocolMessageType('UploadBlueprintOutput', (_message.Message,), dict( + DESCRIPTOR = _UPLOADBLUEPRINTOUTPUT, + __module__ = 'CommandExecutor_pb2' + # @@protoc_insertion_point(class_scope:org.onap.ccsdk.cds.controllerblueprints.command.api.UploadBlueprintOutput) + )) +_sym_db.RegisterMessage(UploadBlueprintOutput) + PrepareEnvInput = _reflection.GeneratedProtocolMessageType('PrepareEnvInput', (_message.Message,), dict( DESCRIPTOR = _PREPAREENVINPUT, __module__ = 'CommandExecutor_pb2' @@ -456,13 +629,22 @@ _COMMANDEXECUTORSERVICE = _descriptor.ServiceDescriptor( file=DESCRIPTOR, index=0, serialized_options=None, - serialized_start=1325, - serialized_end=1662, + serialized_start=1864, + serialized_end=2372, methods=[ _descriptor.MethodDescriptor( + name='uploadBlueprint', + full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.CommandExecutorService.uploadBlueprint', + index=0, + containing_service=None, + input_type=_UPLOADBLUEPRINTINPUT, + output_type=_UPLOADBLUEPRINTOUTPUT, + serialized_options=None, + ), + _descriptor.MethodDescriptor( name='prepareEnv', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.CommandExecutorService.prepareEnv', - index=0, + index=1, containing_service=None, input_type=_PREPAREENVINPUT, output_type=_EXECUTIONOUTPUT, @@ -471,7 +653,7 @@ _COMMANDEXECUTORSERVICE = _descriptor.ServiceDescriptor( _descriptor.MethodDescriptor( name='executeCommand', full_name='org.onap.ccsdk.cds.controllerblueprints.command.api.CommandExecutorService.executeCommand', - index=1, + index=2, containing_service=None, input_type=_EXECUTIONINPUT, output_type=_EXECUTIONOUTPUT, @@ -482,4 +664,4 @@ _sym_db.RegisterServiceDescriptor(_COMMANDEXECUTORSERVICE) DESCRIPTOR.services_by_name['CommandExecutorService'] = _COMMANDEXECUTORSERVICE -# @@protoc_insertion_point(module_scope)
\ No newline at end of file +# @@protoc_insertion_point(module_scope) diff --git a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2_grpc.py b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2_grpc.py index 3ea8ec4d6..b5c2c26ff 100644 --- a/ms/command-executor/src/main/python/proto/CommandExecutor_pb2_grpc.py +++ b/ms/command-executor/src/main/python/proto/CommandExecutor_pb2_grpc.py @@ -1,7 +1,7 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc -import proto.CommandExecutor_pb2 as CommandExecutor__pb2 +import CommandExecutor_pb2 as CommandExecutor__pb2 class CommandExecutorServiceStub(object): @@ -14,6 +14,11 @@ class CommandExecutorServiceStub(object): Args: channel: A grpc.Channel. """ + self.uploadBlueprint = channel.unary_unary( + '/org.onap.ccsdk.cds.controllerblueprints.command.api.CommandExecutorService/uploadBlueprint', + request_serializer=CommandExecutor__pb2.UploadBlueprintInput.SerializeToString, + response_deserializer=CommandExecutor__pb2.UploadBlueprintOutput.FromString, + ) self.prepareEnv = channel.unary_unary( '/org.onap.ccsdk.cds.controllerblueprints.command.api.CommandExecutorService/prepareEnv', request_serializer=CommandExecutor__pb2.PrepareEnvInput.SerializeToString, @@ -30,16 +35,23 @@ class CommandExecutorServiceServicer(object): # missing associated documentation comment in .proto file pass + def uploadBlueprint(self, request, context): + """rpc to upload the CBA + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def prepareEnv(self, request, context): - # missing associated documentation comment in .proto file - pass + """prepare Python environment + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def executeCommand(self, request, context): - # missing associated documentation comment in .proto file - pass + """execute the actual command. + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -47,6 +59,11 @@ class CommandExecutorServiceServicer(object): def add_CommandExecutorServiceServicer_to_server(servicer, server): rpc_method_handlers = { + 'uploadBlueprint': grpc.unary_unary_rpc_method_handler( + servicer.uploadBlueprint, + request_deserializer=CommandExecutor__pb2.UploadBlueprintInput.FromString, + response_serializer=CommandExecutor__pb2.UploadBlueprintOutput.SerializeToString, + ), 'prepareEnv': grpc.unary_unary_rpc_method_handler( servicer.prepareEnv, request_deserializer=CommandExecutor__pb2.PrepareEnvInput.FromString, diff --git a/ms/command-executor/src/main/python/utils.py b/ms/command-executor/src/main/python/utils.py index c624a3d2e..e7924a936 100644 --- a/ms/command-executor/src/main/python/utils.py +++ b/ms/command-executor/src/main/python/utils.py @@ -23,16 +23,34 @@ CDS_IS_SUCCESSFUL_KEY = "cds_is_successful" ERR_MSG_KEY = "err_msg" RESULTS_KEY = "results" RESULTS_LOG_KEY = "results_log" -RESPONSE_MAX_SIZE = 4 * 1024 * 1024 # 4Mb +REUPLOAD_CBA_KEY = "reupload_cba" +RESPONSE_MAX_SIZE = 4 * 1024 * 1024 # 4Mb -def get_blueprint_id(request): + +def blueprint_name_version_uuid(request): blueprint_name = request.identifiers.blueprintName blueprint_version = request.identifiers.blueprintVersion - return blueprint_name + '/' + blueprint_version + blueprint_uuid = request.identifiers.blueprintUUID + return blueprint_name + '/' + blueprint_version + '/' + blueprint_uuid + +def get_blueprint_name(request): + return request.identifiers.blueprintName + +def get_blueprint_version(request): + return request.identifiers.blueprintVersion + +def get_blueprint_uuid(request): + return request.identifiers.blueprintUUID def get_blueprint_timeout(request): return request.timeOut +def get_blueprint_requestid(request): + return request.requestId + +def get_blueprint_subRequestId(request): + return request.subRequestId + # Create a response for grpc. Fills in the timestamp as well as removes cds_is_successful element def build_grpc_response(request_id, response): if response[CDS_IS_SUCCESSFUL_KEY]: @@ -50,36 +68,51 @@ def build_grpc_response(request_id, response): timestamp.GetCurrentTime() execution_output = CommandExecutor_pb2.ExecutionOutput(requestId=request_id, - response=logs, - status=status, - payload=payload, - timestamp=timestamp) + response=logs, + status=status, + payload=payload, + timestamp=timestamp) return truncate_execution_output(execution_output) +def build_grpc_blueprint_upload_response(request_id, subrequest_id, success=True, payload=[]): + timestamp = Timestamp() + timestamp.GetCurrentTime() + return CommandExecutor_pb2.UploadBlueprintOutput(requestId=request_id, + subRequestId=subrequest_id, + status=CommandExecutor_pb2.SUCCESS if success else CommandExecutor_pb2.FAILURE, + timestamp=timestamp, + payload=json.dumps(payload)) + # build a ret data structure used to populate the ExecutionOutput -def build_ret_data(cds_is_successful, results_log=[], error=None): +def build_ret_data(cds_is_successful, results_log=[], error=None, reupload_cba = False): ret_data = { CDS_IS_SUCCESSFUL_KEY: cds_is_successful, RESULTS_LOG_KEY: results_log } if error: ret_data[ERR_MSG_KEY] = error + # CBA needs to be reuploaded case: + if reupload_cba: + ret_data[REUPLOAD_CBA_KEY] = True return ret_data + # Truncate execution logs to make sure gRPC response doesn't exceed the gRPC buffer capacity def truncate_execution_output(execution_output): sum_truncated_chars = 0 if execution_output.ByteSize() > RESPONSE_MAX_SIZE: while execution_output.ByteSize() > RESPONSE_MAX_SIZE: - removed_item = execution_output.response.pop() - sum_truncated_chars += len(removed_item) - execution_output.response.append("[...] TRUNCATED CHARS : {}".format(sum_truncated_chars)) + removed_item = execution_output.response.pop() + sum_truncated_chars += len(removed_item) + execution_output.response.append( + "[...] TRUNCATED CHARS : {}".format(sum_truncated_chars)) return execution_output # Read temp file 'outputfile' into results_log and split out the returned payload into payload_result -def parse_cmd_exec_output(outputfile, logger, payload_result, results_log, extra): +def parse_cmd_exec_output(outputfile, logger, payload_result, results_log, + extra): payload_section = [] is_payload_section = False outputfile.seek(0) @@ -103,12 +136,13 @@ def parse_cmd_exec_output(outputfile, logger, payload_result, results_log, extra else: payload_section.append(output.strip()) + def getExtraLogData(request=None): - extra = {'request_id' : '', 'subrequest_id' : '', 'originator_id': ''} - if request is not None: - extra = { - 'request_id' : request.requestId, - 'subrequest_id' : request.subRequestId, - 'originator_id': request.originatorId - } - return extra
\ No newline at end of file + extra = {'request_id': '', 'subrequest_id': '', 'originator_id': ''} + if request is not None: + extra = { + 'request_id': request.requestId, + 'subrequest_id': request.subRequestId, + 'originator_id': request.originatorId + } + return extra |