From 0094e13ddec035faf7f80943783943003f12889a Mon Sep 17 00:00:00 2001 From: Oleg Mitsura Date: Thu, 19 May 2022 11:05:13 -0400 Subject: CCSDK-3671 add workflows list for grpc Issue-ID: CCSDK-3671 1. added grpc 'getWorkflows' for BlueprintManagement 2. during CBA upload, the workflows get cached to BLUEPRINT_MODEL.workflows 3. reworked HTTP endpoint to use above "/workflows/blueprint-name/{name}/version/{version}" 4. If CDS is upgraded, with existing CBAs present, fallback by parsing the CBA instead of DB lookup Signed-off-by: Oleg Mitsura Change-Id: I68bebfe23c0b16ea288512f1087bfe1ceef57686 --- .../designer/api/BluePrintManagementGRPCHandler.kt | 30 ++++++ .../designer/api/BlueprintModelController.kt | 9 +- .../designer/api/handler/BluePrintModelHandler.kt | 112 ++++++++++++++------- 3 files changed, 106 insertions(+), 45 deletions(-) (limited to 'ms/blueprintsprocessor/modules/inbounds') diff --git a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandler.kt b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandler.kt index 54f8dbc9d..0dc0941c2 100644 --- a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandler.kt +++ b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandler.kt @@ -29,10 +29,12 @@ import org.onap.ccsdk.cds.controllerblueprints.common.api.Status import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException import org.onap.ccsdk.cds.controllerblueprints.core.asJsonString +import org.onap.ccsdk.cds.controllerblueprints.core.asJsonType import org.onap.ccsdk.cds.controllerblueprints.core.emptyTONull import org.onap.ccsdk.cds.controllerblueprints.core.utils.currentTimestamp import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintBootstrapInput import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintDownloadInput +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintGetWorkflowsInput import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementOutput import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementServiceGrpc import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintRemoveInput @@ -241,6 +243,33 @@ open class BluePrintManagementGRPCHandler( } } + @PreAuthorize("hasRole('USER')") + override fun getWorkflows( + request: BluePrintGetWorkflowsInput, + responseObserver: StreamObserver + ) { + runBlocking { + val blueprintName = request.blueprintName + val blueprintVersion = request.blueprintVersion + val blueprint = "blueprint $blueprintName:$blueprintVersion" + + log.info("request(${request.commonHeader.requestId}): Received getWorkflows $blueprint") + try { + val workflowsSetJson = bluePrintModelHandler.getWorkflowNamesFromRepository(blueprintName, blueprintVersion) + responseObserver.onNext(successStatus(request.commonHeader, mapOf("workflows" to workflowsSetJson).asJsonType().toString())) + } catch (e: Exception) { + responseObserver.onNext( + failStatus( + request.commonHeader, + "request(${request.commonHeader.requestId}): Failed to get workflows for ($blueprint)", e + ) + ) + } finally { + responseObserver.onCompleted() + } + } + } + private fun outputWithFileBytes(header: CommonHeader, byteArray: ByteArray): BluePrintManagementOutput = BluePrintManagementOutput.newBuilder() .setCommonHeader(header) @@ -258,6 +287,7 @@ open class BluePrintManagementGRPCHandler( private fun successStatus(header: CommonHeader, propertyContent: String? = null): BluePrintManagementOutput { // Populate Response Payload val propertiesBuilder = BluePrintManagementOutput.newBuilder().propertiesBuilder + // propertyContent is expected to have a string which contains a JSON map propertyContent?.let { JsonFormat.parser().merge(propertyContent, propertiesBuilder) } diff --git a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BlueprintModelController.kt b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BlueprintModelController.kt index 66d4b0e16..777a21457 100644 --- a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BlueprintModelController.kt +++ b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BlueprintModelController.kt @@ -393,10 +393,7 @@ open class BlueprintModelController(private val bluePrintModelHandler: BluePrint @PostMapping( path = arrayOf("/workflow-spec"), - produces = arrayOf( - MediaType - .APPLICATION_JSON_VALUE - ), + produces = arrayOf(MediaType.APPLICATION_JSON_VALUE), consumes = arrayOf(MediaType.APPLICATION_JSON_VALUE) ) @ApiOperation( @@ -418,9 +415,7 @@ open class BlueprintModelController(private val bluePrintModelHandler: BluePrint } @GetMapping( - path = arrayOf( - "/workflows/blueprint-name/{name}/version/{version}" - ), + path = arrayOf("/workflows/blueprint-name/{name}/version/{version}"), produces = arrayOf(MediaType.APPLICATION_JSON_VALUE) ) @ApiOperation( diff --git a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/handler/BluePrintModelHandler.kt b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/handler/BluePrintModelHandler.kt index 7bbaa8c59..a5fcd322c 100644 --- a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/handler/BluePrintModelHandler.kt +++ b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/handler/BluePrintModelHandler.kt @@ -39,6 +39,7 @@ import org.onap.ccsdk.cds.controllerblueprints.core.data.DataType import org.onap.ccsdk.cds.controllerblueprints.core.data.PropertyDefinition import org.onap.ccsdk.cds.controllerblueprints.core.deleteNBDir import org.onap.ccsdk.cds.controllerblueprints.core.httpProcessorException +import org.onap.ccsdk.cds.controllerblueprints.core.data.Workflow import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintCatalogService import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintEnhancerService import org.onap.ccsdk.cds.controllerblueprints.core.logger @@ -107,42 +108,78 @@ open class BluePrintModelHandler( } @Throws(BluePrintException::class) - open suspend fun prepareWorkFlowSpec(req: WorkFlowSpecRequest): - WorkFlowSpecResponse { - val basePath = blueprintsProcessorCatalogService.getFromDatabase( - req - .blueprintName, - req.version - ) - log.info("blueprint base path $basePath") + private suspend fun getBlueprintCtxByNameAndVersion(blueprintName: String, version: String): BluePrintContext { + val basePath = blueprintsProcessorCatalogService.getFromDatabase(blueprintName, version) + log.info("blueprint base path $basePath for blueprint: $blueprintName version:$version") + return BluePrintMetadataUtils.getBluePrintContext(basePath.toString()) + } + + @Throws(BluePrintException::class) + /** + * Try to get workflows cached from the BLUEPRINT_MODEL table first, + * failing that, load the CBA from the filesystem and extract workflows. + * The second case is possible during update scenario - current CDS DB already contains desired CBAs and reupload + * is not feasible. + */ + open suspend fun getWorkflowsFromRepository(name: String, version: String): Map { + var workflowsFromCache: Map + try { + workflowsFromCache = blueprintModelRepository.findByArtifactNameAndArtifactVersion(name, version)?.workflows!! + if (workflowsFromCache.isEmpty()) { + log.info("findByArtifactNameAndArtifactVersion did not return list of workflows for blueprintName:($name) version:($version), falling back to loading CBA from filesystem.") + workflowsFromCache = getBlueprintCtxByNameAndVersion(name, version).workflows()!! + // TODO: does it make sense to update the BLUEPRINT_MODEL workflows in this case or just wait for CBA reupload? + } + } catch (e: Exception) { + throw BluePrintException("Failed to get workflows from DB cache or by reading CBA", e) + } + return workflowsFromCache + } + + // lookup workflows list from field. + open suspend fun getWorkflowNamesFromRepository(name: String, version: String): Set { + return getWorkflowsFromRepository(name, version).keys + } - val blueprintContext = BluePrintMetadataUtils.getBluePrintContext(basePath.toString()) - val workFlow = blueprintContext.workflowByName(req.workflowName) + @Throws(BluePrintException::class) + open suspend fun prepareWorkFlowSpec(req: WorkFlowSpecRequest): WorkFlowSpecResponse { + val basePath = blueprintsProcessorCatalogService.getFromDatabase(req.blueprintName, req.version) + log.info("blueprint base path $basePath") - val wfRes = WorkFlowSpecResponse() - wfRes.blueprintName = req.blueprintName - wfRes.version = req.version + val blueprintContext = BluePrintMetadataUtils.getBluePrintContext(basePath.toString()) + val workFlow = blueprintContext.workflowByName(req.workflowName) - val workFlowData = WorkFlowData() - workFlowData.workFlowName = req.workflowName - workFlowData.inputs = workFlow.inputs - workFlowData.outputs = workFlow.outputs - wfRes.workFlowData = workFlowData + val wfRes = WorkFlowSpecResponse() + wfRes.blueprintName = req.blueprintName + wfRes.version = req.version - if (workFlow.inputs != null) { - for ((k, v) in workFlow.inputs!!) { - addPropertyInfo(k, v, blueprintContext, wfRes) - } + val workFlowData = WorkFlowData() + workFlowData.workFlowName = req.workflowName + workFlowData.inputs = workFlow.inputs + workFlowData.outputs = workFlow.outputs + + if (workFlow.inputs != null) { + for ((k, v) in workFlow.inputs!!) { + addPropertyInfo(v, blueprintContext, wfRes) } + } - if (workFlow.outputs != null) { - for ((k, v) in workFlow.outputs!!) { - addPropertyInfo(k, v, blueprintContext, wfRes) - } + if (workFlow.outputs != null) { + for ((k, v) in workFlow.outputs!!) { + addPropertyInfo(k, v, blueprintContext, wfRes) } + } - return wfRes + wfRes.workFlowData = workFlowData + return wfRes + } + + private fun addPropertyInfo(prop: PropertyDefinition, ctx: BluePrintContext, res: WorkFlowSpecResponse) { + addDataType(prop.type, ctx, res) + if (prop.entrySchema != null && prop.entrySchema!!.type != null) { + addDataType(prop.entrySchema!!.type, ctx, res) } + } private fun addPropertyInfo(propName: String, prop: PropertyDefinition, ctx: BluePrintContext, res: WorkFlowSpecResponse) { updatePropertyInfo(propName, prop, ctx, res) @@ -192,23 +229,19 @@ open class BluePrintModelHandler( } } + // wrap CBA workflows list in WorkflowsResponse object @Throws(BluePrintException::class) open suspend fun getWorkflowNames(name: String, version: String): WorkFlowsResponse { - val basePath = blueprintsProcessorCatalogService.getFromDatabase( - name, version + var workflows = getWorkflowsFromRepository(name, version) + if (workflows == null) throw httpProcessorException( + ErrorCatalogCodes.RESOURCE_NOT_FOUND, DesignerApiDomains.DESIGNER_API, + "Failed to find workflows list for blueprint: ($name) version: ($version) from filesystem." ) - log.info("blueprint base path $basePath") var res = WorkFlowsResponse() res.blueprintName = name res.version = version - - val blueprintContext = BluePrintMetadataUtils.getBluePrintContext( - basePath.toString() - ) - if (blueprintContext.workflows() != null) { - res.workflows = blueprintContext.workflows()!!.keys - } + res.workflows = workflows.keys.toMutableSet() return res } @@ -242,6 +275,7 @@ open class BluePrintModelHandler( try { return upload(filePart, false) } catch (e: IOException) { + log.error("saveBlueprintModel fails ${e.message}", e) throw httpProcessorException( ErrorCatalogCodes.IO_FILE_INTERRUPT, DesignerApiDomains.DESIGNER_API, "Error in Save CBA: ${e.message}", e.errorCauseOrDefault() @@ -527,13 +561,15 @@ open class BluePrintModelHandler( val enhancedByteArray = enrichBlueprintFileSource(filePart) return upload(enhancedByteArray, true) } catch (e: BluePrintProcessorException) { - e.http(ErrorCatalogCodes.IO_FILE_INTERRUPT) val errorMsg = "Error while enhancing and uploading the CBA package." + log.error(errorMsg, e) + e.http(ErrorCatalogCodes.IO_FILE_INTERRUPT) throw e.updateErrorMessage( DesignerApiDomains.DESIGNER_API, errorMsg, "Wrong CBA file provided, please verify the source CBA." ) } catch (e: Exception) { + log.error("Error enriching/uploading CBA", e) throw httpProcessorException( ErrorCatalogCodes.IO_FILE_INTERRUPT, DesignerApiDomains.DESIGNER_API, "EnrichBlueprint: ${e.message}", e.errorCauseOrDefault() -- cgit 1.2.3-korg