From 5c186597021771a83f1a0faae5b9955123171c84 Mon Sep 17 00:00:00 2001 From: Brinda Santh Date: Wed, 20 Nov 2019 17:04:52 -0500 Subject: Bootstrap Rest and GRPC API CDS loads model types, dictionaries and CBA multiple times based on cluster size. This commits avoid auto loading, and provides an API to bootstrap, if necessity. Issue-ID: CCSDK-1950 Signed-off-by: Brinda Santh Change-Id: Ibe6266ebfd399efbaf0c5ccb1019c3a73b08ddac --- .../src/main/resources/application-dev.properties | 6 +-- .../src/main/resources/application.properties | 11 ++--- .../designer/api/BluePrintManagementGRPCHandler.kt | 23 +++++++++++ .../designer/api/BlueprintModelController.kt | 15 +++++-- .../designer/api/DesignerApiData.kt | 8 ++++ .../designer/api/handler/BluePrintModelHandler.kt | 36 +++++++++++----- .../api/load/BluePrintDatabaseLoadService.kt | 48 +++++++--------------- .../api/BluePrintManagementGRPCHandlerTest.kt | 27 ++++++++++++ 8 files changed, 114 insertions(+), 60 deletions(-) (limited to 'ms/blueprintsprocessor') diff --git a/ms/blueprintsprocessor/application/src/main/resources/application-dev.properties b/ms/blueprintsprocessor/application/src/main/resources/application-dev.properties index 485b0bfee..fc2e7f244 100755 --- a/ms/blueprintsprocessor/application/src/main/resources/application-dev.properties +++ b/ms/blueprintsprocessor/application/src/main/resources/application-dev.properties @@ -34,14 +34,10 @@ resourceSourceMappings=processor-db=source-db,input=source-input,default=source- blueprintsprocessor.blueprintDeployPath=blueprints/deploy blueprintsprocessor.blueprintArchivePath=blueprints/archive blueprintsprocessor.blueprintWorkingPath=blueprints/work + # Controller Blueprint Load Configurations -# blueprints.load.initial-data may be overridden by ENV variables -blueprintsprocessor.loadInitialData=false -blueprintsprocessor.loadBluePrint=false blueprintsprocessor.loadBluePrintPaths=./../../../components/model-catalog/blueprint-model/service-blueprint -blueprintsprocessor.loadModelType=true blueprintsprocessor.loadModeTypePaths=./../../../components/model-catalog/definition-type/starter-type -blueprintsprocessor.loadResourceDictionary=true blueprintsprocessor.loadResourceDictionaryPaths=./../../../components/model-catalog/resource-dictionary/starter-dictionary # CBA file extension diff --git a/ms/blueprintsprocessor/application/src/main/resources/application.properties b/ms/blueprintsprocessor/application/src/main/resources/application.properties index 6708dcf69..bd713080f 100755 --- a/ms/blueprintsprocessor/application/src/main/resources/application.properties +++ b/ms/blueprintsprocessor/application/src/main/resources/application.properties @@ -24,14 +24,9 @@ blueprintsprocessor.blueprintArchivePath=/opt/app/onap/blueprints/archive blueprintsprocessor.blueprintWorkingPath=/opt/app/onap/blueprints/working # Controller Blueprint Load Configurations -# blueprints.load.initial-data may be overridden by ENV variables -blueprintsprocessor.loadInitialData=false -blueprintsprocessor.loadBluePrint=false -blueprintsprocessor.loadBluePrintPaths=/opt/app/onap/model-catalog/blueprint-model/service-blueprint -blueprintsprocessor.loadModelType=false -blueprintsprocessor.loadModeTypePaths=/opt/app/onap/model-catalog/definition-type/starter-type -blueprintsprocessor.loadResourceDictionary=false -blueprintsprocessor.loadResourceDictionaryPaths=/opt/app/onap/model-catalog/resource-dictionary/starter-dictionary +blueprintsprocessor.loadBluePrintPaths=/opt/app/onap/model-catalog/blueprint-model +blueprintsprocessor.loadModeTypePaths=/opt/app/onap/model-catalog/definition-type +blueprintsprocessor.loadResourceDictionaryPaths=/opt/app/onap/model-catalog/resource-dictionary # CBA file extension controllerblueprints.loadCbaExtension=zip diff --git a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandler.kt b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandler.kt index 0f804b8b2..5f2a6dfb6 100644 --- a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandler.kt +++ b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandler.kt @@ -160,6 +160,29 @@ open class BluePrintManagementGRPCHandler(private val bluePrintModelHandler: Blu } } + override fun bootstrapBlueprint(request: BluePrintBootstrapInput, + responseObserver: StreamObserver) { + runBlocking { + try { + log.info("request(${request.commonHeader.requestId}): Received bootstrap request") + val bootstrapRequest = BootstrapRequest().apply { + loadModelType = request.loadModelType + loadResourceDictionary = request.loadResourceDictionary + loadCBA = request.loadCBA + } + /** Perform bootstrap of Model Types, Resource Definitions and CBA */ + bluePrintModelHandler.bootstrapBlueprint(bootstrapRequest) + responseObserver.onNext(successStatus(request.commonHeader)) + + } catch (e: Exception) { + responseObserver.onNext(failStatus(request.commonHeader, + "request(${request.commonHeader.requestId}): Failed to bootstrap", e)) + } finally { + responseObserver.onCompleted() + } + } + } + private fun outputWithFileBytes(header: CommonHeader, byteArray: ByteArray): BluePrintManagementOutput = BluePrintManagementOutput.newBuilder() .setCommonHeader(header) diff --git a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BlueprintModelController.kt b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BlueprintModelController.kt index 94c095047..5ef50b09f 100644 --- a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BlueprintModelController.kt +++ b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BlueprintModelController.kt @@ -35,9 +35,7 @@ import org.springframework.http.ResponseEntity import org.springframework.http.codec.multipart.FilePart import org.springframework.security.access.prepost.PreAuthorize import org.springframework.web.bind.annotation.* -import reactor.core.publisher.Flux import reactor.core.publisher.Mono -import java.time.Duration /** * BlueprintModelController Purpose: Handle controllerBlueprint API request @@ -49,6 +47,15 @@ import java.time.Duration @RequestMapping("/api/v1/blueprint-model") open class BlueprintModelController(private val bluePrintModelHandler: BluePrintModelHandler) { + @PostMapping(path = arrayOf("/bootstrap"), produces = arrayOf(MediaType.APPLICATION_JSON_VALUE), + consumes = arrayOf(MediaType.APPLICATION_JSON_VALUE)) + @ResponseBody + @Throws(BluePrintException::class) + @PreAuthorize("hasRole('USER')") + fun bootstrap(@RequestBody bootstrapRequest: BootstrapRequest): Mono = monoMdc { + bluePrintModelHandler.bootstrapBlueprint(bootstrapRequest) + } + @PostMapping("", produces = [MediaType.APPLICATION_JSON_VALUE], consumes = [MediaType.MULTIPART_FORM_DATA_VALUE]) @ResponseBody @Throws(BluePrintException::class) @@ -78,7 +85,7 @@ open class BlueprintModelController(private val bluePrintModelHandler: BluePrint @ResponseBody @PreAuthorize("hasRole('USER')") fun allBlueprintModelMetaData(@NotNull @PathVariable(value = "keyword") keyWord: String): List { - return this.bluePrintModelHandler.searchBluePrintModelsByKeyWord(keyWord) + return this.bluePrintModelHandler.searchBluePrintModelsByKeyWord(keyWord) } @@ -91,7 +98,7 @@ open class BlueprintModelController(private val bluePrintModelHandler: BluePrint @RequestParam(defaultValue = "DATE") sort: BlueprintSortByOption ): Page { val pageRequest = PageRequest.of(offset, limit, Sort.Direction.ASC, sort.columnName) - return this.bluePrintModelHandler.searchBluePrintModelsByKeyWordPaged(keyWord,pageRequest) + return this.bluePrintModelHandler.searchBluePrintModelsByKeyWordPaged(keyWord, pageRequest) } diff --git a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/DesignerApiData.kt b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/DesignerApiData.kt index 5259ee1c9..346015bc4 100644 --- a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/DesignerApiData.kt +++ b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/DesignerApiData.kt @@ -1,5 +1,6 @@ /* * Copyright © 2019 IBM. + * Modifications Copyright © 2018-2019 AT&T Intellectual Property. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +26,13 @@ import org.onap.ccsdk.cds.controllerblueprints.resource.dict.ResourceAssignment import java.io.Serializable import java.util.* + +class BootstrapRequest { + var loadModelType: Boolean = false + var loadResourceDictionary: Boolean = false + var loadCBA: Boolean = false +} + /** * ArtifactRequest.java Purpose: Provide Configuration Generator ArtifactRequest Model * diff --git a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/handler/BluePrintModelHandler.kt b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/handler/BluePrintModelHandler.kt index 216648369..5f7eacee8 100644 --- a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/handler/BluePrintModelHandler.kt +++ b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/handler/BluePrintModelHandler.kt @@ -24,6 +24,8 @@ import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.domain.BlueprintModelSe import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.repository.BlueprintModelContentRepository import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.repository.BlueprintModelRepository import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.repository.BlueprintModelSearchRepository +import org.onap.ccsdk.cds.blueprintsprocessor.designer.api.BootstrapRequest +import org.onap.ccsdk.cds.blueprintsprocessor.designer.api.load.BluePrintDatabaseLoadService import org.onap.ccsdk.cds.blueprintsprocessor.designer.api.utils.BluePrintEnhancerUtils import org.onap.ccsdk.cds.controllerblueprints.core.* import org.onap.ccsdk.cds.controllerblueprints.core.config.BluePrintLoadConfiguration @@ -36,6 +38,7 @@ import org.springframework.core.io.ByteArrayResource import org.springframework.core.io.Resource import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Pageable import org.springframework.http.HttpHeaders import org.springframework.http.MediaType import org.springframework.http.ResponseEntity @@ -44,7 +47,6 @@ import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import java.io.IOException import java.util.* -import org.springframework.data.domain.Pageable /** @@ -55,7 +57,8 @@ import org.springframework.data.domain.Pageable */ @Service -open class BluePrintModelHandler(private val blueprintsProcessorCatalogService: BluePrintCatalogService, +open class BluePrintModelHandler(private val bluePrintDatabaseLoadService: BluePrintDatabaseLoadService, + private val blueprintsProcessorCatalogService: BluePrintCatalogService, private val bluePrintLoadConfiguration: BluePrintLoadConfiguration, private val blueprintModelSearchRepository: BlueprintModelSearchRepository, private val blueprintModelRepository: BlueprintModelRepository, @@ -64,6 +67,22 @@ open class BluePrintModelHandler(private val blueprintsProcessorCatalogService: private val log = logger(BluePrintModelHandler::class) + + open suspend fun bootstrapBlueprint(bootstrapRequest: BootstrapRequest) { + log.info("Bootstrap request with type load(${bootstrapRequest.loadModelType}), " + + "resource dictionary load(${bootstrapRequest.loadResourceDictionary}) and " + + "cba load(${bootstrapRequest.loadCBA})") + if (bootstrapRequest.loadModelType) { + bluePrintDatabaseLoadService.initModelTypes() + } + if (bootstrapRequest.loadResourceDictionary) { + bluePrintDatabaseLoadService.initResourceDictionary() + } + if (bootstrapRequest.loadCBA) { + bluePrintDatabaseLoadService.initBluePrintCatalog() + } + } + /** * This is a getAllBlueprintModel method to retrieve all the BlueprintModel in Database * @@ -242,13 +261,11 @@ open class BluePrintModelHandler(private val blueprintsProcessorCatalogService: * @return List list of the controller blueprint */ open fun searchBluePrintModelsByKeyWord(keyWord: String): List { - return blueprintModelSearchRepository. - findByUpdatedByOrTagsOrOrArtifactNameOrOrArtifactVersionOrArtifactType( - keyWord,keyWord,keyWord,keyWord,keyWord) + return blueprintModelSearchRepository.findByUpdatedByOrTagsOrOrArtifactNameOrOrArtifactVersionOrArtifactType( + keyWord, keyWord, keyWord, keyWord, keyWord) } - /** * This is a searchBluePrintModelsByKeyWordPagebale method to retrieve specific BlueprintModel in Database * where keyword equals updatedBy or tags or artifcat name or artifcat version or artifact type and pageable @@ -257,11 +274,10 @@ open class BluePrintModelHandler(private val blueprintsProcessorCatalogService: * * @return List list of the controller blueprint */ - open fun searchBluePrintModelsByKeyWordPaged(keyWord: String, pageRequest: PageRequest): Page - { - return blueprintModelSearchRepository. - findByUpdatedByOrTagsOrOrArtifactNameOrOrArtifactVersionOrArtifactType(keyWord,keyWord,keyWord,keyWord,keyWord,pageRequest) + open fun searchBluePrintModelsByKeyWordPaged(keyWord: String, pageRequest: PageRequest): Page { + return blueprintModelSearchRepository.findByUpdatedByOrTagsOrOrArtifactNameOrOrArtifactVersionOrArtifactType(keyWord, keyWord, keyWord, keyWord, keyWord, pageRequest) } + /** * This is a deleteBlueprintModel method * diff --git a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/load/BluePrintDatabaseLoadService.kt b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/load/BluePrintDatabaseLoadService.kt index f30d156ef..31bd2c953 100644 --- a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/load/BluePrintDatabaseLoadService.kt +++ b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/load/BluePrintDatabaseLoadService.kt @@ -20,8 +20,6 @@ package org.onap.ccsdk.cds.blueprintsprocessor.designer.api.load import kotlinx.coroutines.runBlocking import org.onap.ccsdk.cds.controllerblueprints.core.config.BluePrintLoadConfiguration import org.slf4j.LoggerFactory -import org.springframework.boot.context.event.ApplicationReadyEvent -import org.springframework.context.event.EventListener import org.springframework.stereotype.Service @Service @@ -32,52 +30,36 @@ open class BluePrintDatabaseLoadService(private val bluePrintLoadConfiguration: private val log = LoggerFactory.getLogger(BluePrintDatabaseLoadService::class.java) - - @EventListener(ApplicationReadyEvent::class) open fun init() = runBlocking { - if (bluePrintLoadConfiguration.loadInitialData) { - initModelTypes() - initResourceDictionary() - initBluePrintCatalog() - } else { - log.info("Initial data load is disabled") - } - + initModelTypes() + initResourceDictionary() + initBluePrintCatalog() } open suspend fun initModelTypes() { - log.info("model types load configuration(${bluePrintLoadConfiguration.loadModelType}) " + - "under paths(${bluePrintLoadConfiguration.loadModeTypePaths})") + log.info("model types load from paths(${bluePrintLoadConfiguration.loadModeTypePaths})") - if (bluePrintLoadConfiguration.loadModelType) { - val paths = bluePrintLoadConfiguration.loadModeTypePaths?.split(",") - paths?.let { - modelTypeLoadService.loadPathsModelType(paths) - } + val paths = bluePrintLoadConfiguration.loadModeTypePaths?.split(",") + paths?.let { + modelTypeLoadService.loadPathsModelType(paths) } } open suspend fun initResourceDictionary() { - log.info("resource dictionary load configuration(${bluePrintLoadConfiguration.loadResourceDictionary}) " + - "under paths(${bluePrintLoadConfiguration.loadResourceDictionaryPaths})") + log.info("resource dictionary load from paths(${bluePrintLoadConfiguration.loadResourceDictionaryPaths})") - if (bluePrintLoadConfiguration.loadResourceDictionary) { - val paths = bluePrintLoadConfiguration.loadResourceDictionaryPaths?.split(",") - paths?.let { - resourceDictionaryLoadService.loadPathsResourceDictionary(paths) - } + val paths = bluePrintLoadConfiguration.loadResourceDictionaryPaths?.split(",") + paths?.let { + resourceDictionaryLoadService.loadPathsResourceDictionary(paths) } } open suspend fun initBluePrintCatalog() { - log.info("blueprint load configuration(${bluePrintLoadConfiguration.loadBluePrint}) " + - "under paths(${bluePrintLoadConfiguration.loadBluePrintPaths})") + log.info("cba load from paths(${bluePrintLoadConfiguration.loadBluePrintPaths})") - if (bluePrintLoadConfiguration.loadBluePrint) { - val paths = bluePrintLoadConfiguration.loadBluePrintPaths?.split(",") - paths?.let { - bluePrintCatalogLoadService.loadPathsBluePrintModelCatalog(paths) - } + val paths = bluePrintLoadConfiguration.loadBluePrintPaths?.split(",") + paths?.let { + bluePrintCatalogLoadService.loadPathsBluePrintModelCatalog(paths) } } } \ No newline at end of file diff --git a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandlerTest.kt b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandlerTest.kt index 54dd46ef7..61f515017 100644 --- a/ms/blueprintsprocessor/modules/inbounds/designer-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandlerTest.kt +++ b/ms/blueprintsprocessor/modules/inbounds/designer-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/designer/api/BluePrintManagementGRPCHandlerTest.kt @@ -67,6 +67,18 @@ class BluePrintManagementGRPCHandlerTest { deleteDir("target", "blueprints") } + @Test + fun testBootstrap() { + val blockingStub = BluePrintManagementServiceGrpc.newBlockingStub(grpcServerRule.channel) + val id = "123_Bootstrap" + val req = createBootstrapInputRequest(id) + val bootstrapOutput = blockingStub.bootstrapBlueprint(req) + assertEquals(200, bootstrapOutput.status.code) + assertTrue(bootstrapOutput.status.message.contentEquals(BluePrintConstants.STATUS_SUCCESS), + "failed to get success status") + assertEquals(id, bootstrapOutput.commonHeader.requestId) + } + @Test fun `test upload and download blueprint`() { val blockingStub = BluePrintManagementServiceGrpc.newBlockingStub(grpcServerRule.channel) @@ -128,6 +140,21 @@ class BluePrintManagementGRPCHandlerTest { } } + private fun createBootstrapInputRequest(id: String): BluePrintBootstrapInput { + val commonHeader = CommonHeader + .newBuilder() + .setTimestamp("2012-04-23T18:25:43.511Z") + .setOriginatorId("System") + .setRequestId(id) + .setSubRequestId("1234-56").build() + + return BluePrintBootstrapInput.newBuilder() + .setCommonHeader(commonHeader) + .setLoadModelType(false) + .setLoadResourceDictionary(false) + .setLoadCBA(false) + .build() + } private fun createUploadInputRequest(id: String, action: String): BluePrintUploadInput { val file = normalizedFile("./src/test/resources/test-cba.zip") -- cgit 1.2.3-korg