From c3811effed948926f8d94615df7530c99d490092 Mon Sep 17 00:00:00 2001 From: "Muthuramalingam, Brinda Santh" Date: Mon, 25 Mar 2019 17:01:08 -0400 Subject: Improve blueprint save Change-Id: Ibac2ef9cd7e217db809a6a695ea0ee39a6bd2e21 Issue-ID: CCSDK-1137 Signed-off-by: Muthuramalingam, Brinda Santh --- .../db/BlueprintProcessorCatalogServiceImpl.kt | 43 ++++++--- .../db/BlueprintProcessorCatalogServiceImplTest.kt | 32 +++++-- .../src/test/resources/application-test.properties | 1 + .../core/BluePrintCoreConfiguration.kt | 25 +++-- .../api/BluePrintManagementGRPCHandler.kt | 102 +++++++++++---------- .../selfservice/api/ExecutionServiceController.kt | 8 +- .../selfservice/api/ExecutionServiceHandler.kt | 58 +++++++----- .../api/BluePrintManagementGRPCHandlerTest.kt | 45 ++++++--- .../selfservice/api/ExecutionServiceHandlerTest.kt | 78 +++++++++++----- .../src/test/resources/application-test.properties | 1 + 10 files changed, 249 insertions(+), 144 deletions(-) (limited to 'ms/blueprintsprocessor/modules') diff --git a/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/BlueprintProcessorCatalogServiceImpl.kt b/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/BlueprintProcessorCatalogServiceImpl.kt index 0a625007f..3234c9a3c 100755 --- a/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/BlueprintProcessorCatalogServiceImpl.kt +++ b/ms/blueprintsprocessor/modules/commons/db-lib/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/BlueprintProcessorCatalogServiceImpl.kt @@ -18,13 +18,12 @@ package org.onap.ccsdk.cds.blueprintsprocessor.db -import org.onap.ccsdk.cds.blueprintsprocessor.core.BluePrintCoreConfiguration import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.domain.BlueprintProcessorModel import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.domain.BlueprintProcessorModelContent import org.onap.ccsdk.cds.blueprintsprocessor.db.primary.repository.BlueprintProcessorModelRepository -import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants -import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintException +import org.onap.ccsdk.cds.controllerblueprints.core.* import org.onap.ccsdk.cds.controllerblueprints.core.common.ApplicationConstants +import org.onap.ccsdk.cds.controllerblueprints.core.config.BluePrintPathConfiguration import org.onap.ccsdk.cds.controllerblueprints.core.data.ErrorCode import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintValidatorService import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintArchiveUtils @@ -35,16 +34,16 @@ import org.springframework.stereotype.Service import java.io.File import java.nio.file.Files import java.nio.file.Path -import java.nio.file.Paths +import java.util.* /** * Similar/Duplicate implementation in [org.onap.ccsdk.cds.controllerblueprints.service.load.ControllerBlueprintCatalogServiceImpl] */ @Service class BlueprintProcessorCatalogServiceImpl(bluePrintRuntimeValidatorService: BluePrintValidatorService, - private val blueprintConfig: BluePrintCoreConfiguration, + private val bluePrintPathConfiguration: BluePrintPathConfiguration, private val blueprintModelRepository: BlueprintProcessorModelRepository) - : BlueprintCatalogServiceImpl(bluePrintRuntimeValidatorService) { + : BlueprintCatalogServiceImpl(bluePrintPathConfiguration, bluePrintRuntimeValidatorService) { private val log = LoggerFactory.getLogger(BlueprintProcessorCatalogServiceImpl::class.toString()) @@ -53,33 +52,47 @@ class BlueprintProcessorCatalogServiceImpl(bluePrintRuntimeValidatorService: Blu log.info("BlueprintProcessorCatalogServiceImpl initialized") } - override fun delete(name: String, version: String) = blueprintModelRepository.deleteByArtifactNameAndArtifactVersion(name, version) + override suspend fun delete(name: String, version: String) { + // Cleaning Deployed Blueprint + deleteNBDir(bluePrintPathConfiguration.blueprintDeployPath, name, version) + // Cleaning Data Base + blueprintModelRepository + .deleteByArtifactNameAndArtifactVersion(name, version) + } + + + override suspend fun get(name: String, version: String, extract: Boolean): Path? { + val getId = UUID.randomUUID().toString() + var path = "${bluePrintPathConfiguration.blueprintArchivePath}/$getId/cba.zip" - override fun get(name: String, version: String, extract: Boolean): Path? { - var path = "${blueprintConfig.archivePath}/$name/$version.zip" + // TODO("Check first location for the file", If not get from database") blueprintModelRepository.findByArtifactNameAndArtifactVersion(name, version)?.also { it.blueprintModelContent.run { - val file = File(path) - file.parentFile.mkdirs() - file.createNewFile() + val file = normalizedFile(path) + file.parentFile.reCreateDirs() + file.writeBytes(this!!.content!!).let { if (extract) { - path = "${blueprintConfig.archivePath}/$name/$version" + path = "${bluePrintPathConfiguration.blueprintDeployPath}/$name/$version" BluePrintArchiveUtils.deCompress(file, path) } - return Paths.get(path) + return normalizedPath(path) } } } return null } - override fun save(metadata: MutableMap, archiveFile: File) { + override suspend fun save(metadata: MutableMap, archiveFile: File) { val artifactName = metadata[BluePrintConstants.METADATA_TEMPLATE_NAME] val artifactVersion = metadata[BluePrintConstants.METADATA_TEMPLATE_VERSION] + check(archiveFile.isFile && !archiveFile.isDirectory) { + throw BluePrintException("Not a valid Archive file(${archiveFile.absolutePath})") + } + blueprintModelRepository.findByArtifactNameAndArtifactVersion(artifactName!!, artifactVersion!!)?.let { log.info("Overwriting blueprint model :$artifactName::$artifactVersion") blueprintModelRepository.deleteByArtifactNameAndArtifactVersion(artifactName, artifactVersion) diff --git a/ms/blueprintsprocessor/modules/commons/db-lib/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/BlueprintProcessorCatalogServiceImplTest.kt b/ms/blueprintsprocessor/modules/commons/db-lib/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/BlueprintProcessorCatalogServiceImplTest.kt index a9a2ae73e..2fda15906 100644 --- a/ms/blueprintsprocessor/modules/commons/db-lib/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/BlueprintProcessorCatalogServiceImplTest.kt +++ b/ms/blueprintsprocessor/modules/commons/db-lib/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/db/BlueprintProcessorCatalogServiceImplTest.kt @@ -15,16 +15,19 @@ */ package org.onap.ccsdk.cds.blueprintsprocessor.db +import kotlinx.coroutines.runBlocking import org.junit.Test import org.junit.runner.RunWith +import org.onap.ccsdk.cds.controllerblueprints.core.deleteDir import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintCatalogService +import org.onap.ccsdk.cds.controllerblueprints.core.normalizedFile import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.EnableAutoConfiguration import org.springframework.context.annotation.ComponentScan import org.springframework.test.context.TestPropertySource import org.springframework.test.context.junit4.SpringRunner -import java.io.File -import java.nio.file.Paths +import kotlin.test.AfterTest +import kotlin.test.BeforeTest import kotlin.test.assertTrue @RunWith(SpringRunner::class) @@ -36,17 +39,30 @@ class BlueprintProcessorCatalogServiceImplTest { @Autowired lateinit var blueprintCatalog: BluePrintCatalogService + @BeforeTest + fun setup() { + deleteDir("target", "blueprints") + } + + @AfterTest + fun cleanDir() { + deleteDir("target", "blueprints") + } + @Test fun `test catalog service`() { - val file = Paths.get("./src/test/resources/test-cba.zip").toFile() - assertTrue(file.exists(), "couldnt get file ${file.absolutePath}") + runBlocking { + //FIXME("Create ZIP from test blueprints") + + val file = normalizedFile("./src/test/resources/test-cba.zip") + assertTrue(file.exists(), "couldn't get file ${file.absolutePath}") - blueprintCatalog.saveToDatabase(file) + blueprintCatalog.saveToDatabase("1234", file) - blueprintCatalog.getFromDatabase("baseconfiguration", "1.0.0") + blueprintCatalog.getFromDatabase("baseconfiguration", "1.0.0") - blueprintCatalog.deleteFromDatabase("baseconfiguration", "1.0.0") + blueprintCatalog.deleteFromDatabase("baseconfiguration", "1.0.0") - File("./src/test/resources/baseconfiguration").deleteRecursively() + } } } \ No newline at end of file diff --git a/ms/blueprintsprocessor/modules/commons/db-lib/src/test/resources/application-test.properties b/ms/blueprintsprocessor/modules/commons/db-lib/src/test/resources/application-test.properties index 3ac7ec38b..9dda71eb2 100644 --- a/ms/blueprintsprocessor/modules/commons/db-lib/src/test/resources/application-test.properties +++ b/ms/blueprintsprocessor/modules/commons/db-lib/src/test/resources/application-test.properties @@ -25,3 +25,4 @@ blueprintsprocessor.db.primary.hibernateDialect=org.hibernate.dialect.H2Dialect # Controller Blueprints Core Configuration blueprintsprocessor.blueprintDeployPath=./target/blueprints/deploy blueprintsprocessor.blueprintArchivePath=./target/blueprints/archive +blueprintsprocessor.blueprintWorkingPath=./target/blueprints/work diff --git a/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/BluePrintCoreConfiguration.kt b/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/BluePrintCoreConfiguration.kt index 03b847e51..5f1ae7d37 100644 --- a/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/BluePrintCoreConfiguration.kt +++ b/ms/blueprintsprocessor/modules/commons/processor-core/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/core/BluePrintCoreConfiguration.kt @@ -16,23 +16,29 @@ package org.onap.ccsdk.cds.blueprintsprocessor.core +import org.onap.ccsdk.cds.controllerblueprints.core.config.BluePrintPathConfiguration import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.context.properties.bind.Bindable import org.springframework.boot.context.properties.bind.Binder import org.springframework.boot.context.properties.source.ConfigurationPropertySources import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.core.env.Environment +import org.springframework.stereotype.Service @Configuration -open class BluePrintCoreConfiguration { +open class BluePrintCoreConfiguration(private val bluePrintProperties: BlueprintProcessorProperties) { - @Value("\${blueprintsprocessor.blueprintDeployPath}") - lateinit var deployPath: String + companion object { + const val PREFIX_BLUEPRINT_PROCESSOR = "blueprintsprocessor" + } - @Value("\${blueprintsprocessor.blueprintArchivePath}") - lateinit var archivePath: String + @Bean + open fun bluePrintPathConfiguration(): BluePrintPathConfiguration { + return bluePrintProperties + .propertyBeanType(PREFIX_BLUEPRINT_PROCESSOR, BluePrintPathConfiguration::class.java) + } } @@ -46,4 +52,11 @@ open class BlueprintPropertyConfiguration { val configurationPropertySource = ConfigurationPropertySources.get(environment) return Binder(configurationPropertySource) } +} + +@Service +open class BlueprintProcessorProperties(private var bluePrintPropertyBinder: Binder) { + fun propertyBeanType(prefix: String, type: Class): T { + return bluePrintPropertyBinder.bind(prefix, Bindable.of(type)).get() + } } \ No newline at end of file diff --git a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/BluePrintManagementGRPCHandler.kt b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/BluePrintManagementGRPCHandler.kt index 1fa141034..251ae2c3a 100644 --- a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/BluePrintManagementGRPCHandler.kt +++ b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/BluePrintManagementGRPCHandler.kt @@ -1,6 +1,7 @@ /* * Copyright © 2017-2018 AT&T Intellectual Property. * Modifications Copyright © 2019 Bell Canada. + * Modifications Copyright © 2019 IBM. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,78 +20,87 @@ package org.onap.ccsdk.cds.blueprintsprocessor.selfservice.api import io.grpc.StatusException import io.grpc.stub.StreamObserver -import org.onap.ccsdk.cds.blueprintsprocessor.core.BluePrintCoreConfiguration +import kotlinx.coroutines.runBlocking import org.onap.ccsdk.cds.blueprintsprocessor.selfservice.api.utils.currentTimestamp import org.onap.ccsdk.cds.controllerblueprints.common.api.CommonHeader import org.onap.ccsdk.cds.controllerblueprints.common.api.Status +import org.onap.ccsdk.cds.controllerblueprints.core.config.BluePrintPathConfiguration +import org.onap.ccsdk.cds.controllerblueprints.core.deleteDir import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintCatalogService -import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementInput +import org.onap.ccsdk.cds.controllerblueprints.core.normalizedFile +import org.onap.ccsdk.cds.controllerblueprints.core.reCreateDirs import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementOutput import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementServiceGrpc +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintRemoveInput +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintUploadInput import org.slf4j.LoggerFactory import org.springframework.security.access.prepost.PreAuthorize import org.springframework.stereotype.Service import java.io.File +import java.util.* @Service -open class BluePrintManagementGRPCHandler(private val bluePrintCoreConfiguration: BluePrintCoreConfiguration, - private val bluePrintCatalogService: BluePrintCatalogService) +open class BluePrintManagementGRPCHandler(private val bluePrintPathConfiguration: BluePrintPathConfiguration, + private val bluePrintCatalogService: BluePrintCatalogService) : BluePrintManagementServiceGrpc.BluePrintManagementServiceImplBase() { private val log = LoggerFactory.getLogger(BluePrintManagementGRPCHandler::class.java) @PreAuthorize("hasRole('USER')") - override fun uploadBlueprint(request: BluePrintManagementInput, responseObserver: StreamObserver) { - val blueprintName = request.blueprintName - val blueprintVersion = request.blueprintVersion - val blueprint = "blueprint $blueprintName:$blueprintVersion" + override fun uploadBlueprint(request: BluePrintUploadInput, responseObserver: + StreamObserver) { + runBlocking { + + log.info("request(${request.commonHeader.requestId})") + val uploadId = UUID.randomUUID().toString() + try { + val cbaFile = normalizedFile(bluePrintPathConfiguration.blueprintArchivePath, uploadId, "cba-zip") + + saveToDisk(request, cbaFile) + + val blueprintId = bluePrintCatalogService.saveToDatabase(uploadId, cbaFile) + responseObserver.onNext(successStatus("Successfully uploaded CBA($blueprintId)...", request.commonHeader)) + responseObserver.onCompleted() + } catch (e: Exception) { + failStatus("request(${request.commonHeader.requestId}): Failed to upload CBA", e) + } finally { + deleteDir(bluePrintPathConfiguration.blueprintArchivePath, uploadId) + deleteDir(bluePrintPathConfiguration.blueprintWorkingPath, uploadId) + } + } + } - log.info("request(${request.commonHeader.requestId}): Received upload $blueprint") + @PreAuthorize("hasRole('USER')") + override fun removeBlueprint(request: BluePrintRemoveInput, responseObserver: + StreamObserver) { - val blueprintArchivedFilePath = "${bluePrintCoreConfiguration.archivePath}/$blueprintName/$blueprintVersion/$blueprintName.zip" - try { - val blueprintArchivedFile = File(blueprintArchivedFilePath) + runBlocking { + val blueprintName = request.blueprintName + val blueprintVersion = request.blueprintVersion + val blueprint = "blueprint $blueprintName:$blueprintVersion" - saveToDisk(request, blueprintArchivedFile) - val blueprintId = bluePrintCatalogService.saveToDatabase(blueprintArchivedFile) + log.info("request(${request.commonHeader.requestId}): Received delete $blueprint") - File("${bluePrintCoreConfiguration.archivePath}/$blueprintName").deleteRecursively() - responseObserver.onNext(successStatus("Successfully uploaded $blueprint with id($blueprintId)", request.commonHeader)) - responseObserver.onCompleted() - } catch (e: Exception) { - failStatus("request(${request.commonHeader.requestId}): Failed to upload $blueprint at path $blueprintArchivedFilePath", e) + try { + bluePrintCatalogService.deleteFromDatabase(blueprintName, blueprintVersion) + responseObserver.onNext(successStatus("Successfully deleted $blueprint", request.commonHeader)) + responseObserver.onCompleted() + } catch (e: Exception) { + failStatus("request(${request.commonHeader.requestId}): Failed to delete $blueprint", e) + } } } - @PreAuthorize("hasRole('USER')") - override fun removeBlueprint(request: BluePrintManagementInput, responseObserver: StreamObserver) { - val blueprintName = request.blueprintName - val blueprintVersion = request.blueprintVersion - val blueprint = "blueprint $blueprintName:$blueprintVersion" - - log.info("request(${request.commonHeader.requestId}): Received delete $blueprint") - - try { - bluePrintCatalogService.deleteFromDatabase(blueprintName, blueprintVersion) - responseObserver.onNext(successStatus("Successfully deleted $blueprint", request.commonHeader)) - responseObserver.onCompleted() - } catch (e: Exception) { - failStatus("request(${request.commonHeader.requestId}): Failed to delete $blueprint", e) - } - } + private fun saveToDisk(request: BluePrintUploadInput, cbaFile: File) { + log.info("request(${request.commonHeader.requestId}): Writing CBA File under :${cbaFile.absolutePath}") - private fun saveToDisk(request: BluePrintManagementInput, blueprintDir: File) { - log.info("request(${request.commonHeader.requestId}): Writing CBA File under :${blueprintDir.absolutePath}") - if (blueprintDir.exists()) { - log.info("request(${request.commonHeader.requestId}): Re-creating blueprint directory(${blueprintDir.absolutePath})") - //FileUtils.deleteDirectory(blueprintDir.parentFile) - blueprintDir.parentFile.deleteRecursively() - } - blueprintDir.parentFile.mkdirs() - //FileUtils.forceMkdir(blueprintDir.parentFile) - blueprintDir.writeBytes(request.fileChunk.chunk.toByteArray()).apply { - log.info("request(${request.commonHeader.requestId}): CBA file(${blueprintDir.absolutePath} written successfully") + // Recreate Folder + cbaFile.parentFile.reCreateDirs() + + // Write the File + cbaFile.writeBytes(request.fileChunk.chunk.toByteArray()).apply { + log.info("request(${request.commonHeader.requestId}): CBA file(${cbaFile.absolutePath} written successfully") } } diff --git a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceController.kt b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceController.kt index 1039d5cd2..41e78e518 100644 --- a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceController.kt +++ b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceController.kt @@ -27,7 +27,6 @@ import org.springframework.http.MediaType import org.springframework.http.codec.multipart.FilePart import org.springframework.security.access.prepost.PreAuthorize import org.springframework.web.bind.annotation.* -import reactor.core.publisher.Mono @RestController @RequestMapping("/api/v1/execution-service") @@ -46,11 +45,8 @@ open class ExecutionServiceController { @ApiOperation(value = "Upload CBA", notes = "Takes a File and load it in the runtime database") @ResponseBody @PreAuthorize("hasRole('USER')") - fun upload(@RequestPart("file") parts: Mono): Mono { - return parts - .filter { it is FilePart } - .ofType(FilePart::class.java) - .flatMap(executionServiceHandler::upload) + fun upload(@RequestPart("file") filePart: FilePart): String = runBlocking { + executionServiceHandler.upload(filePart) } @RequestMapping(path = ["/process"], method = [RequestMethod.POST], produces = [MediaType.APPLICATION_JSON_VALUE]) diff --git a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceHandler.kt b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceHandler.kt index f3af254be..bf0fb44d6 100644 --- a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceHandler.kt +++ b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceHandler.kt @@ -22,44 +22,50 @@ import io.grpc.stub.StreamObserver import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.launch -import org.onap.ccsdk.cds.blueprintsprocessor.core.BluePrintCoreConfiguration -import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ACTION_MODE_ASYNC -import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ACTION_MODE_SYNC -import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput -import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceOutput -import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.Status -import org.onap.ccsdk.cds.blueprintsprocessor.selfservice.api.utils.saveCBAFile +import kotlinx.coroutines.reactive.awaitSingle +import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.* import org.onap.ccsdk.cds.blueprintsprocessor.selfservice.api.utils.toProto import org.onap.ccsdk.cds.controllerblueprints.common.api.EventType -import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants -import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintException +import org.onap.ccsdk.cds.controllerblueprints.core.* +import org.onap.ccsdk.cds.controllerblueprints.core.config.BluePrintPathConfiguration +import org.onap.ccsdk.cds.controllerblueprints.core.data.ErrorCode import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintCatalogService import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintWorkflowExecutionService -import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintFileUtils import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintMetadataUtils import org.slf4j.LoggerFactory import org.springframework.http.codec.multipart.FilePart import org.springframework.stereotype.Service -import reactor.core.publisher.Mono +import java.io.File +import java.io.IOException +import java.util.* import java.util.stream.Collectors @Service -class ExecutionServiceHandler(private val bluePrintCoreConfiguration: BluePrintCoreConfiguration, +class ExecutionServiceHandler(private val bluePrintPathConfiguration: BluePrintPathConfiguration, private val bluePrintCatalogService: BluePrintCatalogService, private val bluePrintWorkflowExecutionService : BluePrintWorkflowExecutionService) { private val log = LoggerFactory.getLogger(ExecutionServiceHandler::class.toString()) - fun upload(filePart: FilePart): Mono { + suspend fun upload(filePart: FilePart): String { + val saveId = UUID.randomUUID().toString() + val blueprintArchive = normalizedPathName(bluePrintPathConfiguration.blueprintArchivePath, saveId) + val blueprintWorking = normalizedPathName(bluePrintPathConfiguration.blueprintWorkingPath, saveId) try { - val archivedPath = BluePrintFileUtils.getCbaStorageDirectory(bluePrintCoreConfiguration.archivePath) - val cbaPath = saveCBAFile(filePart, archivedPath) - bluePrintCatalogService.saveToDatabase(cbaPath.toFile()).let { - return Mono.just("{\"status\": \"Successfully uploaded blueprint with id($it)\"}") - } - } catch (e: Exception) { - return Mono.error(BluePrintException("Error uploading the CBA file.", e)) + + val compressedFile = normalizedFile(blueprintArchive, "cba.zip") + compressedFile.parentFile.reCreateNBDirs() + // Copy the File Part to Local File + copyFromFilePart(filePart, compressedFile) + // Save the Copied file to Database + return bluePrintCatalogService.saveToDatabase(saveId, compressedFile, false) + } catch (e: IOException) { + throw BluePrintException(ErrorCode.IO_FILE_INTERRUPT.value, + "Error in Upload CBA: ${e.message}", e) + } finally { + deleteNBDir(blueprintArchive) + deleteNBDir(blueprintWorking) } } @@ -80,8 +86,8 @@ class ExecutionServiceHandler(private val bluePrintCoreConfiguration: BluePrintC responseObserver.onCompleted() } else -> responseObserver.onNext(response(executionServiceInput, - "Failed to process request, 'actionIdentifiers.mode' not specified. Valid value are: 'sync' or 'async'.", - true).toProto()); + "Failed to process request, 'actionIdentifiers.mode' not specified. Valid value are: 'sync' or 'async'.", + true).toProto()); } } @@ -100,7 +106,7 @@ class ExecutionServiceHandler(private val bluePrintCoreConfiguration: BluePrintC val blueprintRuntimeService = BluePrintMetadataUtils.getBluePrintRuntime(requestId, basePath.toString()) val output = bluePrintWorkflowExecutionService.executeBluePrintWorkflow(blueprintRuntimeService, - executionServiceInput, hashMapOf()) + executionServiceInput, hashMapOf()) val errors = blueprintRuntimeService.getBluePrintError().errors if (errors.isNotEmpty()) { @@ -111,6 +117,12 @@ class ExecutionServiceHandler(private val bluePrintCoreConfiguration: BluePrintC return output } + private suspend fun copyFromFilePart(filePart: FilePart, targetFile: File): File { + return filePart.transferTo(targetFile) + .thenReturn(targetFile) + .awaitSingle() + } + private fun setErrorStatus(errorMessage: String, status: Status) { status.errorMessage = errorMessage status.eventType = EventType.EVENT_COMPONENT_FAILURE.name diff --git a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/BluePrintManagementGRPCHandlerTest.kt b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/BluePrintManagementGRPCHandlerTest.kt index a03ad9e47..fd764d78f 100644 --- a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/BluePrintManagementGRPCHandlerTest.kt +++ b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/BluePrintManagementGRPCHandlerTest.kt @@ -1,6 +1,7 @@ /* * Copyright © 2017-2018 AT&T Intellectual Property. * Modifications Copyright © 2019 Bell Canada. + * Modifications Copyright © 2019 IBM. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +24,11 @@ import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith import org.onap.ccsdk.cds.controllerblueprints.common.api.CommonHeader -import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementInput +import org.onap.ccsdk.cds.controllerblueprints.core.deleteDir +import org.onap.ccsdk.cds.controllerblueprints.core.normalizedFile import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementServiceGrpc +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintRemoveInput +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintUploadInput import org.onap.ccsdk.cds.controllerblueprints.management.api.FileChunk import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.EnableAutoConfiguration @@ -32,7 +36,6 @@ import org.springframework.context.annotation.ComponentScan import org.springframework.test.annotation.DirtiesContext import org.springframework.test.context.TestPropertySource import org.springframework.test.context.junit4.SpringRunner -import java.io.File import kotlin.test.AfterTest import kotlin.test.BeforeTest import kotlin.test.assertEquals @@ -55,25 +58,23 @@ class BluePrintManagementGRPCHandlerTest { fun init() { // Create a server, add service, start, and register for automatic graceful shutdown. grpcServerRule.serviceRegistry.addService(bluePrintManagementGRPCHandler) + deleteDir("target", "blueprints") } @AfterTest fun cleanDir() { - //TODO It's giving fluctuating results, need to look for another way to cleanup - // works sometimes otherwise results IO Exception - // Most probably bufferReader stream is not getting closed when cleanDir is getting invoked - File("./target/blueprints").deleteRecursively() + deleteDir("target", "blueprints") } @Test fun `test upload blueprint`() { val blockingStub = BluePrintManagementServiceGrpc.newBlockingStub(grpcServerRule.channel) val id = "123_upload" - val req = createInputRequest(id) + val req = createUploadInputRequest(id) val output = blockingStub.uploadBlueprint(req) assertEquals(200, output.status.code) - assertTrue(output.status.message.contains("Successfully uploaded blueprint sample:1.0.0 with id(")) + assertTrue(output.status.message.contains("Successfully uploaded CBA")) assertEquals(id, output.commonHeader.requestId) } @@ -81,19 +82,20 @@ class BluePrintManagementGRPCHandlerTest { fun `test delete blueprint`() { val blockingStub = BluePrintManagementServiceGrpc.newBlockingStub(grpcServerRule.channel) val id = "123_delete" - val req = createInputRequest(id) + val req = createUploadInputRequest(id) var output = blockingStub.uploadBlueprint(req) assertEquals(200, output.status.code) - assertTrue(output.status.message.contains("Successfully uploaded blueprint sample:1.0.0 with id(")) + assertTrue(output.status.message.contains("Successfully uploaded CBA")) assertEquals(id, output.commonHeader.requestId) - output = blockingStub.removeBlueprint(req) + val removeReq = createRemoveInputRequest(id) + output = blockingStub.removeBlueprint(removeReq) assertEquals(200, output.status.code) } - private fun createInputRequest(id: String): BluePrintManagementInput { - val file = File("./src/test/resources/test-cba.zip") + private fun createUploadInputRequest(id: String): BluePrintUploadInput { + val file = normalizedFile("./src/test/resources/test-cba.zip") assertTrue(file.exists(), "couldnt get file ${file.absolutePath}") val commonHeader = CommonHeader @@ -106,11 +108,24 @@ class BluePrintManagementGRPCHandlerTest { val fileChunk = FileChunk.newBuilder().setChunk(ByteString.copyFrom(file.inputStream().readBytes())) .build() - return BluePrintManagementInput.newBuilder() + return BluePrintUploadInput.newBuilder() + .setCommonHeader(commonHeader) + .setFileChunk(fileChunk) + .build() + } + + private fun createRemoveInputRequest(id: String): BluePrintRemoveInput { + val commonHeader = CommonHeader + .newBuilder() + .setTimestamp("2012-04-23T18:25:43.511Z") + .setOriginatorId("System") + .setRequestId(id) + .setSubRequestId("1234-56").build() + + return BluePrintRemoveInput.newBuilder() .setCommonHeader(commonHeader) .setBlueprintName("sample") .setBlueprintVersion("1.0.0") - .setFileChunk(fileChunk) .build() } } diff --git a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceHandlerTest.kt b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceHandlerTest.kt index b131fb7d1..d14761cc9 100644 --- a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceHandlerTest.kt +++ b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/selfservice/api/ExecutionServiceHandlerTest.kt @@ -17,10 +17,13 @@ package org.onap.ccsdk.cds.blueprintsprocessor.selfservice.api +import kotlinx.coroutines.reactive.awaitSingle +import kotlinx.coroutines.runBlocking import org.junit.Test import org.junit.runner.RunWith import org.onap.ccsdk.cds.blueprintsprocessor.core.BluePrintCoreConfiguration import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput +import org.onap.ccsdk.cds.controllerblueprints.core.deleteDir import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintCatalogService import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils import org.springframework.beans.factory.annotation.Autowired @@ -33,9 +36,13 @@ import org.springframework.test.context.ContextConfiguration import org.springframework.test.context.TestPropertySource import org.springframework.test.context.junit4.SpringRunner import org.springframework.test.web.reactive.server.WebTestClient +import org.springframework.test.web.reactive.server.returnResult import org.springframework.web.reactive.function.BodyInserters import java.nio.file.Files import java.nio.file.Paths +import java.util.* +import kotlin.test.AfterTest +import kotlin.test.BeforeTest import kotlin.test.assertTrue @RunWith(SpringRunner::class) @@ -50,40 +57,61 @@ class ExecutionServiceHandlerTest { @Autowired lateinit var webTestClient: WebTestClient + @BeforeTest + fun init() { + deleteDir("target", "blueprints") + } + + @AfterTest + fun cleanDir() { + deleteDir("target", "blueprints") + } + @Test fun `test rest upload blueprint`() { - val file = Paths.get("./src/test/resources/test-cba.zip").toFile() - assertTrue(file.exists(), "couldnt get file ${file.absolutePath}") + runBlocking { + val file = Paths.get("./src/test/resources/test-cba.zip").toFile() + assertTrue(file.exists(), "couldn't get file ${file.absolutePath}") + + val body = MultipartBodyBuilder().apply { + part("file", object : ByteArrayResource(Files.readAllBytes(Paths.get("./src/test/resources/test-cba.zip"))) { + override fun getFilename(): String { + return "test-cba.zip" + } + }) + }.build() - val body = MultipartBodyBuilder().apply { - part("file", object : ByteArrayResource(Files.readAllBytes(Paths.get("./src/test/resources/test-cba.zip"))) { - override fun getFilename(): String { - return "test-cba.zip" - } - }) - }.build() + webTestClient + .post() + .uri("/api/v1/execution-service/upload") + .body(BodyInserters.fromMultipartData(body)) + .exchange() + .expectStatus().isOk + .returnResult() + .responseBody + .awaitSingle() + } - webTestClient - .post() - .uri("/api/v1/execution-service/upload") - .body(BodyInserters.fromMultipartData(body)) - .exchange() - .expectStatus().isOk } @Test fun `test rest process`() { - val file = Paths.get("./src/test/resources/test-cba.zip").toFile() - assertTrue(file.exists(), "couldnt get file ${file.absolutePath}") - blueprintCatalog.saveToDatabase(file) + runBlocking { + val file = Paths.get("./src/test/resources/test-cba.zip").toFile() + assertTrue(file.exists(), "couldnt get file ${file.absolutePath}") + blueprintCatalog.saveToDatabase(UUID.randomUUID().toString(), file) + + val executionServiceInput = JacksonUtils + .readValueFromClassPathFile("execution-input/default-input.json", + ExecutionServiceInput::class.java)!! - val executionServiceInput = JacksonUtils.readValueFromClassPathFile("execution-input/default-input.json", ExecutionServiceInput::class.java)!! - webTestClient - .post() - .uri("/api/v1/execution-service/process") - .body(BodyInserters.fromObject(executionServiceInput)) - .exchange() - .expectStatus().isOk + webTestClient + .post() + .uri("/api/v1/execution-service/process") + .body(BodyInserters.fromObject(executionServiceInput)) + .exchange() + .expectStatus().isOk + } } } \ No newline at end of file diff --git a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/resources/application-test.properties b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/resources/application-test.properties index 6d8b62ff9..6705523df 100644 --- a/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/resources/application-test.properties +++ b/ms/blueprintsprocessor/modules/inbounds/selfservice-api/src/test/resources/application-test.properties @@ -25,6 +25,7 @@ blueprintsprocessor.db.primary.hibernateNamingStrategy=org.hibernate.cfg.Improve blueprintsprocessor.db.primary.hibernateDialect=org.hibernate.dialect.H2Dialect # Controller Blueprints Core Configuration blueprintsprocessor.blueprintDeployPath=./target/blueprints/deploy +blueprintsprocessor.blueprintWorkingPath=./target/blueprints/work blueprintsprocessor.blueprintArchivePath=./target/blueprints/archive # Python executor -- cgit 1.2.3-korg