diff options
author | prathamesh morde <prathamesh.morde@bell.ca> | 2019-04-01 23:19:42 -0400 |
---|---|---|
committer | prathamesh morde <prathamesh.morde@bell.ca> | 2019-04-04 18:51:16 -0400 |
commit | 4140b0716234e7540d57b8a746944018fa591d69 (patch) | |
tree | 49030514fddc4f9433398df033bdfcabae11d246 /ms/cds-sdc-listener/application/src/main | |
parent | 41af46c85eafe85b20833c99def94ab530afa01b (diff) |
CDS-SDC Listener application
Things done
-Implementation of GRPC client to store CBA archive using CDS Backend
-Unit test coverage.
-Tested locally.
-Minor bug fix, performance improvement, more logging and make use of BluePrintProcessorHandler api
in ListenerServiceImpl class.
-Extract csar artifact from IDistributionClientResult and store it into local file(disk).
Change-Id: I08b0de017396bb76d5bc13ddb9e7b430990e8df0
Issue-ID: CCSDK-1184
Signed-off-by: prathamesh morde <prathamesh.morde@bell.ca>
Diffstat (limited to 'ms/cds-sdc-listener/application/src/main')
8 files changed, 291 insertions, 21 deletions
diff --git a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/CdsSdcListenerNotificationCallback.java b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/CdsSdcListenerNotificationCallback.java index aaab8d81f..e2aae9654 100644 --- a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/CdsSdcListenerNotificationCallback.java +++ b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/CdsSdcListenerNotificationCallback.java @@ -9,7 +9,13 @@ package org.onap.ccsdk.cds.cdssdclistener; import static org.onap.sdc.utils.DistributionActionResultEnum.SUCCESS; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; import java.util.Objects; +import java.util.Optional; +import org.onap.ccsdk.cds.cdssdclistener.dto.CdsSdcListenerDto; import org.onap.ccsdk.cds.cdssdclistener.service.ListenerServiceImpl; import org.onap.sdc.api.IDistributionClient; import org.onap.sdc.api.consumer.INotificationCallback; @@ -19,9 +25,14 @@ import org.onap.sdc.api.results.IDistributionClientDownloadResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.ComponentScan; import org.springframework.stereotype.Component; +@ConfigurationProperties("listenerservice") @Component +@ComponentScan("org.onap.ccsdk.cds.cdssdclistener.dto") public class CdsSdcListenerNotificationCallback implements INotificationCallback { @Autowired @@ -30,6 +41,9 @@ public class CdsSdcListenerNotificationCallback implements INotificationCallback @Autowired private ListenerServiceImpl listenerService; + @Value("${listenerservice.config.archivePath}") + private String pathToStoreArchives; + private static final Logger LOGGER = LoggerFactory.getLogger(CdsSdcListenerNotificationCallback.class); @Override @@ -45,7 +59,7 @@ public class CdsSdcListenerNotificationCallback implements INotificationCallback } /** - * Download the TOSCA CSAR artifact. + * Download the TOSCA CSAR artifact and process it. * * @param info - Artifact information * @param distributionClient - SDC distribution client @@ -53,6 +67,7 @@ public class CdsSdcListenerNotificationCallback implements INotificationCallback private void downloadCsarArtifacts(IArtifactInfo info, IDistributionClient distributionClient) { final String url = info.getArtifactURL(); final String id = info.getArtifactUUID(); + if (Objects.equals(info.getArtifactType(), CdsSdcListenerConfiguration.TOSCA_CSAR)) { LOGGER.info("Trying to download the artifact from : {} and UUID is {} ", url, id); @@ -63,8 +78,30 @@ public class CdsSdcListenerNotificationCallback implements INotificationCallback LOGGER.error("Failed to download the artifact from : {} due to {} ", url, result.getDistributionActionResult()); } else { - // TODO Store the CSAR into CSARArchive path and extract the Blueprint using ListenerServiceImpl.extractBluePrint + LOGGER.info("Trying to write CSAR artifact to file with URL {} and UUID {}", url, id); + processCsarArtifact(result); } } } + + public void processCsarArtifact(IDistributionClientDownloadResult result) { + Path cbaArchivePath = Paths.get(pathToStoreArchives, "cba-archive"); + Path csarArchivePath = Paths.get(pathToStoreArchives, "csar-archive"); + + // extract and store the CSAR archive into local disk. + listenerService.extractCsarAndStore(result, csarArchivePath.toString()); + + Optional<List<File>> csarFiles = listenerService.getFilesFromDisk(csarArchivePath); + + if (csarFiles.isPresent()) { + + //Extract CBA archive from CSAR package and and store it into CDS Database. + csarFiles.get() + .forEach(file -> listenerService.extractBluePrint(file.getAbsolutePath(), cbaArchivePath.toString())); + + listenerService.saveBluePrintToCdsDatabase(cbaArchivePath); + } else { + LOGGER.error("The CSAR file is not present at this location {}", csarArchivePath); + } + } } diff --git a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/client/CdsSdcListenerAuthClientInterceptor.java b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/client/CdsSdcListenerAuthClientInterceptor.java new file mode 100644 index 000000000..528fbe2f3 --- /dev/null +++ b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/client/CdsSdcListenerAuthClientInterceptor.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2019 Bell Canada. All rights reserved. + * + * NOTICE: All the intellectual and technical concepts contained herein are + * proprietary to Bell Canada and are protected by trade secret or copyright law. + * Unauthorized copying of this file, via any medium is strictly prohibited. + */ + +package org.onap.ccsdk.cds.cdssdclistener.client; + +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ForwardingClientCall; +import io.grpc.Metadata; +import io.grpc.Metadata.Key; +import io.grpc.MethodDescriptor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * To provide authentication with GRPC server. + */ +@ConfigurationProperties("listenerservice") +@Component +public class CdsSdcListenerAuthClientInterceptor implements ClientInterceptor { + + @Value("${listenerservice.config.authHeader}") + private String basicAuth; + + @Override + public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(MethodDescriptor<ReqT, RespT> methodDescriptor, + CallOptions callOptions, Channel channel) { + Key<String> authHeader = Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER); + return new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>( + channel.newCall(methodDescriptor, callOptions)) { + @Override + public void start(Listener<RespT> responseListener, Metadata headers) { + headers.put(authHeader, basicAuth); + super.start(responseListener, headers); + } + }; + } +} diff --git a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/CdsSdcListenerClient.java b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/client/CdsSdcListenerClient.java index 76295bacb..6f888dd0b 100644 --- a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/CdsSdcListenerClient.java +++ b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/client/CdsSdcListenerClient.java @@ -5,9 +5,12 @@ * proprietary to Bell Canada and are protected by trade secret or copyright law. * Unauthorized copying of this file, via any medium is strictly prohibited. */ -package org.onap.ccsdk.cds.cdssdclistener; +package org.onap.ccsdk.cds.cdssdclistener.client; import java.util.Optional; +import org.onap.ccsdk.cds.cdssdclistener.CdsSdcListenerConfiguration; +import org.onap.ccsdk.cds.cdssdclistener.dto.CdsSdcListenerDto; +import org.onap.ccsdk.cds.cdssdclistener.CdsSdcListenerNotificationCallback; import org.onap.ccsdk.cds.cdssdclistener.exceptions.CdsSdcListenerException; import org.onap.sdc.api.IDistributionClient; import org.onap.sdc.api.results.IDistributionClientResult; @@ -17,10 +20,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; @Component +@ComponentScan("org.onap.ccsdk.cds.cdssdclistener.dto") public class CdsSdcListenerClient { private static Logger LOG = LoggerFactory.getLogger(CdsSdcListenerClient.class); diff --git a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/CdsSdcListenerDto.java b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/dto/CdsSdcListenerDto.java index 7d154da42..41039eb28 100644 --- a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/CdsSdcListenerDto.java +++ b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/dto/CdsSdcListenerDto.java @@ -5,10 +5,12 @@ * proprietary to Bell Canada and are protected by trade secret or copyright law. * Unauthorized copying of this file, via any medium is strictly prohibited. */ -package org.onap.ccsdk.cds.cdssdclistener; +package org.onap.ccsdk.cds.cdssdclistener.dto; import org.onap.sdc.api.IDistributionClient; +import org.springframework.stereotype.Component; +@Component public class CdsSdcListenerDto { private IDistributionClient distributionClient; diff --git a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/handler/BluePrintProcesssorHandler.java b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/handler/BluePrintProcesssorHandler.java new file mode 100644 index 000000000..6b03b6da2 --- /dev/null +++ b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/handler/BluePrintProcesssorHandler.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2019 Bell Canada. All rights reserved. + * + * NOTICE: All the intellectual and technical concepts contained herein are + * proprietary to Bell Canada and are protected by trade secret or copyright law. + * Unauthorized copying of this file, via any medium is strictly prohibited. + */ + +package org.onap.ccsdk.cds.cdssdclistener.handler; + +import io.grpc.ManagedChannel; +import org.onap.ccsdk.cds.controllerblueprints.common.api.Status; +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementOutput; +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementServiceGrpc; +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintManagementServiceGrpc.BluePrintManagementServiceBlockingStub; +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintUploadInput; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties("listenerservice") +@Component +public class BluePrintProcesssorHandler implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(BluePrintProcesssorHandler.class); + + private ManagedChannel channel; + + /** + * Sending CBA archive to CDS backend to store into its Database. + * + * @param request BluePrintManagementInput object holds CBA archive, its version and blueprints. + * @param managedChannel - ManagedChannel object helps to access the server or application end point. + * @return A response object + */ + public Status sendRequest(BluePrintUploadInput request, ManagedChannel managedChannel) { + LOGGER.info("Sending request to blueprint processor"); + + this.channel = managedChannel; + + final BluePrintManagementServiceBlockingStub syncStub = BluePrintManagementServiceGrpc.newBlockingStub(channel); + + // Send the request to CDS backend. + final BluePrintManagementOutput response = syncStub.uploadBlueprint(request); + + return response.getStatus(); + } + + @Override + public void close() { + if (channel != null) { + channel.shutdown(); + } + LOGGER.info("Stopping GRPC connection to CDS backend"); + } +} diff --git a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/service/ListenerService.java b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/service/ListenerService.java index 5dc0c2194..1efbe8f33 100644 --- a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/service/ListenerService.java +++ b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/service/ListenerService.java @@ -8,7 +8,8 @@ package org.onap.ccsdk.cds.cdssdclistener.service; -import java.util.zip.ZipFile; +import java.nio.file.Path; +import org.onap.sdc.api.results.IDistributionClientDownloadResult; public interface ListenerService { @@ -23,7 +24,15 @@ public interface ListenerService { /** * Store the Zip file into CDS database. * - * @param file The file to be stored. + * @param path path where zip file exists. */ - void saveBluePrintToCdsDatabase(ZipFile file); + void saveBluePrintToCdsDatabase(Path path); + + /** + * Extract and store CSAR to file. + * + * @param result - IDistributionClientDownloadResult contains payload. + * @param csarArchivePath The destination path where CSAR will be stored. + */ + void extractCsarAndStore(IDistributionClientDownloadResult result, String csarArchivePath); } diff --git a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/service/ListenerServiceImpl.java b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/service/ListenerServiceImpl.java index 4ff2a6ea8..37052082a 100644 --- a/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/service/ListenerServiceImpl.java +++ b/ms/cds-sdc-listener/application/src/main/java/org/onap/ccsdk/cds/cdssdclistener/service/ListenerServiceImpl.java @@ -8,6 +8,10 @@ package org.onap.ccsdk.cds.cdssdclistener.service; +import static java.nio.file.Files.walk; +import com.google.protobuf.ByteString; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -17,12 +21,24 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Enumeration; +import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import org.apache.commons.io.FileUtils; import org.apache.tomcat.util.http.fileupload.IOUtils; +import org.onap.ccsdk.cds.cdssdclistener.client.CdsSdcListenerAuthClientInterceptor; +import org.onap.ccsdk.cds.cdssdclistener.handler.BluePrintProcesssorHandler; +import org.onap.ccsdk.cds.controllerblueprints.common.api.Status; +import org.onap.ccsdk.cds.controllerblueprints.management.api.BluePrintUploadInput; +import org.onap.ccsdk.cds.controllerblueprints.management.api.FileChunk; +import org.onap.sdc.api.results.IDistributionClientDownloadResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @@ -31,13 +47,21 @@ import org.springframework.stereotype.Component; @ConfigurationProperties("listenerservice") public class ListenerServiceImpl implements ListenerService { - @Value("${listenerservice.config.csarArchive}") - private String csarArchivePath; + @Autowired + private BluePrintProcesssorHandler bluePrintProcesssorHandler; - @Value("${listenerservice.config.cbaArchive}") - private String cbaArchivePath; + @Autowired + private CdsSdcListenerAuthClientInterceptor cdsSdcListenerAuthClientInterceptor; + + @Value("${listenerservice.config.grpcAddress}") + private String grpcAddress; + + @Value("${listenerservice.config.grpcPort}") + private int grpcPort; private static final String CBA_ZIP_PATH = "Artifacts/Resources/[a-zA-Z0-9-_]+/Deployment/CONTROLLER_BLUEPRINT_ARCHIVE/[a-zA-Z0-9-_]+[.]zip"; + private static final int SUCCESS_CODE = 200; + private static final String CSAR_FILE_EXTENSION = ".csar"; private static final Logger LOGGER = LoggerFactory.getLogger(ListenerServiceImpl.class); @Override @@ -50,6 +74,7 @@ public class ListenerServiceImpl implements ListenerService { String fileName = entry.getName(); if (Pattern.matches(CBA_ZIP_PATH, fileName)) { final String cbaArchiveName = Paths.get(fileName).getFileName().toString(); + LOGGER.info("Storing the CBA archive {}", cbaArchiveName); storeBluePrint(zipFile, cbaArchiveName, cbaStorageDir, entry); } } @@ -59,13 +84,12 @@ public class ListenerServiceImpl implements ListenerService { } private void storeBluePrint(ZipFile zipFile, String fileName, Path cbaArchivePath, ZipEntry entry) { - final String changedFileName = fileName + ".zip"; - Path targetLocation = cbaArchivePath.resolve(changedFileName); + Path targetLocation = cbaArchivePath.resolve(fileName); + LOGGER.info("The target location for zip file is {}", targetLocation); File targetZipFile = new File(targetLocation.toString()); try { targetZipFile.createNewFile(); - } catch (IOException e) { LOGGER.error("Could not able to create file {}", targetZipFile, e); } @@ -73,15 +97,43 @@ public class ListenerServiceImpl implements ListenerService { try (InputStream inputStream = zipFile.getInputStream(entry); OutputStream out = new FileOutputStream( targetZipFile)) { IOUtils.copy(inputStream, out); + LOGGER.info("Succesfully store the CBA archive {} at this location", targetZipFile); + } catch (Exception e) { + LOGGER.error("Failed to put zip file into target location {}, {}", targetLocation, e); + } + } + + @Override + public void saveBluePrintToCdsDatabase(Path cbaArchivePath) { + Optional<List<File>> zipFiles = getFilesFromDisk(cbaArchivePath); + zipFiles.ifPresent(this::prepareRequestForCdsBackend); + } + + @Override + public void extractCsarAndStore(IDistributionClientDownloadResult result, String csarArchivePath) { + + // Create CSAR storage directory + Path csarStorageDir = getStorageDirectory(csarArchivePath); + + byte[] payload = result.getArtifactPayload(); + String csarFileName = result.getArtifactFilename() + CSAR_FILE_EXTENSION; + Path targetLocation = csarStorageDir.resolve(csarFileName); + + LOGGER.info("The target location for the CSAR file is {}", targetLocation); + + File targetCsarFile = new File(targetLocation.toString()); + + try (FileOutputStream outFile = new FileOutputStream(targetCsarFile)) { + outFile.write(payload, 0, payload.length); } catch (Exception e) { - LOGGER.error("Failed to put zip file into target location {}", targetLocation, e); + LOGGER.error("Failed to put CSAR file into target location {}, {}", targetLocation, e); } } private Path getStorageDirectory(String path) { Path fileStorageLocation = Paths.get(path).toAbsolutePath().normalize(); - if (!Files.exists(fileStorageLocation)) { + if (!fileStorageLocation.toFile().exists()) { try { return Files.createDirectories(fileStorageLocation); } catch (IOException e) { @@ -91,8 +143,68 @@ public class ListenerServiceImpl implements ListenerService { return fileStorageLocation; } - @Override - public void saveBluePrintToCdsDatabase(ZipFile file) { - //TODO + private void prepareRequestForCdsBackend(List<File> files) { + final ManagedChannel channel = getManagedChannel(); + + files.forEach(zipFile -> { + try { + final BluePrintUploadInput request = generateBluePrintUploadInputBuilder(zipFile); + + // Send request to CDS Backend. + final Status responseStatus = bluePrintProcesssorHandler.sendRequest(request, channel); + + if (responseStatus.getCode() != SUCCESS_CODE) { + LOGGER.error("Failed to store the CBA archive into CDS DB due to {}", + responseStatus.getErrorMessage()); + } else { + LOGGER.info(responseStatus.getMessage()); + } + + } catch (Exception e) { + LOGGER.error("Failure due to", e); + } finally { + //Delete the file from the local disk. + boolean fileDeleted = zipFile.delete(); + + if (!fileDeleted) { + LOGGER.error("Could not able to delete the zip file {}", zipFile.toString()); + } + } + }); + } + + private ManagedChannel getManagedChannel() { + return ManagedChannelBuilder.forAddress(grpcAddress, grpcPort) + .usePlaintext() + .intercept(cdsSdcListenerAuthClientInterceptor) + .build(); + } + + private BluePrintUploadInput generateBluePrintUploadInputBuilder(File file) throws IOException { + byte[] bytes = FileUtils.readFileToByteArray(file); + FileChunk fileChunk = FileChunk.newBuilder().setChunk(ByteString.copyFrom(bytes)).build(); + + return BluePrintUploadInput.newBuilder() + .setFileChunk(fileChunk) + .build(); + } + + /** + * Extract files from the given path + * + * @param path where files reside. + * @return list of files. + */ + public Optional<List<File>> getFilesFromDisk(Path path) { + try (Stream<Path> fileTree = walk(path)) { + // Get the list of files from the path + return Optional.of(fileTree.filter(Files::isRegularFile) + .map(Path::toFile) + .collect(Collectors.toList())); + } catch (IOException e) { + LOGGER.error("Failed to find the file", e); + } + + return Optional.empty(); } } diff --git a/ms/cds-sdc-listener/application/src/main/resources/application.yml b/ms/cds-sdc-listener/application/src/main/resources/application.yml index 88de3b182..657ea9e80 100644 --- a/ms/cds-sdc-listener/application/src/main/resources/application.yml +++ b/ms/cds-sdc-listener/application/src/main/resources/application.yml @@ -14,6 +14,8 @@ listenerservice: keyStorePath: activateServerTLSAuth : false isUseHttpsWithDmaap: false - csarArchive: /opt/app/onap/cds-sdc-listener/csar-archive - cbaArchive: /opt/app/onap/cds/sdc-listener/cba-archive + archivePath: opt/app/onap/cds-sdc-listener/ + grpcAddress: localhost + grpcPort: 9111 + authHeader: Basic Y2NzZGthcHBzOmNjc2RrYXBwcw== |