diff options
Diffstat (limited to 'catalog-be/src/main/java')
| -rw-r--r-- | catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java | 209 | ||||
| -rw-r--r-- | catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ToscaExportHandler.java | 161 |
2 files changed, 275 insertions, 95 deletions
diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java index 2721fda2c5..e50523854d 100644 --- a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java @@ -22,14 +22,36 @@ package org.openecomp.sdc.be.tosca; import fj.data.Either; +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang.WordUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.ImmutableTriple; import org.apache.commons.lang3.tuple.Triple; +import org.onap.sdc.tosca.services.YamlUtil; import org.openecomp.sdc.be.components.impl.ImportUtils; +import org.openecomp.sdc.be.components.impl.exceptions.ByResponseFormatComponentException; import org.openecomp.sdc.be.config.Configuration.ArtifactTypeConfig; import org.openecomp.sdc.be.config.ConfigurationManager; import org.openecomp.sdc.be.dao.api.ActionStatus; @@ -45,6 +67,7 @@ import org.openecomp.sdc.be.model.Component; import org.openecomp.sdc.be.model.ComponentInstance; import org.openecomp.sdc.be.model.InterfaceDefinition; import org.openecomp.sdc.be.model.LifecycleStateEnum; +import org.openecomp.sdc.be.model.Product; import org.openecomp.sdc.be.model.Resource; import org.openecomp.sdc.be.model.Service; import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade; @@ -57,6 +80,7 @@ import org.openecomp.sdc.be.resources.data.SdcSchemaFilesData; import org.openecomp.sdc.be.tosca.model.ToscaTemplate; import org.openecomp.sdc.be.tosca.utils.OperationArtifactUtil; import org.openecomp.sdc.be.utils.CommonBeUtils; +import org.openecomp.sdc.be.utils.TypeUtils.ToscaTagNamesEnum; import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum; import org.openecomp.sdc.common.api.ArtifactTypeEnum; import org.openecomp.sdc.common.impl.ExternalConfiguration; @@ -69,26 +93,7 @@ import org.openecomp.sdc.common.util.ValidationUtils; import org.openecomp.sdc.common.zip.ZipUtils; import org.openecomp.sdc.exception.ResponseFormat; import org.springframework.beans.factory.annotation.Autowired; - -import java.io.BufferedOutputStream; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; +import org.yaml.snakeyaml.Yaml; /** * @author tg851x @@ -290,16 +295,16 @@ public class CsarUtils { Either<byte[], ResponseFormat> latestSchemaFilesFromCassandra = getLatestSchemaFilesFromCassandra(); if(latestSchemaFilesFromCassandra.isRight()){ - log.error("Error retrieving SDC Schema files from cassandra" ); + log.error("Error retrieving SDC Schema files from cassandra"); return Either.right(latestSchemaFilesFromCassandra.right().value()); } - //add files from retrieved SDC.zip to Definitions folder in CSAR - Either<ZipOutputStream, ResponseFormat> addSchemaFilesFromCassandra = addSchemaFilesFromCassandra(zip, latestSchemaFilesFromCassandra.left().value()); + final byte[] schemaFileZip = latestSchemaFilesFromCassandra.left().value(); - if(addSchemaFilesFromCassandra.isRight()){ - return addSchemaFilesFromCassandra; - } + final List<String> nodesFromPackage = findNonRootNodesFromPackage(dependencies); + + //add files from retrieved SDC.zip to Definitions folder in CSAR + addSchemaFilesFromCassandra(zip, schemaFileZip, nodesFromPackage); Either<CsarDefinition, ResponseFormat> collectedComponentCsarDefinition = collectComponentCsarDefinition(component); @@ -311,8 +316,8 @@ public class CsarUtils { for (CsarEntryGenerator generator: generators) { log.debug("Invoking CsarEntryGenerator: {}", generator.getClass().getName()); for (Entry<String, byte[]> pluginGeneratedFile : generator.generateCsarEntries(component).entrySet()) { - zip.putNextEntry(new ZipEntry(pluginGeneratedFile.getKey())); - zip.write(pluginGeneratedFile.getValue()); + zip.putNextEntry(new ZipEntry(pluginGeneratedFile.getKey())); + zip.write(pluginGeneratedFile.getValue()); } } } @@ -320,6 +325,84 @@ public class CsarUtils { return writeAllFilesToCsar(component, collectedComponentCsarDefinition.left().value(), zip, isInCertificationRequest); } + /** + * Create a list of all derived nodes found on the package + * + * @param dependencies all node dependencies + * @return a list of nodes + */ + private List<String> findNonRootNodesFromPackage(final List<Triple<String, String, Component>> dependencies) { + final List<String> nodes = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(dependencies)) { + final String NATIVE_ROOT = "tosca.nodes.Root"; + dependencies.forEach(dependency -> { + if (dependency.getRight() instanceof Resource) { + final Resource resource = (Resource) dependency.getRight(); + if (CollectionUtils.isNotEmpty(resource.getDerivedList())) { + resource.getDerivedList().stream() + .filter(node -> !nodes.contains(node) && !NATIVE_ROOT.equalsIgnoreCase(node)) + .forEach(node -> nodes.add(node)); + } + } + }); + } + return nodes; + } + + /** + * Writes a new zip entry + * + * @param zipInputStream the zip entry to be read + * @return a map of the given zip entry + */ + private Map<String, Object> readYamlZipEntry(final ZipInputStream zipInputStream) throws IOException { + final int initSize = 2048; + final StringBuilder zipEntry = new StringBuilder(); + final byte[] buffer = new byte[initSize]; + int read = 0; + while ((read = zipInputStream.read(buffer, 0, initSize)) >= 0) { + zipEntry.append(new String(buffer, 0, read)); + } + + return (Map<String, Object>) new Yaml().load(zipEntry.toString()); + } + + /** + * Filters and removes all duplicated nodes found + * + * @param nodesFromPackage a List of all derived nodes found on the given package + * @param nodesFromArtifactFile represents the nodes.yml file stored in Cassandra + * @return a nodes Map updated + */ + private Map<String, Object> updateNodeYml(final List<String> nodesFromPackage, + final Map<String, Object> nodesFromArtifactFile) { + + if (MapUtils.isNotEmpty(nodesFromArtifactFile)) { + final String nodeTypeBlock = ToscaTagNamesEnum.NODE_TYPES.getElementName(); + final Map<String, Object> nodeTypes = (Map<String, Object>) nodesFromArtifactFile.get(nodeTypeBlock); + nodesFromPackage.stream() + .filter(nodeTypes::containsKey) + .forEach(nodeTypes::remove); + + nodesFromArtifactFile.replace(nodeTypeBlock, nodeTypes); + } + + return nodesFromArtifactFile; + } + + /** + * Updates the zip entry from the given parameters + * + * @param byteArrayOutputStream an output stream in which the data is written into a byte array. + * @param nodesYaml a Map of nodes to be written + */ + private void updateZipEntry(final ByteArrayOutputStream byteArrayOutputStream, + final Map<String, Object> nodesYaml) throws IOException { + if (MapUtils.isNotEmpty(nodesYaml)) { + byteArrayOutputStream.write(new YamlUtil().objectToYaml(nodesYaml).getBytes()); + } + } + private Either<ZipOutputStream, ResponseFormat> getZipOutputStreamResponseFormatEither(ZipOutputStream zip, List<Triple<String, String, Component>> dependencies, Map<String, ImmutableTriple<String, String, Component>> innerComponentsCache) throws IOException { String fileName; if (dependencies != null && !dependencies.isEmpty()) { @@ -374,43 +457,57 @@ public class CsarUtils { return null; } - private Either<ZipOutputStream, ResponseFormat> addSchemaFilesFromCassandra(ZipOutputStream zip, byte[] schemaFileZip) { - - final int initSize = 2048; - - log.debug("Starting copy from Schema file zip to CSAR zip"); - try (final ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(schemaFileZip)); - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - final BufferedOutputStream bos = new BufferedOutputStream(out, initSize)) { - - ZipEntry entry; - while ((entry = zipStream.getNextEntry()) != null) { - ZipUtils.checkForZipSlipInRead(entry); - final String entryName = entry.getName(); - int readSize = initSize; - final byte[] entryData = new byte[initSize]; - + private void addSchemaFilesFromCassandra(final ZipOutputStream zip, + final byte[] schemaFileZip, + final List<String> nodesFromPackage) { + final int initSize = 2048; + log.debug("Starting copy from Schema file zip to CSAR zip"); + try (final ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(schemaFileZip)); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + final BufferedOutputStream bos = new BufferedOutputStream(out, initSize)) { + + ZipEntry entry; + while ((entry = zipStream.getNextEntry()) != null) { + ZipUtils.checkForZipSlipInRead(entry); + final String entryName = entry.getName(); + int readSize = initSize; + final byte[] entryData = new byte[initSize]; + if (entryName.equalsIgnoreCase("nodes.yml")) { + handleNode(zipStream, out, nodesFromPackage); + } else { while ((readSize = zipStream.read(entryData, 0, readSize)) != -1) { bos.write(entryData, 0, readSize); } - bos.flush(); - out.flush(); - zip.putNextEntry(new ZipEntry(DEFINITIONS_PATH + entryName)); - zip.write(out.toByteArray()); - zip.flush(); - out.reset(); } - } catch (final Exception e) { - log.error("Error while writing the SDC schema file to the CSAR", e); - return Either.right(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR)); + out.flush(); + zip.putNextEntry(new ZipEntry(DEFINITIONS_PATH + entryName)); + zip.write(out.toByteArray()); + zip.flush(); + out.reset(); } - - log.debug("Finished coppy from Schema file zip to CSAR zip"); - return Either.left(zip); + } catch (final Exception e) { + log.error("Error while writing the SDC schema file to the CSAR", e); + throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR)); } + log.debug("Finished copy from Schema file zip to CSAR zip"); + } + /** + * Handles the nodes.yml zip entry, updating the nodes.yml to avoid duplicated nodes on it. + * + * @param zipInputStream the zip entry to be read + * @param byteArrayOutputStream an output stream in which the data is written into a byte array. + * @param nodesFromPackage list of all nodes found on the onboarded package + */ + private void handleNode(final ZipInputStream zipInputStream, + final ByteArrayOutputStream byteArrayOutputStream, + final List<String> nodesFromPackage) throws IOException { + final Map<String, Object> nodesFromArtifactFile = readYamlZipEntry(zipInputStream); + final Map<String, Object> nodesYaml = updateNodeYml(nodesFromPackage, nodesFromArtifactFile); + updateZipEntry(byteArrayOutputStream, nodesYaml); + } private void addInnerComponentsToCache(Map<String, ImmutableTriple<String, String, Component>> componentCache, Component childComponent) { diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ToscaExportHandler.java b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ToscaExportHandler.java index 976842136f..64afee7904 100644 --- a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ToscaExportHandler.java +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ToscaExportHandler.java @@ -163,7 +163,8 @@ public class ToscaExportHandler { private static final String FAILED_TO_GET_DEFAULT_IMPORTS_CONFIGURATION = "convertToToscaTemplate - failed to get Default Imports section from configuration"; private static final String NOT_SUPPORTED_COMPONENT_TYPE = "Not supported component type {}"; private static final List<Map<String, Map<String, String>>> DEFAULT_IMPORTS = ConfigurationManager - .getConfigurationManager().getConfiguration().getDefaultImports(); + .getConfigurationManager().getConfiguration().getDefaultImports(); + private static final String NATIVE_ROOT = "tosca.nodes.Root"; private static YamlUtil yamlUtil = new YamlUtil(); public ToscaExportHandler(){} @@ -505,60 +506,142 @@ public class ToscaExportHandler { return Either.left(new ImmutablePair<>(toscaTemplate, componentCache)); } - private void createDependency(Map<String, Component> componentCache, List<Map<String, Map<String, String>>> imports, - List<Triple<String, String, Component>> dependecies, ComponentInstance ci) { - Map<String, String> files = new HashMap<>(); - Map<String, Map<String, String>> importsListMember = new HashMap<>(); - StringBuilder keyNameBuilder; - - Component componentRI = componentCache.get(ci.getComponentUid()); + private void createDependency(final Map<String, Component> componentCache, + final List<Map<String, Map<String, String>>> imports, + final List<Triple<String, String, Component>> dependencies, + final ComponentInstance componentInstance) { + log.debug("createDependency componentCache {}",componentCache); + final Component componentRI = componentCache.get(componentInstance.getComponentUid()); if (componentRI == null) { // all resource must be only once! - Either<Component, StorageOperationStatus> resource = toscaOperationFacade - .getToscaFullElement(ci.getComponentUid()); + final Either<Component, StorageOperationStatus> resource = toscaOperationFacade + .getToscaFullElement(componentInstance.getComponentUid()); if ((resource.isRight()) && (log.isDebugEnabled())) { - log.debug("Failed to fetch resource with id {} for instance {}",ci.getComponentUid() ,ci.getUniqueId()); + log.debug("Failed to fetch resource with id {} for instance {}", componentInstance.getComponentUid(), + componentInstance.getUniqueId()); return ; } + final Component fetchedComponent = resource.left().value(); + setComponentCache(componentCache, componentInstance, fetchedComponent); + addDependencies(imports, dependencies, fetchedComponent); + } + } - Component fetchedComponent = resource.left().value(); - componentCache.put(fetchedComponent.getUniqueId(), fetchedComponent); + /** + * Sets a componentCache from the given component/resource. + */ + private void setComponentCache(final Map<String, Component> componentCache, + final ComponentInstance componentInstance, + final Component fetchedComponent) { + componentCache.put(fetchedComponent.getUniqueId(), fetchedComponent); + if (componentInstance.getOriginType() == OriginTypeEnum.ServiceProxy) { + final Either<Component, StorageOperationStatus> sourceService = toscaOperationFacade + .getToscaFullElement(componentInstance.getSourceModelUid()); + if (sourceService.isRight() && (log.isDebugEnabled())) { + log.debug("Failed to fetch source service with id {} for proxy {}", + componentInstance.getSourceModelUid(), componentInstance.getUniqueId()); + } + final Component fetchedSource = sourceService.left().value(); + componentCache.put(fetchedSource.getUniqueId(), fetchedSource); + } + } - if (ci.getOriginType() == OriginTypeEnum.ServiceProxy){ - Either<Component, StorageOperationStatus> sourceService = toscaOperationFacade - .getToscaFullElement(ci.getSourceModelUid()); - if (sourceService.isRight() && (log.isDebugEnabled())) { - log.debug("Failed to fetch source service with id {} for proxy {}", ci.getSourceModelUid(), ci.getUniqueId()); - } - Component fetchedSource = sourceService.left().value(); - componentCache.put(fetchedSource.getUniqueId(), fetchedSource); + /** + * Retrieves all derived_from nodes and stores it in a predictable order. + */ + private void addDependencies(final List<Map<String, Map<String, String>>> imports, + final List<Triple<String, String, Component>> dependencies, + final Component fetchedComponent) { + final Set<Component> componentsList = new LinkedHashSet<>(); + if (fetchedComponent instanceof Resource) { + log.debug("fetchedComponent is a resource {}",fetchedComponent); + + final Optional<Map<String, String>> derivedFromMapOfIdToName = getDerivedFromMapOfIdToName(fetchedComponent, componentsList); + if (derivedFromMapOfIdToName.isPresent()) { + derivedFromMapOfIdToName.get().entrySet().forEach(entry -> { + log.debug("Started entry.getValue() : {}",entry.getValue()); + if (!NATIVE_ROOT.equals(entry.getValue())) { + Either<Resource, StorageOperationStatus> resourcefetched = toscaOperationFacade + .getToscaElement(entry.getKey()); + if (resourcefetched != null && resourcefetched.isLeft()) { + componentsList.add(resourcefetched.left().value()); + } + } + }); } + setImports(imports, dependencies, componentsList); + } + } - componentRI = fetchedComponent; + /** + * Returns all derived_from nodes found. + */ + private Optional<Map<String, String>> getDerivedFromMapOfIdToName(final Component fetchedComponent, + final Set<Component> componentsList) { + final Resource parentResource = (Resource) fetchedComponent; + Map<String, String> derivedFromMapOfIdToName = new HashMap<>(); + if(CollectionUtils.isNotEmpty(parentResource.getComponentInstances())) { + componentsList.add(fetchedComponent); + for (final ComponentInstance componentInstance : parentResource.getComponentInstances()) { + final Either<Resource, StorageOperationStatus> resourcefetched = toscaOperationFacade + .getToscaElement(componentInstance.getComponentUid()); + if (resourcefetched != null && resourcefetched.isLeft()) { + final Map<String, String> derivedWithId = resourcefetched.left().value().getDerivedFromMapOfIdToName(); + if (MapUtils.isNotEmpty(derivedWithId)) { + derivedFromMapOfIdToName.putAll(derivedWithId); + } + } + } + } else { + derivedFromMapOfIdToName = parentResource.getDerivedFromMapOfIdToName(); + } + log.debug("Started derivedFromMapOfIdToName: {}", derivedFromMapOfIdToName); + return Optional.ofNullable(derivedFromMapOfIdToName); + } - Map<String, ArtifactDefinition> toscaArtifacts = componentRI.getToscaArtifacts(); - ArtifactDefinition artifactDefinition = toscaArtifacts.get(ASSET_TOSCA_TEMPLATE); + /** + * Creates a resource map and adds it to the import list. + */ + private void setImports(final List<Map<String, Map<String, String>>> imports, + final List<Triple<String, String, Component>> dependencies, + final Set<Component> componentsList) { + componentsList.forEach(component -> { + final Map<String, ArtifactDefinition> toscaArtifacts = component.getToscaArtifacts(); + final ArtifactDefinition artifactDefinition = toscaArtifacts.get(ASSET_TOSCA_TEMPLATE); if (artifactDefinition != null) { - String artifactName = artifactDefinition.getArtifactName(); + final Map<String, String> files = new HashMap<>(); + final String artifactName = artifactDefinition.getArtifactName(); files.put(IMPORTS_FILE_KEY, artifactName); - keyNameBuilder = new StringBuilder(); - keyNameBuilder.append(fetchedComponent.getComponentType().toString().toLowerCase()); + final StringBuilder keyNameBuilder = new StringBuilder(); + keyNameBuilder.append(component.getComponentType().toString().toLowerCase()); keyNameBuilder.append("-"); - keyNameBuilder.append(ci.getComponentName()); - importsListMember.put(keyNameBuilder.toString(), files); - imports.add(importsListMember); - dependecies.add(new ImmutableTriple<>(artifactName, - artifactDefinition.getEsId(), fetchedComponent)); - - if (!ModelConverter.isAtomicComponent(componentRI)) { - importsListMember = new HashMap<>(); - Map<String, String> interfaceFiles = new HashMap<>(); + keyNameBuilder.append(component.getName()); + addImports(imports, keyNameBuilder, files); + dependencies + .add(new ImmutableTriple<String, String, Component>(artifactName, artifactDefinition.getEsId(), + component)); + + if (!ModelConverter.isAtomicComponent(component)) { + final Map<String, String> interfaceFiles = new HashMap<>(); interfaceFiles.put(IMPORTS_FILE_KEY, getInterfaceFilename(artifactName)); keyNameBuilder.append("-interface"); - importsListMember.put(keyNameBuilder.toString(), interfaceFiles); - imports.add(importsListMember); + addImports(imports, keyNameBuilder, interfaceFiles); } } + }); + } + + /** + * Adds the found resource to the import definition list. + */ + private void addImports(final List<Map<String, Map<String, String>>> imports, + final StringBuilder keyNameBuilder, + final Map<String, String> files) { + final String mapKey = keyNameBuilder.toString(); + if (imports.stream().allMatch(stringMapMap -> stringMapMap.get(mapKey) == null)) { + final Map<String, Map<String, String>> importsListMember = new HashMap<>(); + importsListMember.put(keyNameBuilder.toString(), files); + imports.add(importsListMember); } } @@ -952,7 +1035,7 @@ public class ToscaExportHandler { toscaNodeType.setDescription(component.getDescription()); } else { String derivedFrom = null != component.getDerivedFromGenericType() ? component.getDerivedFromGenericType() - : "tosca.nodes.Root"; + : NATIVE_ROOT; toscaNodeType.setDerived_from(derivedFrom); } return toscaNodeType; |