diff options
12 files changed, 310 insertions, 208 deletions
diff --git a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/VnfDescriptorGeneratorImpl.java b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/VnfDescriptorGeneratorImpl.java index 902911b817..053bdde73a 100644 --- a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/VnfDescriptorGeneratorImpl.java +++ b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/VnfDescriptorGeneratorImpl.java @@ -1,178 +1,198 @@ - -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2020 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ -package org.openecomp.sdc.be.plugins.etsi.nfv.nsd.generator; + /* + * ============LICENSE_START======================================================= + * Copyright (C) 2020 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + package org.openecomp.sdc.be.plugins.etsi.nfv.nsd.generator; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOUtils; -import org.onap.sdc.tosca.services.YamlUtil; -import org.openecomp.core.utilities.file.FileContentHandler; -import org.openecomp.core.utilities.file.FileUtils; -import org.openecomp.sdc.be.model.ArtifactDefinition; -import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.builder.NsdToscaMetadataBuilder; -import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.exception.VnfDescriptorException; -import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.model.VnfDescriptor; -import org.openecomp.sdc.common.zip.exception.ZipException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import org.yaml.snakeyaml.Yaml; + import java.io.ByteArrayInputStream; + import java.io.FileNotFoundException; + import java.io.IOException; + import java.io.InputStream; + import java.nio.charset.StandardCharsets; + import java.util.Collections; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + import java.util.Optional; + import java.util.stream.Collectors; + import org.apache.commons.collections.MapUtils; + import org.apache.commons.io.FilenameUtils; + import org.apache.commons.io.IOUtils; + import org.onap.sdc.tosca.services.YamlUtil; + import org.openecomp.core.utilities.file.FileContentHandler; + import org.openecomp.core.utilities.file.FileUtils; + import org.openecomp.sdc.be.csar.storage.ArtifactStorageManager; + import org.openecomp.sdc.be.csar.storage.StorageFactory; + import org.openecomp.sdc.be.model.ArtifactDefinition; + import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.builder.NsdToscaMetadataBuilder; + import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.exception.VnfDescriptorException; + import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.model.VnfDescriptor; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + import org.springframework.stereotype.Component; + import org.yaml.snakeyaml.Yaml; -/** - * Implementation of a VNF Descriptor Generator - */ -@Component("vnfPackageGenerator") -public class VnfDescriptorGeneratorImpl implements VnfDescriptorGenerator { + /** + * Implementation of a VNF Descriptor Generator + */ + @Component("vnfPackageGenerator") + public class VnfDescriptorGeneratorImpl implements VnfDescriptorGenerator { - private static final Logger LOGGER = LoggerFactory.getLogger(VnfDescriptorGeneratorImpl.class); - private static final String SPACE_REGEX = "\\s+"; - private static final String CSAR = "csar"; - private static final String COLON = ":"; - private static final String EMPTY_STRING = ""; - private static final String SLASH = "/"; - private static final String DEFINITIONS_DIRECTORY = "Definitions"; - private static final String TOSCA_META_PATH = "TOSCA-Metadata/TOSCA.meta"; + private static final Logger LOGGER = LoggerFactory.getLogger(VnfDescriptorGeneratorImpl.class); + private static final String SPACE_REGEX = "\\s+"; + private static final String CSAR = "csar"; + private static final String COLON = ":"; + private static final String EMPTY_STRING = ""; + private static final String SLASH = "/"; + private static final String DEFINITIONS_DIRECTORY = "Definitions"; + private static final String TOSCA_META_PATH = "TOSCA-Metadata/TOSCA.meta"; - private static boolean isACsarArtifact(final ArtifactDefinition definition) { - return definition.getPayloadData() != null && definition.getArtifactName() != null && CSAR - .equalsIgnoreCase(FilenameUtils.getExtension(definition.getArtifactName())); - } + private static boolean isACsarArtifact(final ArtifactDefinition definition) { + return definition.getPayloadData() != null && definition.getArtifactName() != null && CSAR + .equalsIgnoreCase(FilenameUtils.getExtension(definition.getArtifactName())); + } - public Optional<VnfDescriptor> generate(final String name, final ArtifactDefinition onboardedPackageArtifact) throws VnfDescriptorException { - if (!isACsarArtifact(onboardedPackageArtifact)) { - return Optional.empty(); - } - final FileContentHandler fileContentHandler; - try { - fileContentHandler = FileUtils.getFileContentMapFromZip(onboardedPackageArtifact.getPayloadData()); - } catch (final ZipException e) { - final String errorMsg = String.format("Could not unzip artifact '%s' content", onboardedPackageArtifact.getArtifactName()); - throw new VnfDescriptorException(errorMsg, e); - } - if (MapUtils.isEmpty(fileContentHandler.getFiles())) { - return Optional.empty(); - } - final String mainDefinitionFile; - try { - mainDefinitionFile = getMainFilePathFromMetaFile(fileContentHandler).orElse(null); - } catch (final IOException e) { - final String errorMsg = String.format("Could not read main definition file of artifact '%s'", onboardedPackageArtifact.getArtifactName()); - throw new VnfDescriptorException(errorMsg, e); - } - LOGGER.debug("found main file: {}", mainDefinitionFile); - if (mainDefinitionFile == null) { - return Optional.empty(); - } - final VnfDescriptor vnfDescriptor = new VnfDescriptor(); - vnfDescriptor.setName(name); - final String vnfdFileName = FilenameUtils.getName(mainDefinitionFile); - vnfDescriptor.setVnfdFileName(vnfdFileName); - vnfDescriptor.setDefinitionFiles(getFiles(fileContentHandler, mainDefinitionFile)); - vnfDescriptor.setNodeType(getNodeType(getFileContent(fileContentHandler, mainDefinitionFile))); - return Optional.of(vnfDescriptor); - } + public Optional<VnfDescriptor> generate(final String name, final ArtifactDefinition onboardedPackageArtifact) throws VnfDescriptorException { + if (!isACsarArtifact(onboardedPackageArtifact)) { + return Optional.empty(); + } + final FileContentHandler fileContentHandler; + try { + final var artifactStorageManager = new StorageFactory().createArtifactStorageManager(); + final byte[] payloadData = onboardedPackageArtifact.getPayloadData(); + if (artifactStorageManager.isEnabled()) { + final var inputStream = artifactStorageManager.get(getFromPayload(payloadData, "bucket"), + getFromPayload(payloadData, "object") + ".reduced"); + fileContentHandler = FileUtils.getFileContentMapFromZip(inputStream); + } else { + fileContentHandler = FileUtils.getFileContentMapFromZip(new ByteArrayInputStream(payloadData)); + } + } catch (final IOException e) { + final String errorMsg = String.format("Could not unzip artifact '%s' content", onboardedPackageArtifact.getArtifactName()); + throw new VnfDescriptorException(errorMsg, e); + } + if (MapUtils.isEmpty(fileContentHandler.getFiles())) { + return Optional.empty(); + } + final String mainDefinitionFile; + try { + mainDefinitionFile = getMainFilePathFromMetaFile(fileContentHandler).orElse(null); + } catch (final IOException e) { + final String errorMsg = String.format("Could not read main definition file of artifact '%s'", + onboardedPackageArtifact.getArtifactName()); + throw new VnfDescriptorException(errorMsg, e); + } + LOGGER.debug("found main file: {}", mainDefinitionFile); + if (mainDefinitionFile == null) { + return Optional.empty(); + } + final var vnfDescriptor = new VnfDescriptor(); + vnfDescriptor.setName(name); + final String vnfdFileName = FilenameUtils.getName(mainDefinitionFile); + vnfDescriptor.setVnfdFileName(vnfdFileName); + vnfDescriptor.setDefinitionFiles(getFiles(fileContentHandler, mainDefinitionFile)); + vnfDescriptor.setNodeType(getNodeType(getFileContent(fileContentHandler, mainDefinitionFile))); + return Optional.of(vnfDescriptor); + } - private Map<String, byte[]> getFiles(final FileContentHandler fileContentHandler, final String filePath) { - final Map<String, byte[]> files = new HashMap<>(); - final byte[] fileContent = fileContentHandler.getFileContent(filePath); - if (fileContent != null) { - final String mainYmlFile = new String(fileContent); - LOGGER.debug("file content: {}", mainYmlFile); - files.put(appendDefinitionDirPath(filePath.substring(filePath.lastIndexOf(SLASH) + 1)), getVnfdWithoutTopologyTemplate(fileContent)); - final List<Object> imports = getImportFilesPath(mainYmlFile); - LOGGER.info("found imports {}", imports); - for (final Object importObject : imports) { - if (importObject != null) { - final String importFilename = importObject.toString(); - final String importFileFullPath = appendDefinitionDirPath(importFilename); - final byte[] importFileContent = fileContentHandler.getFileContent(importFileFullPath); - files.put(appendDefinitionDirPath(importFilename), importFileContent); - } - } - } - return files; - } + private String getFromPayload(final byte[] payload, final String name) { + final String[] strings = new String(payload).split("\n"); + for (final String str : strings) { + if (str.contains(name)) { + return str.split(": ")[1]; + } + } + return ""; + } - private String getFileContent(final FileContentHandler fileContentHandler, final String filePath) { - final byte[] fileContent = fileContentHandler.getFileContent(filePath); - if (fileContent != null) { - return new String(fileContent); - } - return null; - } + private Map<String, byte[]> getFiles(final FileContentHandler fileContentHandler, final String filePath) { + final Map<String, byte[]> files = new HashMap<>(); + final byte[] fileContent = fileContentHandler.getFileContent(filePath); + if (fileContent != null) { + final String mainYmlFile = new String(fileContent); + LOGGER.debug("file content: {}", mainYmlFile); + files.put(appendDefinitionDirPath(filePath.substring(filePath.lastIndexOf(SLASH) + 1)), getVnfdWithoutTopologyTemplate(fileContent)); + final List<Object> imports = getImportFilesPath(mainYmlFile); + LOGGER.info("found imports {}", imports); + for (final Object importObject : imports) { + if (importObject != null) { + final String importFilename = importObject.toString(); + final String importFileFullPath = appendDefinitionDirPath(importFilename); + final byte[] importFileContent = fileContentHandler.getFileContent(importFileFullPath); + files.put(appendDefinitionDirPath(importFilename), importFileContent); + } + } + } + return files; + } - private Optional<String> getMainFilePathFromMetaFile(final FileContentHandler fileContentHandler) throws IOException { - final Map<String, String> metaFileContent = getMetaFileContent(fileContentHandler); - final String mainFile = metaFileContent.get(NsdToscaMetadataBuilder.ENTRY_DEFINITIONS); - if (mainFile != null) { - return Optional.of(mainFile.replaceAll(SPACE_REGEX, EMPTY_STRING)); - } - LOGGER.error("{} entry not found in {}", NsdToscaMetadataBuilder.ENTRY_DEFINITIONS, TOSCA_META_PATH); - return Optional.empty(); - } + private String getFileContent(final FileContentHandler fileContentHandler, final String filePath) { + final byte[] fileContent = fileContentHandler.getFileContent(filePath); + if (fileContent != null) { + return new String(fileContent); + } + return null; + } - private Map<String, String> getMetaFileContent(final FileContentHandler fileContentHandler) throws IOException { - final InputStream inputStream = fileContentHandler.getFileContentAsStream(TOSCA_META_PATH); - if (inputStream == null) { - throw new FileNotFoundException("Unable find " + TOSCA_META_PATH + " file"); - } - final List<String> lines = IOUtils.readLines(inputStream, StandardCharsets.UTF_8); - return lines.stream().map(str -> str.split(COLON)).collect(Collectors.toMap(str -> str[0], str -> str.length > 1 ? str[1] : EMPTY_STRING)); - } + private Optional<String> getMainFilePathFromMetaFile(final FileContentHandler fileContentHandler) throws IOException { + final Map<String, String> metaFileContent = getMetaFileContent(fileContentHandler); + final String mainFile = metaFileContent.get(NsdToscaMetadataBuilder.ENTRY_DEFINITIONS); + if (mainFile != null) { + return Optional.of(mainFile.replaceAll(SPACE_REGEX, EMPTY_STRING)); + } + LOGGER.error("{} entry not found in {}", NsdToscaMetadataBuilder.ENTRY_DEFINITIONS, TOSCA_META_PATH); + return Optional.empty(); + } - private String appendDefinitionDirPath(final String filename) { - return DEFINITIONS_DIRECTORY + SLASH + filename; - } + private Map<String, String> getMetaFileContent(final FileContentHandler fileContentHandler) throws IOException { + final InputStream inputStream = fileContentHandler.getFileContentAsStream(TOSCA_META_PATH); + if (inputStream == null) { + throw new FileNotFoundException("Unable find " + TOSCA_META_PATH + " file"); + } + final List<String> lines = IOUtils.readLines(inputStream, StandardCharsets.UTF_8); + return lines.stream().map(str -> str.split(COLON)).collect(Collectors.toMap(str -> str[0], str -> str.length > 1 ? str[1] : EMPTY_STRING)); + } - private List<Object> getImportFilesPath(final String mainYmlFile) { - final Map<Object, Object> fileContentMap = new YamlUtil().yamlToObject(mainYmlFile, Map.class); - final Object importsObject = fileContentMap.get("imports"); - if (importsObject instanceof List) { - return (List<Object>) importsObject; - } - return Collections.emptyList(); - } + private String appendDefinitionDirPath(final String filename) { + return DEFINITIONS_DIRECTORY + SLASH + filename; + } - private byte[] getVnfdWithoutTopologyTemplate(final byte[] vnfdFileContent) { - final Yaml yaml = new Yaml(); - final Map<String, Object> toscaFileContent = (Map<String, Object>) yaml.load(new String(vnfdFileContent)); - toscaFileContent.remove("topology_template"); - return yaml.dumpAsMap(toscaFileContent).getBytes(); - } + private List<Object> getImportFilesPath(final String mainYmlFile) { + final Map<Object, Object> fileContentMap = new YamlUtil().yamlToObject(mainYmlFile, Map.class); + final Object importsObject = fileContentMap.get("imports"); + if (importsObject instanceof List) { + return (List<Object>) importsObject; + } + return Collections.emptyList(); + } - private String getNodeType(final String mainYmlFile) { - final Map<Object, Object> fileContentMap = new YamlUtil().yamlToObject(mainYmlFile, Map.class); - final Object nodeTypesObject = fileContentMap.get("node_types"); - if (nodeTypesObject instanceof Map && ((Map<String, Object>) nodeTypesObject).size() == 1) { - return ((Map<String, Object>) nodeTypesObject).keySet().iterator().next(); - } - return null; - } -} + private byte[] getVnfdWithoutTopologyTemplate(final byte[] vnfdFileContent) { + final Yaml yaml = new Yaml(); + final Map<String, Object> toscaFileContent = (Map<String, Object>) yaml.load(new String(vnfdFileContent)); + toscaFileContent.remove("topology_template"); + return yaml.dumpAsMap(toscaFileContent).getBytes(); + } + + private String getNodeType(final String mainYmlFile) { + final Map<Object, Object> fileContentMap = new YamlUtil().yamlToObject(mainYmlFile, Map.class); + final Object nodeTypesObject = fileContentMap.get("node_types"); + if (nodeTypesObject instanceof Map && ((Map<String, Object>) nodeTypesObject).size() == 1) { + return ((Map<String, Object>) nodeTypesObject).keySet().iterator().next(); + } + return null; + } + } diff --git a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/config/NsDescriptorConfig.java b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/config/NsDescriptorConfig.java index d17b03f7c4..96c9c813a6 100644 --- a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/config/NsDescriptorConfig.java +++ b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/config/NsDescriptorConfig.java @@ -1,4 +1,3 @@ - /* * ============LICENSE_START======================================================= * Copyright (C) 2021 Nordix Foundation @@ -19,17 +18,16 @@ */ package org.openecomp.sdc.be.plugins.etsi.nfv.nsd.generator.config; +import lombok.AllArgsConstructor; import lombok.Getter; /** * Configuration parameters for the Network Service Descriptor generation */ @Getter +@AllArgsConstructor public class NsDescriptorConfig { - final EtsiVersion nsVersion; + private final EtsiVersion nsVersion; - public NsDescriptorConfig(final EtsiVersion etsiVersion) { - this.nsVersion = etsiVersion; - } } diff --git a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/test/resources/configuration.yaml b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/test/resources/configuration.yaml new file mode 100644 index 0000000000..fa9fe016c4 --- /dev/null +++ b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/test/resources/configuration.yaml @@ -0,0 +1,2 @@ +externalCsarStore: + storageType: NONE # NONE, MINIO diff --git a/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb b/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb index 3b9d437ba6..aa9d3f275b 100644 --- a/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb +++ b/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb @@ -1261,3 +1261,14 @@ directives: - selectable - substitute - substitutable + +externalCsarStore: + storageType: NONE # NONE, MINIO + endpoint: + host: 127.0.0.1 + port: 9000 + secure: false + credentials: + accessKey: "login" + secretKey: "password" + tempPath: "/home/onap/temp/" diff --git a/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java b/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java index 6ac82a547d..cabda08ce7 100644 --- a/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java +++ b/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java @@ -144,6 +144,7 @@ public class Configuration extends BasicConfiguration { private EcompPortalConfig ecompPortal; private List<ArtifactConfiguration> artifacts; private Map<String, Map<String, List<String>>> componentAllowedInstanceTypes; + private ExternalCsarStore externalCsarStore; @SuppressWarnings("unchecked") private <K, V> Map<K, V> safeGetCapsInsensitiveMap(Map<K, V> map) { @@ -165,8 +166,8 @@ public class Configuration extends BasicConfiguration { public Map<String, Set<String>> getExcludedGroupTypesMapping() { return safeGetCapsInsensitiveMap(excludedGroupTypesMapping); } - - public List<Map<String, Map<String, String>>> getDefaultImports(){ + + public List<Map<String, Map<String, String>>> getDefaultImports() { return Collections.unmodifiableList(defaultImports); } @@ -483,4 +484,35 @@ public class Configuration extends BasicConfiguration { private String artifactType; private List<PathsAndNamesDefinition> pathsAndNamesDefinitions; } + + @Getter + @Setter + @NoArgsConstructor + public static class ExternalCsarStore { + + private String storageType; + private Endpoint endpoint; + private Credentials credentials; + private String tempPath; + + @Getter + @Setter + @NoArgsConstructor + public static class Endpoint { + + private String host; + private int port; + private boolean secure; + } + + @Getter + @Setter + @NoArgsConstructor + public static class Credentials { + + private String accessKey; + private String secretKey; + } + } + } diff --git a/common-app-api/src/main/java/org/openecomp/sdc/be/csar/storage/ArtifactStorageManager.java b/common-app-api/src/main/java/org/openecomp/sdc/be/csar/storage/ArtifactStorageManager.java index 0a4f355642..f7d611b717 100644 --- a/common-app-api/src/main/java/org/openecomp/sdc/be/csar/storage/ArtifactStorageManager.java +++ b/common-app-api/src/main/java/org/openecomp/sdc/be/csar/storage/ArtifactStorageManager.java @@ -47,6 +47,8 @@ public interface ArtifactStorageManager { */ ArtifactInfo upload(String vspId, String versionId, InputStream fileToUpload); + void put(String vspId, String name, InputStream fileToUpload); + /** * Checks if the Artifact Storage is enabled. * @@ -63,5 +65,7 @@ public interface ArtifactStorageManager { InputStream get(final ArtifactInfo artifactInfo); + InputStream get(final String vspId, final String versionId); + void delete(ArtifactInfo artifactInfo); } diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java index 0a48c2233c..850b4e1dda 100644 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java @@ -108,6 +108,18 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage LOGGER.info("Bucket '{}' already exists.", vspId); } + put(vspId, name, fileToUpload); + + } catch (final Exception e) { + throw new ArtifactStorageException("Failed to upload artifact", e); + } + + return new MinIoArtifactInfo(vspId, name); + } + + @Override + public void put(final String vspId, final String name, final InputStream fileToUpload) { + try { minioClient.putObject( PutObjectArgs.builder() .bucket(vspId) @@ -115,12 +127,9 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage .stream(fileToUpload, fileToUpload.available(), -1) .build() ); - } catch (final Exception e) { throw new ArtifactStorageException("Failed to upload artifact", e); } - - return new MinIoArtifactInfo(vspId, name); } @Override @@ -132,9 +141,18 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage public InputStream get(final ArtifactInfo artifactInfo) { final MinIoArtifactInfo minioObject = (MinIoArtifactInfo) artifactInfo; try { + return get(minioObject.getBucket(), minioObject.getObjectName()); + } catch (final Exception e) { + throw new ArtifactStorageException("Failed to get Object", e); + } + } + + @Override + public InputStream get(final String bucketID, final String objectID) { + try { return minioClient.getObject(GetObjectArgs.builder() - .bucket(minioObject.getBucket()) - .object(minioObject.getObjectName()) + .bucket(bucketID) + .object(objectID) .build()); } catch (final Exception e) { throw new ArtifactStorageException("Failed to get Object", e); diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java index 3fa22d41be..aec69d5bd7 100644 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java @@ -37,6 +37,11 @@ public class NoneStorageManager implements ArtifactStorageManager { } @Override + public void put(final String vspId, final String versionId, final InputStream fileToUpload) { + throw new UnsupportedOperationException(); + } + + @Override public ArtifactStorageConfig getStorageConfiguration() { throw new UnsupportedOperationException(); } @@ -47,6 +52,11 @@ public class NoneStorageManager implements ArtifactStorageManager { } @Override + public InputStream get(final String vspId, final String versionId) { + throw new UnsupportedOperationException(); + } + + @Override public void delete(final ArtifactInfo artifactInfo) { throw new UnsupportedOperationException(); } diff --git a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java index 6fe7f9dd0a..e8ee6b3c4b 100644 --- a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java +++ b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java @@ -32,6 +32,7 @@ import static org.openecomp.sdc.common.errors.Messages.NO_FILE_WAS_UPLOADED_OR_F import static org.openecomp.sdc.common.errors.Messages.PACKAGE_PROCESS_ERROR; import static org.openecomp.sdc.common.errors.Messages.UNEXPECTED_PROBLEM_HAPPENED_WHILE_GETTING; +import java.io.ByteArrayInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -183,8 +184,12 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate final var vspDetails = vendorSoftwareProductManager.getVsp(vspId, version); final Response response = processOnboardPackage(onboardPackageInfo, vspDetails, errorMessages); final UploadFileResponseDto entity = (UploadFileResponseDto) response.getEntity(); - if (artifactStorageManager.isEnabled() && !entity.getErrors().isEmpty()) { - artifactStorageManager.delete(artifactInfo); + if (artifactStorageManager.isEnabled()) { + if (!entity.getErrors().isEmpty()) { + artifactStorageManager.delete(artifactInfo); + } else { + artifactStorageManager.put(vspId, versionId + ".reduced", new ByteArrayInputStream(fileToUploadBytes)); + } } return response; } @@ -255,7 +260,8 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate FilesDataStructure fileDataStructure = copyFilesDataStructureDtoToFilesDataStructure(fileDataStructureDto); ValidationResponse response = candidateManager.updateFilesDataStructure(vspId, new Version(versionId), fileDataStructure); if (!response.isValid()) { - return Response.status(EXPECTATION_FAILED).entity(new MapValidationResponseToDto().applyMapping(response, ValidationResponseDto.class)) + return Response.status(EXPECTATION_FAILED) + .entity(new MapValidationResponseToDto().applyMapping(response, ValidationResponseDto.class)) .build(); } return Response.ok(fileDataStructureDto).build(); diff --git a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java index 60ff1c172a..d6df24e4c2 100644 --- a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java +++ b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java @@ -30,12 +30,12 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Function; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.onap.sdc.tosca.services.YamlUtil; import org.openecomp.core.utilities.json.JsonUtil; -import org.openecomp.sdc.common.zip.ZipUtils; -import org.openecomp.sdc.common.zip.exception.ZipException; /** * The type File utils. @@ -206,20 +206,24 @@ public class FileUtils { /** * Gets file content map from zip. * - * @param zipData the zip data + * @param inputStream the zip data * @return the file content map from zip - * @throws ZipException when an error occurs while extracting zip files + * @throws IOException when an error occurs while extracting zip files */ - public static FileContentHandler getFileContentMapFromZip(byte[] zipData) throws ZipException { - final Map<String, byte[]> zipFileAndByteMap = ZipUtils.readZip(zipData, true); - final FileContentHandler fileContentHandler = new FileContentHandler(); - zipFileAndByteMap.forEach((path, bytes) -> { - if (bytes == null) { - fileContentHandler.addFolder(path); + public static FileContentHandler getFileContentMapFromZip(final InputStream inputStream) throws IOException { + + final var zipInputStream = new ZipInputStream(inputStream); + ZipEntry zipEntry; + final var fileContentHandler = new FileContentHandler(); + while ((zipEntry = zipInputStream.getNextEntry()) != null) { + final var entryName = zipEntry.getName(); + if (zipEntry.isDirectory()) { + fileContentHandler.addFolder(entryName); } else { - fileContentHandler.addFile(path, bytes); + fileContentHandler.addFile(entryName, zipInputStream.readAllBytes()); } - }); + + } return fileContentHandler; } diff --git a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java index facfe57622..05ec5a478d 100644 --- a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java +++ b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java @@ -35,7 +35,6 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.io.IOUtils; import org.junit.Assert; import org.junit.Test; import org.openecomp.sdc.common.zip.exception.ZipException; @@ -47,7 +46,7 @@ import org.openecomp.sdc.common.zip.exception.ZipException; public class FileUtilsTest { private static final String TEST_RESOURCE = FileUtilsTest.class.getPackage().getName() - .replace('.', '/') + "/test-resource.txt"; + .replace('.', '/') + "/test-resource.txt"; private static final Function<InputStream, Integer> TEST_FUNCTION = (s) -> { @@ -82,13 +81,11 @@ public class FileUtilsTest { public void testWriteFilesFromFileContentHandler() throws IOException, ZipException { final Path tempDirectory = Files.createTempDirectory("CSAR_" + System.currentTimeMillis()); try { - byte[] uploadedFileData = - IOUtils.toByteArray(FileUtilsTest.class.getResource("resource-Spgw-csar-ZTE.csar")); - final FileContentHandler contentMap = FileUtils.getFileContentMapFromZip(uploadedFileData); + final var contentMap = FileUtils.getFileContentMapFromZip(FileUtilsTest.class.getResourceAsStream("resource-Spgw-csar-ZTE.csar")); final Map<String, String> filePaths = FileUtils.writeFilesFromFileContentHandler(contentMap, tempDirectory); assertThat("The file map should not be empty", filePaths, is(not(anEmptyMap()))); - assertThat("The file map should have size 20", filePaths, is(aMapWithSize(20))); + assertThat("The file map should have size 20", filePaths, is(aMapWithSize(18))); for (final Map.Entry<String, String> fileEntry : filePaths.entrySet()) { final File f = new File(fileEntry.getValue()); assertThat(String.format("The file '%s' is expected to", f.getAbsolutePath()), f.exists(), is(true)); @@ -151,7 +148,7 @@ public class FileUtilsTest { @Test public void testConvertToBytes() { byte[] bytesArray = FileUtils.convertToBytes(Stream.of("Json", "Util", "Test").collect(Collectors.toList()), - FileUtils.FileExtension.YAML); + FileUtils.FileExtension.YAML); Assert.assertNotNull(bytesArray); } @@ -159,7 +156,7 @@ public class FileUtilsTest { @Test public void testConvertToBytesNotYaml() { byte[] bytesArray = FileUtils.convertToBytes(Stream.of("Json", "Util", "Test").collect(Collectors.toList()), - FileUtils.FileExtension.JSON); + FileUtils.FileExtension.JSON); Assert.assertNotNull(bytesArray); } @@ -172,7 +169,7 @@ public class FileUtilsTest { @Test public void testConvertToInputStream() { InputStream inputStream = FileUtils.convertToInputStream(Stream.of("Json", "Util", "Test") - .collect(Collectors.toList()), FileUtils.FileExtension.YAML); + .collect(Collectors.toList()), FileUtils.FileExtension.YAML); Assert.assertNotNull(inputStream); } @@ -183,13 +180,13 @@ public class FileUtilsTest { } @Test - public void testLoadFileToInputStream() throws IOException{ + public void testLoadFileToInputStream() throws IOException { int i; StringBuilder builder = new StringBuilder(20); InputStream inputStream = FileUtils.loadFileToInputStream( - "org/openecomp/core/utilities/file/testFileUtils.txt"); - while((i = inputStream.read())!=-1) { - builder.append((char)i); + "org/openecomp/core/utilities/file/testFileUtils.txt"); + while ((i = inputStream.read()) != -1) { + builder.append((char) i); } Assert.assertNotNull(inputStream); diff --git a/openecomp-be/lib/openecomp-sdc-enrichment-lib/openecomp-sdc-enrichment-impl/src/main/java/org/openecomp/sdc/enrichment/impl/external/artifact/MonitoringMibEnricher.java b/openecomp-be/lib/openecomp-sdc-enrichment-lib/openecomp-sdc-enrichment-impl/src/main/java/org/openecomp/sdc/enrichment/impl/external/artifact/MonitoringMibEnricher.java index 06a5b3a2c2..02d0328d3b 100644 --- a/openecomp-be/lib/openecomp-sdc-enrichment-lib/openecomp-sdc-enrichment-impl/src/main/java/org/openecomp/sdc/enrichment/impl/external/artifact/MonitoringMibEnricher.java +++ b/openecomp-be/lib/openecomp-sdc-enrichment-lib/openecomp-sdc-enrichment-impl/src/main/java/org/openecomp/sdc/enrichment/impl/external/artifact/MonitoringMibEnricher.java @@ -19,6 +19,7 @@ import static org.openecomp.sdc.tosca.services.ToscaConstants.SERVICE_TEMPLATE_F import static org.openecomp.sdc.tosca.services.ToscaConstants.SUBSTITUTE_SERVICE_TEMPLATE_PROPERTY_NAME; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -41,7 +42,6 @@ import org.openecomp.core.model.types.ServiceArtifact; import org.openecomp.core.utilities.file.FileContentHandler; import org.openecomp.core.utilities.file.FileUtils; import org.openecomp.sdc.common.errors.Messages; -import org.openecomp.sdc.common.zip.exception.ZipException; import org.openecomp.sdc.datatypes.error.ErrorLevel; import org.openecomp.sdc.datatypes.error.ErrorMessage; import org.openecomp.sdc.enrichment.EnrichmentInfo; @@ -208,8 +208,8 @@ public class MonitoringMibEnricher implements ExternalArtifactEnricherInterface } FileContentHandler mibs; try { - mibs = FileUtils.getFileContentMapFromZip(FileUtils.toByteArray(monitoringArtifactInfo.getContent())); - } catch (ZipException ex) { + mibs = FileUtils.getFileContentMapFromZip(monitoringArtifactInfo.getContent()); + } catch (final IOException ex) { log.error("Failed to get file content map from zip ", ex); ErrorMessage.ErrorMessageUtil.addMessage(mibServiceArtifact.getName() + "." + type.name(), errors) .add(new ErrorMessage(ErrorLevel.ERROR, Messages.INVALID_ZIP_FILE.getErrorMessage())); |