summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvasraz <vasyl.razinkov@est.tech>2021-11-26 14:40:29 +0000
committerMichael Morris <michael.morris@est.tech>2021-11-29 14:36:12 +0000
commit6c32cbc7343bd56f62007451071c4b26c875382f (patch)
treefa3250eb24d518e5f15cb629bdb462b96be769c5
parente9171113772c0c7dc3f7514ffd0660d1e0f8ef02 (diff)
Add VNFD to NSD when using S3 storage
Change-Id: I8eb810fadfe54b335647e71e21115514963b4be7 Signed-off-by: Vasyl Razinkov <vasyl.razinkov@est.tech> Issue-ID: SDC-3792
-rw-r--r--catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/VnfDescriptorGeneratorImpl.java350
-rw-r--r--catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/config/NsDescriptorConfig.java8
-rw-r--r--catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/test/resources/configuration.yaml2
-rw-r--r--catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb11
-rw-r--r--common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java36
-rw-r--r--common-app-api/src/main/java/org/openecomp/sdc/be/csar/storage/ArtifactStorageManager.java4
-rw-r--r--common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java28
-rw-r--r--common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java10
-rw-r--r--openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java12
-rw-r--r--openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java28
-rw-r--r--openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java23
-rw-r--r--openecomp-be/lib/openecomp-sdc-enrichment-lib/openecomp-sdc-enrichment-impl/src/main/java/org/openecomp/sdc/enrichment/impl/external/artifact/MonitoringMibEnricher.java6
12 files changed, 310 insertions, 208 deletions
diff --git a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/VnfDescriptorGeneratorImpl.java b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/VnfDescriptorGeneratorImpl.java
index 902911b817..053bdde73a 100644
--- a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/VnfDescriptorGeneratorImpl.java
+++ b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/VnfDescriptorGeneratorImpl.java
@@ -1,178 +1,198 @@
-
-/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2020 Nordix Foundation
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-package org.openecomp.sdc.be.plugins.etsi.nfv.nsd.generator;
+ /*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2020 Nordix Foundation
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+ package org.openecomp.sdc.be.plugins.etsi.nfv.nsd.generator;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.io.IOUtils;
-import org.onap.sdc.tosca.services.YamlUtil;
-import org.openecomp.core.utilities.file.FileContentHandler;
-import org.openecomp.core.utilities.file.FileUtils;
-import org.openecomp.sdc.be.model.ArtifactDefinition;
-import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.builder.NsdToscaMetadataBuilder;
-import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.exception.VnfDescriptorException;
-import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.model.VnfDescriptor;
-import org.openecomp.sdc.common.zip.exception.ZipException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-import org.yaml.snakeyaml.Yaml;
+ import java.io.ByteArrayInputStream;
+ import java.io.FileNotFoundException;
+ import java.io.IOException;
+ import java.io.InputStream;
+ import java.nio.charset.StandardCharsets;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Optional;
+ import java.util.stream.Collectors;
+ import org.apache.commons.collections.MapUtils;
+ import org.apache.commons.io.FilenameUtils;
+ import org.apache.commons.io.IOUtils;
+ import org.onap.sdc.tosca.services.YamlUtil;
+ import org.openecomp.core.utilities.file.FileContentHandler;
+ import org.openecomp.core.utilities.file.FileUtils;
+ import org.openecomp.sdc.be.csar.storage.ArtifactStorageManager;
+ import org.openecomp.sdc.be.csar.storage.StorageFactory;
+ import org.openecomp.sdc.be.model.ArtifactDefinition;
+ import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.builder.NsdToscaMetadataBuilder;
+ import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.exception.VnfDescriptorException;
+ import org.openecomp.sdc.be.plugins.etsi.nfv.nsd.model.VnfDescriptor;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.stereotype.Component;
+ import org.yaml.snakeyaml.Yaml;
-/**
- * Implementation of a VNF Descriptor Generator
- */
-@Component("vnfPackageGenerator")
-public class VnfDescriptorGeneratorImpl implements VnfDescriptorGenerator {
+ /**
+ * Implementation of a VNF Descriptor Generator
+ */
+ @Component("vnfPackageGenerator")
+ public class VnfDescriptorGeneratorImpl implements VnfDescriptorGenerator {
- private static final Logger LOGGER = LoggerFactory.getLogger(VnfDescriptorGeneratorImpl.class);
- private static final String SPACE_REGEX = "\\s+";
- private static final String CSAR = "csar";
- private static final String COLON = ":";
- private static final String EMPTY_STRING = "";
- private static final String SLASH = "/";
- private static final String DEFINITIONS_DIRECTORY = "Definitions";
- private static final String TOSCA_META_PATH = "TOSCA-Metadata/TOSCA.meta";
+ private static final Logger LOGGER = LoggerFactory.getLogger(VnfDescriptorGeneratorImpl.class);
+ private static final String SPACE_REGEX = "\\s+";
+ private static final String CSAR = "csar";
+ private static final String COLON = ":";
+ private static final String EMPTY_STRING = "";
+ private static final String SLASH = "/";
+ private static final String DEFINITIONS_DIRECTORY = "Definitions";
+ private static final String TOSCA_META_PATH = "TOSCA-Metadata/TOSCA.meta";
- private static boolean isACsarArtifact(final ArtifactDefinition definition) {
- return definition.getPayloadData() != null && definition.getArtifactName() != null && CSAR
- .equalsIgnoreCase(FilenameUtils.getExtension(definition.getArtifactName()));
- }
+ private static boolean isACsarArtifact(final ArtifactDefinition definition) {
+ return definition.getPayloadData() != null && definition.getArtifactName() != null && CSAR
+ .equalsIgnoreCase(FilenameUtils.getExtension(definition.getArtifactName()));
+ }
- public Optional<VnfDescriptor> generate(final String name, final ArtifactDefinition onboardedPackageArtifact) throws VnfDescriptorException {
- if (!isACsarArtifact(onboardedPackageArtifact)) {
- return Optional.empty();
- }
- final FileContentHandler fileContentHandler;
- try {
- fileContentHandler = FileUtils.getFileContentMapFromZip(onboardedPackageArtifact.getPayloadData());
- } catch (final ZipException e) {
- final String errorMsg = String.format("Could not unzip artifact '%s' content", onboardedPackageArtifact.getArtifactName());
- throw new VnfDescriptorException(errorMsg, e);
- }
- if (MapUtils.isEmpty(fileContentHandler.getFiles())) {
- return Optional.empty();
- }
- final String mainDefinitionFile;
- try {
- mainDefinitionFile = getMainFilePathFromMetaFile(fileContentHandler).orElse(null);
- } catch (final IOException e) {
- final String errorMsg = String.format("Could not read main definition file of artifact '%s'", onboardedPackageArtifact.getArtifactName());
- throw new VnfDescriptorException(errorMsg, e);
- }
- LOGGER.debug("found main file: {}", mainDefinitionFile);
- if (mainDefinitionFile == null) {
- return Optional.empty();
- }
- final VnfDescriptor vnfDescriptor = new VnfDescriptor();
- vnfDescriptor.setName(name);
- final String vnfdFileName = FilenameUtils.getName(mainDefinitionFile);
- vnfDescriptor.setVnfdFileName(vnfdFileName);
- vnfDescriptor.setDefinitionFiles(getFiles(fileContentHandler, mainDefinitionFile));
- vnfDescriptor.setNodeType(getNodeType(getFileContent(fileContentHandler, mainDefinitionFile)));
- return Optional.of(vnfDescriptor);
- }
+ public Optional<VnfDescriptor> generate(final String name, final ArtifactDefinition onboardedPackageArtifact) throws VnfDescriptorException {
+ if (!isACsarArtifact(onboardedPackageArtifact)) {
+ return Optional.empty();
+ }
+ final FileContentHandler fileContentHandler;
+ try {
+ final var artifactStorageManager = new StorageFactory().createArtifactStorageManager();
+ final byte[] payloadData = onboardedPackageArtifact.getPayloadData();
+ if (artifactStorageManager.isEnabled()) {
+ final var inputStream = artifactStorageManager.get(getFromPayload(payloadData, "bucket"),
+ getFromPayload(payloadData, "object") + ".reduced");
+ fileContentHandler = FileUtils.getFileContentMapFromZip(inputStream);
+ } else {
+ fileContentHandler = FileUtils.getFileContentMapFromZip(new ByteArrayInputStream(payloadData));
+ }
+ } catch (final IOException e) {
+ final String errorMsg = String.format("Could not unzip artifact '%s' content", onboardedPackageArtifact.getArtifactName());
+ throw new VnfDescriptorException(errorMsg, e);
+ }
+ if (MapUtils.isEmpty(fileContentHandler.getFiles())) {
+ return Optional.empty();
+ }
+ final String mainDefinitionFile;
+ try {
+ mainDefinitionFile = getMainFilePathFromMetaFile(fileContentHandler).orElse(null);
+ } catch (final IOException e) {
+ final String errorMsg = String.format("Could not read main definition file of artifact '%s'",
+ onboardedPackageArtifact.getArtifactName());
+ throw new VnfDescriptorException(errorMsg, e);
+ }
+ LOGGER.debug("found main file: {}", mainDefinitionFile);
+ if (mainDefinitionFile == null) {
+ return Optional.empty();
+ }
+ final var vnfDescriptor = new VnfDescriptor();
+ vnfDescriptor.setName(name);
+ final String vnfdFileName = FilenameUtils.getName(mainDefinitionFile);
+ vnfDescriptor.setVnfdFileName(vnfdFileName);
+ vnfDescriptor.setDefinitionFiles(getFiles(fileContentHandler, mainDefinitionFile));
+ vnfDescriptor.setNodeType(getNodeType(getFileContent(fileContentHandler, mainDefinitionFile)));
+ return Optional.of(vnfDescriptor);
+ }
- private Map<String, byte[]> getFiles(final FileContentHandler fileContentHandler, final String filePath) {
- final Map<String, byte[]> files = new HashMap<>();
- final byte[] fileContent = fileContentHandler.getFileContent(filePath);
- if (fileContent != null) {
- final String mainYmlFile = new String(fileContent);
- LOGGER.debug("file content: {}", mainYmlFile);
- files.put(appendDefinitionDirPath(filePath.substring(filePath.lastIndexOf(SLASH) + 1)), getVnfdWithoutTopologyTemplate(fileContent));
- final List<Object> imports = getImportFilesPath(mainYmlFile);
- LOGGER.info("found imports {}", imports);
- for (final Object importObject : imports) {
- if (importObject != null) {
- final String importFilename = importObject.toString();
- final String importFileFullPath = appendDefinitionDirPath(importFilename);
- final byte[] importFileContent = fileContentHandler.getFileContent(importFileFullPath);
- files.put(appendDefinitionDirPath(importFilename), importFileContent);
- }
- }
- }
- return files;
- }
+ private String getFromPayload(final byte[] payload, final String name) {
+ final String[] strings = new String(payload).split("\n");
+ for (final String str : strings) {
+ if (str.contains(name)) {
+ return str.split(": ")[1];
+ }
+ }
+ return "";
+ }
- private String getFileContent(final FileContentHandler fileContentHandler, final String filePath) {
- final byte[] fileContent = fileContentHandler.getFileContent(filePath);
- if (fileContent != null) {
- return new String(fileContent);
- }
- return null;
- }
+ private Map<String, byte[]> getFiles(final FileContentHandler fileContentHandler, final String filePath) {
+ final Map<String, byte[]> files = new HashMap<>();
+ final byte[] fileContent = fileContentHandler.getFileContent(filePath);
+ if (fileContent != null) {
+ final String mainYmlFile = new String(fileContent);
+ LOGGER.debug("file content: {}", mainYmlFile);
+ files.put(appendDefinitionDirPath(filePath.substring(filePath.lastIndexOf(SLASH) + 1)), getVnfdWithoutTopologyTemplate(fileContent));
+ final List<Object> imports = getImportFilesPath(mainYmlFile);
+ LOGGER.info("found imports {}", imports);
+ for (final Object importObject : imports) {
+ if (importObject != null) {
+ final String importFilename = importObject.toString();
+ final String importFileFullPath = appendDefinitionDirPath(importFilename);
+ final byte[] importFileContent = fileContentHandler.getFileContent(importFileFullPath);
+ files.put(appendDefinitionDirPath(importFilename), importFileContent);
+ }
+ }
+ }
+ return files;
+ }
- private Optional<String> getMainFilePathFromMetaFile(final FileContentHandler fileContentHandler) throws IOException {
- final Map<String, String> metaFileContent = getMetaFileContent(fileContentHandler);
- final String mainFile = metaFileContent.get(NsdToscaMetadataBuilder.ENTRY_DEFINITIONS);
- if (mainFile != null) {
- return Optional.of(mainFile.replaceAll(SPACE_REGEX, EMPTY_STRING));
- }
- LOGGER.error("{} entry not found in {}", NsdToscaMetadataBuilder.ENTRY_DEFINITIONS, TOSCA_META_PATH);
- return Optional.empty();
- }
+ private String getFileContent(final FileContentHandler fileContentHandler, final String filePath) {
+ final byte[] fileContent = fileContentHandler.getFileContent(filePath);
+ if (fileContent != null) {
+ return new String(fileContent);
+ }
+ return null;
+ }
- private Map<String, String> getMetaFileContent(final FileContentHandler fileContentHandler) throws IOException {
- final InputStream inputStream = fileContentHandler.getFileContentAsStream(TOSCA_META_PATH);
- if (inputStream == null) {
- throw new FileNotFoundException("Unable find " + TOSCA_META_PATH + " file");
- }
- final List<String> lines = IOUtils.readLines(inputStream, StandardCharsets.UTF_8);
- return lines.stream().map(str -> str.split(COLON)).collect(Collectors.toMap(str -> str[0], str -> str.length > 1 ? str[1] : EMPTY_STRING));
- }
+ private Optional<String> getMainFilePathFromMetaFile(final FileContentHandler fileContentHandler) throws IOException {
+ final Map<String, String> metaFileContent = getMetaFileContent(fileContentHandler);
+ final String mainFile = metaFileContent.get(NsdToscaMetadataBuilder.ENTRY_DEFINITIONS);
+ if (mainFile != null) {
+ return Optional.of(mainFile.replaceAll(SPACE_REGEX, EMPTY_STRING));
+ }
+ LOGGER.error("{} entry not found in {}", NsdToscaMetadataBuilder.ENTRY_DEFINITIONS, TOSCA_META_PATH);
+ return Optional.empty();
+ }
- private String appendDefinitionDirPath(final String filename) {
- return DEFINITIONS_DIRECTORY + SLASH + filename;
- }
+ private Map<String, String> getMetaFileContent(final FileContentHandler fileContentHandler) throws IOException {
+ final InputStream inputStream = fileContentHandler.getFileContentAsStream(TOSCA_META_PATH);
+ if (inputStream == null) {
+ throw new FileNotFoundException("Unable find " + TOSCA_META_PATH + " file");
+ }
+ final List<String> lines = IOUtils.readLines(inputStream, StandardCharsets.UTF_8);
+ return lines.stream().map(str -> str.split(COLON)).collect(Collectors.toMap(str -> str[0], str -> str.length > 1 ? str[1] : EMPTY_STRING));
+ }
- private List<Object> getImportFilesPath(final String mainYmlFile) {
- final Map<Object, Object> fileContentMap = new YamlUtil().yamlToObject(mainYmlFile, Map.class);
- final Object importsObject = fileContentMap.get("imports");
- if (importsObject instanceof List) {
- return (List<Object>) importsObject;
- }
- return Collections.emptyList();
- }
+ private String appendDefinitionDirPath(final String filename) {
+ return DEFINITIONS_DIRECTORY + SLASH + filename;
+ }
- private byte[] getVnfdWithoutTopologyTemplate(final byte[] vnfdFileContent) {
- final Yaml yaml = new Yaml();
- final Map<String, Object> toscaFileContent = (Map<String, Object>) yaml.load(new String(vnfdFileContent));
- toscaFileContent.remove("topology_template");
- return yaml.dumpAsMap(toscaFileContent).getBytes();
- }
+ private List<Object> getImportFilesPath(final String mainYmlFile) {
+ final Map<Object, Object> fileContentMap = new YamlUtil().yamlToObject(mainYmlFile, Map.class);
+ final Object importsObject = fileContentMap.get("imports");
+ if (importsObject instanceof List) {
+ return (List<Object>) importsObject;
+ }
+ return Collections.emptyList();
+ }
- private String getNodeType(final String mainYmlFile) {
- final Map<Object, Object> fileContentMap = new YamlUtil().yamlToObject(mainYmlFile, Map.class);
- final Object nodeTypesObject = fileContentMap.get("node_types");
- if (nodeTypesObject instanceof Map && ((Map<String, Object>) nodeTypesObject).size() == 1) {
- return ((Map<String, Object>) nodeTypesObject).keySet().iterator().next();
- }
- return null;
- }
-}
+ private byte[] getVnfdWithoutTopologyTemplate(final byte[] vnfdFileContent) {
+ final Yaml yaml = new Yaml();
+ final Map<String, Object> toscaFileContent = (Map<String, Object>) yaml.load(new String(vnfdFileContent));
+ toscaFileContent.remove("topology_template");
+ return yaml.dumpAsMap(toscaFileContent).getBytes();
+ }
+
+ private String getNodeType(final String mainYmlFile) {
+ final Map<Object, Object> fileContentMap = new YamlUtil().yamlToObject(mainYmlFile, Map.class);
+ final Object nodeTypesObject = fileContentMap.get("node_types");
+ if (nodeTypesObject instanceof Map && ((Map<String, Object>) nodeTypesObject).size() == 1) {
+ return ((Map<String, Object>) nodeTypesObject).keySet().iterator().next();
+ }
+ return null;
+ }
+ }
diff --git a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/config/NsDescriptorConfig.java b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/config/NsDescriptorConfig.java
index d17b03f7c4..96c9c813a6 100644
--- a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/config/NsDescriptorConfig.java
+++ b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/main/java/org/openecomp/sdc/be/plugins/etsi/nfv/nsd/generator/config/NsDescriptorConfig.java
@@ -1,4 +1,3 @@
-
/*
* ============LICENSE_START=======================================================
* Copyright (C) 2021 Nordix Foundation
@@ -19,17 +18,16 @@
*/
package org.openecomp.sdc.be.plugins.etsi.nfv.nsd.generator.config;
+import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* Configuration parameters for the Network Service Descriptor generation
*/
@Getter
+@AllArgsConstructor
public class NsDescriptorConfig {
- final EtsiVersion nsVersion;
+ private final EtsiVersion nsVersion;
- public NsDescriptorConfig(final EtsiVersion etsiVersion) {
- this.nsVersion = etsiVersion;
- }
}
diff --git a/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/test/resources/configuration.yaml b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/test/resources/configuration.yaml
new file mode 100644
index 0000000000..fa9fe016c4
--- /dev/null
+++ b/catalog-be-plugins/etsi-nfv-nsd-csar-plugin/src/test/resources/configuration.yaml
@@ -0,0 +1,2 @@
+externalCsarStore:
+ storageType: NONE # NONE, MINIO
diff --git a/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb b/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb
index 3b9d437ba6..aa9d3f275b 100644
--- a/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb
+++ b/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb
@@ -1261,3 +1261,14 @@ directives:
- selectable
- substitute
- substitutable
+
+externalCsarStore:
+ storageType: NONE # NONE, MINIO
+ endpoint:
+ host: 127.0.0.1
+ port: 9000
+ secure: false
+ credentials:
+ accessKey: "login"
+ secretKey: "password"
+ tempPath: "/home/onap/temp/"
diff --git a/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java b/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java
index 6ac82a547d..cabda08ce7 100644
--- a/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java
+++ b/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java
@@ -144,6 +144,7 @@ public class Configuration extends BasicConfiguration {
private EcompPortalConfig ecompPortal;
private List<ArtifactConfiguration> artifacts;
private Map<String, Map<String, List<String>>> componentAllowedInstanceTypes;
+ private ExternalCsarStore externalCsarStore;
@SuppressWarnings("unchecked")
private <K, V> Map<K, V> safeGetCapsInsensitiveMap(Map<K, V> map) {
@@ -165,8 +166,8 @@ public class Configuration extends BasicConfiguration {
public Map<String, Set<String>> getExcludedGroupTypesMapping() {
return safeGetCapsInsensitiveMap(excludedGroupTypesMapping);
}
-
- public List<Map<String, Map<String, String>>> getDefaultImports(){
+
+ public List<Map<String, Map<String, String>>> getDefaultImports() {
return Collections.unmodifiableList(defaultImports);
}
@@ -483,4 +484,35 @@ public class Configuration extends BasicConfiguration {
private String artifactType;
private List<PathsAndNamesDefinition> pathsAndNamesDefinitions;
}
+
+ @Getter
+ @Setter
+ @NoArgsConstructor
+ public static class ExternalCsarStore {
+
+ private String storageType;
+ private Endpoint endpoint;
+ private Credentials credentials;
+ private String tempPath;
+
+ @Getter
+ @Setter
+ @NoArgsConstructor
+ public static class Endpoint {
+
+ private String host;
+ private int port;
+ private boolean secure;
+ }
+
+ @Getter
+ @Setter
+ @NoArgsConstructor
+ public static class Credentials {
+
+ private String accessKey;
+ private String secretKey;
+ }
+ }
+
}
diff --git a/common-app-api/src/main/java/org/openecomp/sdc/be/csar/storage/ArtifactStorageManager.java b/common-app-api/src/main/java/org/openecomp/sdc/be/csar/storage/ArtifactStorageManager.java
index 0a4f355642..f7d611b717 100644
--- a/common-app-api/src/main/java/org/openecomp/sdc/be/csar/storage/ArtifactStorageManager.java
+++ b/common-app-api/src/main/java/org/openecomp/sdc/be/csar/storage/ArtifactStorageManager.java
@@ -47,6 +47,8 @@ public interface ArtifactStorageManager {
*/
ArtifactInfo upload(String vspId, String versionId, InputStream fileToUpload);
+ void put(String vspId, String name, InputStream fileToUpload);
+
/**
* Checks if the Artifact Storage is enabled.
*
@@ -63,5 +65,7 @@ public interface ArtifactStorageManager {
InputStream get(final ArtifactInfo artifactInfo);
+ InputStream get(final String vspId, final String versionId);
+
void delete(ArtifactInfo artifactInfo);
}
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java
index 0a48c2233c..850b4e1dda 100644
--- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java
@@ -108,6 +108,18 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage
LOGGER.info("Bucket '{}' already exists.", vspId);
}
+ put(vspId, name, fileToUpload);
+
+ } catch (final Exception e) {
+ throw new ArtifactStorageException("Failed to upload artifact", e);
+ }
+
+ return new MinIoArtifactInfo(vspId, name);
+ }
+
+ @Override
+ public void put(final String vspId, final String name, final InputStream fileToUpload) {
+ try {
minioClient.putObject(
PutObjectArgs.builder()
.bucket(vspId)
@@ -115,12 +127,9 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage
.stream(fileToUpload, fileToUpload.available(), -1)
.build()
);
-
} catch (final Exception e) {
throw new ArtifactStorageException("Failed to upload artifact", e);
}
-
- return new MinIoArtifactInfo(vspId, name);
}
@Override
@@ -132,9 +141,18 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage
public InputStream get(final ArtifactInfo artifactInfo) {
final MinIoArtifactInfo minioObject = (MinIoArtifactInfo) artifactInfo;
try {
+ return get(minioObject.getBucket(), minioObject.getObjectName());
+ } catch (final Exception e) {
+ throw new ArtifactStorageException("Failed to get Object", e);
+ }
+ }
+
+ @Override
+ public InputStream get(final String bucketID, final String objectID) {
+ try {
return minioClient.getObject(GetObjectArgs.builder()
- .bucket(minioObject.getBucket())
- .object(minioObject.getObjectName())
+ .bucket(bucketID)
+ .object(objectID)
.build());
} catch (final Exception e) {
throw new ArtifactStorageException("Failed to get Object", e);
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java
index 3fa22d41be..aec69d5bd7 100644
--- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java
@@ -37,6 +37,11 @@ public class NoneStorageManager implements ArtifactStorageManager {
}
@Override
+ public void put(final String vspId, final String versionId, final InputStream fileToUpload) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public ArtifactStorageConfig getStorageConfiguration() {
throw new UnsupportedOperationException();
}
@@ -47,6 +52,11 @@ public class NoneStorageManager implements ArtifactStorageManager {
}
@Override
+ public InputStream get(final String vspId, final String versionId) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void delete(final ArtifactInfo artifactInfo) {
throw new UnsupportedOperationException();
}
diff --git a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java
index 6fe7f9dd0a..e8ee6b3c4b 100644
--- a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java
+++ b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java
@@ -32,6 +32,7 @@ import static org.openecomp.sdc.common.errors.Messages.NO_FILE_WAS_UPLOADED_OR_F
import static org.openecomp.sdc.common.errors.Messages.PACKAGE_PROCESS_ERROR;
import static org.openecomp.sdc.common.errors.Messages.UNEXPECTED_PROBLEM_HAPPENED_WHILE_GETTING;
+import java.io.ByteArrayInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -183,8 +184,12 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate
final var vspDetails = vendorSoftwareProductManager.getVsp(vspId, version);
final Response response = processOnboardPackage(onboardPackageInfo, vspDetails, errorMessages);
final UploadFileResponseDto entity = (UploadFileResponseDto) response.getEntity();
- if (artifactStorageManager.isEnabled() && !entity.getErrors().isEmpty()) {
- artifactStorageManager.delete(artifactInfo);
+ if (artifactStorageManager.isEnabled()) {
+ if (!entity.getErrors().isEmpty()) {
+ artifactStorageManager.delete(artifactInfo);
+ } else {
+ artifactStorageManager.put(vspId, versionId + ".reduced", new ByteArrayInputStream(fileToUploadBytes));
+ }
}
return response;
}
@@ -255,7 +260,8 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate
FilesDataStructure fileDataStructure = copyFilesDataStructureDtoToFilesDataStructure(fileDataStructureDto);
ValidationResponse response = candidateManager.updateFilesDataStructure(vspId, new Version(versionId), fileDataStructure);
if (!response.isValid()) {
- return Response.status(EXPECTATION_FAILED).entity(new MapValidationResponseToDto().applyMapping(response, ValidationResponseDto.class))
+ return Response.status(EXPECTATION_FAILED)
+ .entity(new MapValidationResponseToDto().applyMapping(response, ValidationResponseDto.class))
.build();
}
return Response.ok(fileDataStructureDto).build();
diff --git a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java
index 60ff1c172a..d6df24e4c2 100644
--- a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java
+++ b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java
@@ -30,12 +30,12 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.onap.sdc.tosca.services.YamlUtil;
import org.openecomp.core.utilities.json.JsonUtil;
-import org.openecomp.sdc.common.zip.ZipUtils;
-import org.openecomp.sdc.common.zip.exception.ZipException;
/**
* The type File utils.
@@ -206,20 +206,24 @@ public class FileUtils {
/**
* Gets file content map from zip.
*
- * @param zipData the zip data
+ * @param inputStream the zip data
* @return the file content map from zip
- * @throws ZipException when an error occurs while extracting zip files
+ * @throws IOException when an error occurs while extracting zip files
*/
- public static FileContentHandler getFileContentMapFromZip(byte[] zipData) throws ZipException {
- final Map<String, byte[]> zipFileAndByteMap = ZipUtils.readZip(zipData, true);
- final FileContentHandler fileContentHandler = new FileContentHandler();
- zipFileAndByteMap.forEach((path, bytes) -> {
- if (bytes == null) {
- fileContentHandler.addFolder(path);
+ public static FileContentHandler getFileContentMapFromZip(final InputStream inputStream) throws IOException {
+
+ final var zipInputStream = new ZipInputStream(inputStream);
+ ZipEntry zipEntry;
+ final var fileContentHandler = new FileContentHandler();
+ while ((zipEntry = zipInputStream.getNextEntry()) != null) {
+ final var entryName = zipEntry.getName();
+ if (zipEntry.isDirectory()) {
+ fileContentHandler.addFolder(entryName);
} else {
- fileContentHandler.addFile(path, bytes);
+ fileContentHandler.addFile(entryName, zipInputStream.readAllBytes());
}
- });
+
+ }
return fileContentHandler;
}
diff --git a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java
index facfe57622..05ec5a478d 100644
--- a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java
+++ b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java
@@ -35,7 +35,6 @@ import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.commons.io.IOUtils;
import org.junit.Assert;
import org.junit.Test;
import org.openecomp.sdc.common.zip.exception.ZipException;
@@ -47,7 +46,7 @@ import org.openecomp.sdc.common.zip.exception.ZipException;
public class FileUtilsTest {
private static final String TEST_RESOURCE = FileUtilsTest.class.getPackage().getName()
- .replace('.', '/') + "/test-resource.txt";
+ .replace('.', '/') + "/test-resource.txt";
private static final Function<InputStream, Integer> TEST_FUNCTION = (s) -> {
@@ -82,13 +81,11 @@ public class FileUtilsTest {
public void testWriteFilesFromFileContentHandler() throws IOException, ZipException {
final Path tempDirectory = Files.createTempDirectory("CSAR_" + System.currentTimeMillis());
try {
- byte[] uploadedFileData =
- IOUtils.toByteArray(FileUtilsTest.class.getResource("resource-Spgw-csar-ZTE.csar"));
- final FileContentHandler contentMap = FileUtils.getFileContentMapFromZip(uploadedFileData);
+ final var contentMap = FileUtils.getFileContentMapFromZip(FileUtilsTest.class.getResourceAsStream("resource-Spgw-csar-ZTE.csar"));
final Map<String, String> filePaths = FileUtils.writeFilesFromFileContentHandler(contentMap, tempDirectory);
assertThat("The file map should not be empty", filePaths, is(not(anEmptyMap())));
- assertThat("The file map should have size 20", filePaths, is(aMapWithSize(20)));
+ assertThat("The file map should have size 20", filePaths, is(aMapWithSize(18)));
for (final Map.Entry<String, String> fileEntry : filePaths.entrySet()) {
final File f = new File(fileEntry.getValue());
assertThat(String.format("The file '%s' is expected to", f.getAbsolutePath()), f.exists(), is(true));
@@ -151,7 +148,7 @@ public class FileUtilsTest {
@Test
public void testConvertToBytes() {
byte[] bytesArray = FileUtils.convertToBytes(Stream.of("Json", "Util", "Test").collect(Collectors.toList()),
- FileUtils.FileExtension.YAML);
+ FileUtils.FileExtension.YAML);
Assert.assertNotNull(bytesArray);
}
@@ -159,7 +156,7 @@ public class FileUtilsTest {
@Test
public void testConvertToBytesNotYaml() {
byte[] bytesArray = FileUtils.convertToBytes(Stream.of("Json", "Util", "Test").collect(Collectors.toList()),
- FileUtils.FileExtension.JSON);
+ FileUtils.FileExtension.JSON);
Assert.assertNotNull(bytesArray);
}
@@ -172,7 +169,7 @@ public class FileUtilsTest {
@Test
public void testConvertToInputStream() {
InputStream inputStream = FileUtils.convertToInputStream(Stream.of("Json", "Util", "Test")
- .collect(Collectors.toList()), FileUtils.FileExtension.YAML);
+ .collect(Collectors.toList()), FileUtils.FileExtension.YAML);
Assert.assertNotNull(inputStream);
}
@@ -183,13 +180,13 @@ public class FileUtilsTest {
}
@Test
- public void testLoadFileToInputStream() throws IOException{
+ public void testLoadFileToInputStream() throws IOException {
int i;
StringBuilder builder = new StringBuilder(20);
InputStream inputStream = FileUtils.loadFileToInputStream(
- "org/openecomp/core/utilities/file/testFileUtils.txt");
- while((i = inputStream.read())!=-1) {
- builder.append((char)i);
+ "org/openecomp/core/utilities/file/testFileUtils.txt");
+ while ((i = inputStream.read()) != -1) {
+ builder.append((char) i);
}
Assert.assertNotNull(inputStream);
diff --git a/openecomp-be/lib/openecomp-sdc-enrichment-lib/openecomp-sdc-enrichment-impl/src/main/java/org/openecomp/sdc/enrichment/impl/external/artifact/MonitoringMibEnricher.java b/openecomp-be/lib/openecomp-sdc-enrichment-lib/openecomp-sdc-enrichment-impl/src/main/java/org/openecomp/sdc/enrichment/impl/external/artifact/MonitoringMibEnricher.java
index 06a5b3a2c2..02d0328d3b 100644
--- a/openecomp-be/lib/openecomp-sdc-enrichment-lib/openecomp-sdc-enrichment-impl/src/main/java/org/openecomp/sdc/enrichment/impl/external/artifact/MonitoringMibEnricher.java
+++ b/openecomp-be/lib/openecomp-sdc-enrichment-lib/openecomp-sdc-enrichment-impl/src/main/java/org/openecomp/sdc/enrichment/impl/external/artifact/MonitoringMibEnricher.java
@@ -19,6 +19,7 @@ import static org.openecomp.sdc.tosca.services.ToscaConstants.SERVICE_TEMPLATE_F
import static org.openecomp.sdc.tosca.services.ToscaConstants.SUBSTITUTE_SERVICE_TEMPLATE_PROPERTY_NAME;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -41,7 +42,6 @@ import org.openecomp.core.model.types.ServiceArtifact;
import org.openecomp.core.utilities.file.FileContentHandler;
import org.openecomp.core.utilities.file.FileUtils;
import org.openecomp.sdc.common.errors.Messages;
-import org.openecomp.sdc.common.zip.exception.ZipException;
import org.openecomp.sdc.datatypes.error.ErrorLevel;
import org.openecomp.sdc.datatypes.error.ErrorMessage;
import org.openecomp.sdc.enrichment.EnrichmentInfo;
@@ -208,8 +208,8 @@ public class MonitoringMibEnricher implements ExternalArtifactEnricherInterface
}
FileContentHandler mibs;
try {
- mibs = FileUtils.getFileContentMapFromZip(FileUtils.toByteArray(monitoringArtifactInfo.getContent()));
- } catch (ZipException ex) {
+ mibs = FileUtils.getFileContentMapFromZip(monitoringArtifactInfo.getContent());
+ } catch (final IOException ex) {
log.error("Failed to get file content map from zip ", ex);
ErrorMessage.ErrorMessageUtil.addMessage(mibServiceArtifact.getName() + "." + type.name(), errors)
.add(new ErrorMessage(ErrorLevel.ERROR, Messages.INVALID_ZIP_FILE.getErrorMessage()));