Support handling of 'Large CSARs'

If artifact storage is enabled, stores original onboarded package,
leaving a reference in the VSP, instead of the original onboarded
package itself. Strips files from configured folders in order to reduce
the package size and onboard it. To retrieve the package, one needs to
read the reference and go to the artifact storage to retrieve.
If disabled, it just goes through the current onboarding process.

Change-Id: I3dce0ab8422ea736c8a1ffaeb1136cf8b12a2af4
Signed-off-by: Vasyl Razinkov <vasyl.razinkov@est.tech>
Signed-off-by: André Schmid <andre.schmid@est.tech>
Issue-ID: SDC-3635

4 files changed, 94 insertions, 20 deletions

diff --git a/openecomp-be/lib/openecomp-common-lib/src/main/java/org/openecomp/sdc/common/errors/Messages.java b/openecomp-be/lib/openecomp-common-lib/src/main/java/org/openecomp/sdc/common/errors/Messages.java
index 91ed6ae2f8..28ce5d2a62 100644
--- a/openecomp-be/lib/openecomp-common-lib/src/main/java/org/openecomp/sdc/common/errors/Messages.java
+++ b/openecomp-be/lib/openecomp-common-lib/src/main/java/org/openecomp/sdc/common/errors/Messages.java
@@ -201,7 +201,12 @@ public enum Messages {
     /* Notifications */
     FAILED_TO_MARK_NOTIFICATION_AS_READ("Failed to mark notifications as read"),
     FAILED_TO_UPDATE_LAST_SEEN_NOTIFICATION("Failed to update last seen notification for user %s"),
-    FAILED_TO_VERIFY_SIGNATURE("Could not verify signature of signed package.");
+    FAILED_TO_VERIFY_SIGNATURE("Could not verify signature of signed package."),
+    EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING_FULL_PATH("externalCsarStore configuration failure, missing 'fullPath'"),
+    ERROR_HAS_OCCURRED_WHILE_PERSISTING_THE_ARTIFACT("An error has occurred while persisting the artifact: %s"),
+    ERROR_HAS_OCCURRED_WHILE_REDUCING_THE_ARTIFACT_SIZE("An error has occurred while reducing the artifact's size: %s"),
+    UNEXPECTED_PROBLEM_HAPPENED_WHILE_GETTING("An unexpected problem happened while getting '%s'"),
+    PERSISTENCE_STORE_IS_DOWN_OR_NOT_AVAILABLE("Persistence store is down or not available. Error: '%s'");
     // @formatter:on
 
     private String errorMessage;
diff --git a/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-api/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/type/OrchestrationTemplateCandidateData.java b/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-api/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/type/OrchestrationTemplateCandidateData.java
index fdd143d1d2..b079474d1e 100644
--- a/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-api/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/type/OrchestrationTemplateCandidateData.java
+++ b/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-api/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/type/OrchestrationTemplateCandidateData.java
@@ -20,6 +20,7 @@ import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.openecomp.core.utilities.json.JsonUtil;
+import org.openecomp.sdc.be.csar.storage.ArtifactInfo;
 import org.openecomp.sdc.heat.datatypes.structure.ValidationStructureList;
 
 @Getter
@@ -35,17 +36,19 @@ public class OrchestrationTemplateCandidateData {
     private ByteBuffer originalFileContentData;
     private String originalFileName;
     private String originalFileSuffix;
+    private ArtifactInfo artifactInfo;
 
-    public OrchestrationTemplateCandidateData(final ByteBuffer contentData, final String dataStructureJson, final String fileSuffix,
-                                              final String fileName, final String originalFileName, final String originalFileSuffix,
-                                              final ByteBuffer originalFileContentData) {
+    public OrchestrationTemplateCandidateData(final ByteBuffer contentData, final String fileSuffix, final String fileName,
+                                              final String originalFileName, final String originalFileSuffix,
+                                              final ByteBuffer originalFileContentData, final ArtifactInfo artifactInfo) {
         this.contentData = contentData;
-        this.filesDataStructure = dataStructureJson;
+        this.filesDataStructure = "";
         this.fileSuffix = fileSuffix;
         this.fileName = fileName;
         this.originalFileName = originalFileName;
         this.originalFileSuffix = originalFileSuffix;
         this.originalFileContentData = originalFileContentData;
+        this.artifactInfo = artifactInfo;
     }
 
     public ValidationStructureList getValidationDataStructure() {
diff --git a/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/impl/zusammen/OrchestrationTemplateCandidateDaoZusammenException.java b/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/impl/zusammen/OrchestrationTemplateCandidateDaoZusammenException.java
new file mode 100644
index 0000000000..5a0a53a182
--- /dev/null
+++ b/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/impl/zusammen/OrchestrationTemplateCandidateDaoZusammenException.java
@@ -0,0 +1,28 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.sdc.vendorsoftwareproduct.dao.impl.zusammen;
+
+public class OrchestrationTemplateCandidateDaoZusammenException extends RuntimeException {
+
+    public OrchestrationTemplateCandidateDaoZusammenException(final String message) {
+        super(message);
+    }
+}
diff --git a/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/impl/zusammen/OrchestrationTemplateCandidateDaoZusammenImpl.java b/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/impl/zusammen/OrchestrationTemplateCandidateDaoZusammenImpl.java
index fb26714375..e5c59968fa 100644
--- a/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/impl/zusammen/OrchestrationTemplateCandidateDaoZusammenImpl.java
+++ b/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/dao/impl/zusammen/OrchestrationTemplateCandidateDaoZusammenImpl.java
@@ -12,6 +12,9 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ * ============LICENSE_END=========================================================
+ * Modifications copyright (c) 2021 Nordix Foundation
+ * ================================================================================
  */
 package org.openecomp.sdc.vendorsoftwareproduct.dao.impl.zusammen;
 
@@ -27,10 +30,21 @@ import com.amdocs.zusammen.datatypes.item.ElementContext;
 import com.amdocs.zusammen.utils.fileutils.FileUtils;
 import java.io.ByteArrayInputStream;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
 import java.util.Optional;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
 import org.openecomp.core.utilities.json.JsonUtil;
 import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum;
 import org.openecomp.core.zusammen.api.ZusammenAdaptor;
+import org.openecomp.sdc.be.csar.storage.ArtifactInfo;
+import org.openecomp.sdc.be.csar.storage.ArtifactStorageConfig;
+import org.openecomp.sdc.be.csar.storage.ArtifactStorageManager;
+import org.openecomp.sdc.be.csar.storage.PersistentVolumeArtifactStorageConfig;
+import org.openecomp.sdc.be.csar.storage.PersistentVolumeArtifactStorageManager;
+import org.openecomp.sdc.common.CommonConfigurationManager;
+import org.openecomp.sdc.common.errors.Messages;
 import org.openecomp.sdc.datatypes.model.ElementType;
 import org.openecomp.sdc.heat.datatypes.structure.ValidationStructureList;
 import org.openecomp.sdc.logging.api.Logger;
@@ -44,10 +58,13 @@ public class OrchestrationTemplateCandidateDaoZusammenImpl implements Orchestrat
 
     private static final Logger logger = LoggerFactory.getLogger(OrchestrationTemplateCandidateDaoZusammenImpl.class);
     private static final String EMPTY_DATA = "{}";
+    private static final String EXTERNAL_CSAR_STORE = "externalCsarStore";
     private final ZusammenAdaptor zusammenAdaptor;
+    private final ArtifactStorageManager artifactStorageManager;
 
-    public OrchestrationTemplateCandidateDaoZusammenImpl(ZusammenAdaptor zusammenAdaptor) {
+    public OrchestrationTemplateCandidateDaoZusammenImpl(final ZusammenAdaptor zusammenAdaptor) {
         this.zusammenAdaptor = zusammenAdaptor;
+        this.artifactStorageManager = new PersistentVolumeArtifactStorageManager(readArtifactStorageConfiguration());
     }
 
     @Override
@@ -139,11 +156,24 @@ public class OrchestrationTemplateCandidateDaoZusammenImpl implements Orchestrat
         candidateContentElement.setData(new ByteArrayInputStream(candidateData.getContentData().array()));
         candidateContentElement.getInfo().addProperty(InfoPropertyName.FILE_SUFFIX.getVal(), candidateData.getFileSuffix());
         candidateContentElement.getInfo().addProperty(InfoPropertyName.FILE_NAME.getVal(), candidateData.getFileName());
+        final String versionId = version.getId();
         if (OnboardingTypesEnum.CSAR.toString().equalsIgnoreCase(candidateData.getFileSuffix())) {
             final ZusammenElement originalPackageElement = buildStructuralElement(ElementType.ORIGINAL_ONBOARDED_PACKAGE, Action.UPDATE);
-            originalPackageElement.getInfo().addProperty(InfoPropertyName.ORIGINAL_FILE_NAME.getVal(), candidateData.getOriginalFileName());
-            originalPackageElement.getInfo().addProperty(InfoPropertyName.ORIGINAL_FILE_SUFFIX.getVal(), candidateData.getOriginalFileSuffix());
-            originalPackageElement.setData(new ByteArrayInputStream(candidateData.getOriginalFileContentData().array()));
+            final String originalFileName = candidateData.getOriginalFileName();
+            final String originalFileSuffix = candidateData.getOriginalFileSuffix();
+            originalPackageElement.getInfo().addProperty(InfoPropertyName.ORIGINAL_FILE_NAME.getVal(), originalFileName);
+            originalPackageElement.getInfo().addProperty(InfoPropertyName.ORIGINAL_FILE_SUFFIX.getVal(), originalFileSuffix);
+            originalPackageElement.getInfo().addProperty("storeCsarsExternally", artifactStorageManager.isEnabled());
+            if (artifactStorageManager.isEnabled()) {
+                final ArtifactInfo candidateArtifactInfo = candidateData.getArtifactInfo();
+                if (candidateArtifactInfo == null) {
+                    throw new OrchestrationTemplateCandidateDaoZusammenException("No artifact info provided");
+                }
+                final ArtifactInfo artifactInfo = artifactStorageManager.persist(vspId, versionId, candidateArtifactInfo);
+                originalPackageElement.setData(new ByteArrayInputStream(artifactInfo.getPath().toString().getBytes(StandardCharsets.UTF_8)));
+            } else {
+                originalPackageElement.setData(new ByteArrayInputStream(candidateData.getOriginalFileContentData().array()));
+            }
             candidateElement.addSubElement(originalPackageElement);
         }
         final ZusammenElement validationData = buildStructuralElement(ElementType.OrchestrationTemplateCandidateValidationData, Action.UPDATE);
@@ -152,8 +182,8 @@ public class OrchestrationTemplateCandidateDaoZusammenImpl implements Orchestrat
         }
         candidateElement.addSubElement(validationData);
         candidateElement.addSubElement(candidateContentElement);
-        SessionContext context = createSessionContext();
-        ElementContext elementContext = new ElementContext(vspId, version.getId());
+        final var context = createSessionContext();
+        final var elementContext = new ElementContext(vspId, versionId);
         zusammenAdaptor.saveElement(context, elementContext, candidateElement, "Update Orchestration Template Candidate");
         logger.info("Finished uploading candidate data entity for vsp id {}", vspId);
     }
@@ -197,16 +227,24 @@ public class OrchestrationTemplateCandidateDaoZusammenImpl implements Orchestrat
         return Optional.empty();
     }
 
-    public enum InfoPropertyName {
-        FILE_SUFFIX("fileSuffix"), FILE_NAME("fileName"), ORIGINAL_FILE_NAME("originalFilename"), ORIGINAL_FILE_SUFFIX("originalFileSuffix");
-        private final String val;
-
-        InfoPropertyName(String val) {
-            this.val = val;
+    private ArtifactStorageConfig readArtifactStorageConfiguration() {
+        final var commonConfigurationManager = CommonConfigurationManager.getInstance();
+        final boolean isEnabled = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "storeCsarsExternally", false);
+        logger.info("ArtifactConfig.isEnabled: '{}'", isEnabled);
+        final String storagePathString = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "fullPath", null);
+        logger.info("ArtifactConfig.storagePath: '{}'", storagePathString);
+        if (isEnabled && storagePathString == null) {
+            throw new OrchestrationTemplateCandidateDaoZusammenException(
+                Messages.EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING_FULL_PATH.getErrorMessage());
         }
+        final var storagePath = storagePathString == null ? null : Path.of(storagePathString);
+        return new PersistentVolumeArtifactStorageConfig(isEnabled, storagePath);
+    }
 
-        private String getVal() {
-            return val;
-        }
+    @Getter
+    @AllArgsConstructor
+    private enum InfoPropertyName {
+        FILE_SUFFIX("fileSuffix"), FILE_NAME("fileName"), ORIGINAL_FILE_NAME("originalFilename"), ORIGINAL_FILE_SUFFIX("originalFileSuffix");
+        private final String val;
     }
 }