Support handling of 'Large CSARs'

If artifact storage is enabled, stores original onboarded package,
leaving a reference in the VSP, instead of the original onboarded
package itself. Strips files from configured folders in order to reduce
the package size and onboard it. To retrieve the package, one needs to
read the reference and go to the artifact storage to retrieve.
If disabled, it just goes through the current onboarding process.

Change-Id: I3dce0ab8422ea736c8a1ffaeb1136cf8b12a2af4
Signed-off-by: Vasyl Razinkov <vasyl.razinkov@est.tech>
Signed-off-by: André Schmid <andre.schmid@est.tech>
Issue-ID: SDC-3635

7 files changed, 432 insertions, 0 deletions

diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarPackageReducerConfiguration.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarPackageReducerConfiguration.java
new file mode 100644
index 0000000000..a14222ab17
--- /dev/null
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarPackageReducerConfiguration.java
@@ -0,0 +1,33 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.sdc.be.csar.storage;
+
+import java.nio.file.Path;
+import java.util.Set;
+import lombok.Data;
+
+@Data
+public class CsarPackageReducerConfiguration implements PackageSizeReducerConfig {
+
+    private final Set<Path> foldersToStrip;
+    private final long sizeLimit;
+
+}
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java
new file mode 100644
index 0000000000..cf35c8c4d7
--- /dev/null
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java
@@ -0,0 +1,109 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.sdc.be.csar.storage;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.UUID;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+import java.util.zip.ZipOutputStream;
+import org.openecomp.sdc.be.csar.storage.exception.CsarSizeReducerException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CsarSizeReducer implements PackageSizeReducer {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(CsarSizeReducer.class);
+
+    private final CsarPackageReducerConfiguration configuration;
+
+    public CsarSizeReducer(final CsarPackageReducerConfiguration configuration) {
+        this.configuration = configuration;
+    }
+
+    @Override
+    public byte[] reduce(final Path csarPackagePath) {
+        final var reducedCsarPath = Path.of(csarPackagePath + "." + UUID.randomUUID());
+
+        try (final var zf = new ZipFile(csarPackagePath.toString());
+            final var zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(reducedCsarPath)))) {
+
+            zf.entries().asIterator().forEachRemaining(entry -> {
+                final var entryName = entry.getName();
+                try {
+                    if (!entry.isDirectory()) {
+                        zos.putNextEntry(new ZipEntry(entryName));
+                        if (isCandidateToRemove(entry)) {
+                            // replace with EMPTY string to avoid package description inconsistency/validation errors
+                            zos.write("".getBytes());
+                        } else {
+                            zos.write(zf.getInputStream(entry).readAllBytes());
+                        }
+                    }
+                    zos.closeEntry();
+                } catch (final IOException ei) {
+                    final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
+                    throw new CsarSizeReducerException(errorMsg, ei);
+                }
+            });
+
+        } catch (final IOException ex1) {
+            rollback(reducedCsarPath);
+            final var errorMsg = String.format("An unexpected problem happened while reading the CSAR '%s'", csarPackagePath);
+            throw new CsarSizeReducerException(errorMsg, ex1);
+        }
+        final byte[] reducedCsarBytes;
+        try {
+            reducedCsarBytes = Files.readAllBytes(reducedCsarPath);
+        } catch (final IOException e) {
+            final var errorMsg = String.format("Could not read bytes of file '%s'", csarPackagePath);
+            throw new CsarSizeReducerException(errorMsg, e);
+        }
+        try {
+            Files.delete(reducedCsarPath);
+        } catch (final IOException e) {
+            final var errorMsg = String.format("Could not delete temporary file '%s'", reducedCsarPath);
+            throw new CsarSizeReducerException(errorMsg, e);
+        }
+
+        return reducedCsarBytes;
+    }
+
+    private void rollback(final Path reducedCsarPath) {
+        if (Files.exists(reducedCsarPath)) {
+            try {
+                Files.delete(reducedCsarPath);
+            } catch (final Exception ex2) {
+                LOGGER.warn("Could not delete temporary file '{}'", reducedCsarPath, ex2);
+            }
+        }
+    }
+
+    private boolean isCandidateToRemove(final ZipEntry zipEntry) {
+        final String zipEntryName = zipEntry.getName();
+        return configuration.getFoldersToStrip().stream().anyMatch(Path.of(zipEntryName)::startsWith)
+            || zipEntry.getSize() > configuration.getSizeLimit();
+    }
+
+}
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java
new file mode 100644
index 0000000000..0472661fd9
--- /dev/null
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java
@@ -0,0 +1,33 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.sdc.be.csar.storage;
+
+import java.nio.file.Path;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@AllArgsConstructor
+public class PersistentStorageArtifactInfo implements ArtifactInfo {
+
+    @Getter
+    private final Path path;
+
+}
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java
new file mode 100644
index 0000000000..d3cd6fb302
--- /dev/null
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java
@@ -0,0 +1,32 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.sdc.be.csar.storage;
+
+import java.nio.file.Path;
+import lombok.Data;
+
+@Data
+public class PersistentVolumeArtifactStorageConfig implements ArtifactStorageConfig {
+
+    private final boolean isEnabled;
+    private final Path storagePath;
+
+}
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java
new file mode 100644
index 0000000000..10629b3edb
--- /dev/null
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java
@@ -0,0 +1,161 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.sdc.be.csar.storage;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Optional;
+import java.util.UUID;
+import org.openecomp.sdc.be.csar.storage.exception.PersistentVolumeArtifactStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PersistentVolumeArtifactStorageManager implements ArtifactStorageManager {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(PersistentVolumeArtifactStorageManager.class);
+
+    private final PersistentVolumeArtifactStorageConfig storageConfiguration;
+
+    public PersistentVolumeArtifactStorageManager(final ArtifactStorageConfig storageConfiguration) {
+        this.storageConfiguration = (PersistentVolumeArtifactStorageConfig) storageConfiguration;
+    }
+
+    @Override
+    public ArtifactInfo persist(final String vspId, final String versionId, final ArtifactInfo uploadedArtifactInfo) {
+        final var temporaryPath = uploadedArtifactInfo.getPath();
+        if (!Files.exists(temporaryPath)) {
+            throw new PersistentVolumeArtifactStorageException(String.format("Given artifact does not exist '%s'", uploadedArtifactInfo.getPath()));
+        }
+
+        final var filePath = buildFilePath(vspId, versionId);
+        final var backupPath = backupPreviousVersion(filePath).orElse(null);
+        try {
+            moveFile(temporaryPath, filePath);
+        } catch (final Exception e) {
+            rollback(backupPath, filePath);
+            final var errorMsg = String.format("Could not persist artifact for VSP '%s', version '%s'", vspId, versionId);
+            throw new PersistentVolumeArtifactStorageException(errorMsg, e);
+        }
+
+        removePreviousVersion(backupPath);
+
+        return new PersistentStorageArtifactInfo(filePath);
+    }
+
+    @Override
+    public ArtifactInfo upload(final String vspId, final String versionId, final InputStream artifactInputStream) {
+        final var destinationFolder = buildDestinationFolder(vspId, versionId);
+        try {
+            Files.createDirectories(destinationFolder);
+        } catch (final IOException e) {
+            throw new PersistentVolumeArtifactStorageException(String.format("Could not create directory '%s'", destinationFolder), e);
+        }
+
+        final var filePath = createTempFilePath(destinationFolder);
+        try {
+            persist(artifactInputStream, filePath);
+        } catch (final IOException e) {
+            throw new PersistentVolumeArtifactStorageException(String.format("Could not persist artifact '%s'", filePath), e);
+        }
+
+        return new PersistentStorageArtifactInfo(filePath);
+    }
+
+    private Path buildFilePath(final String vspId, final String versionId) {
+        return buildDestinationFolder(vspId, versionId).resolve(versionId);
+    }
+
+    @Override
+    public boolean isEnabled() {
+        return storageConfiguration != null && storageConfiguration.isEnabled();
+    }
+
+    private Optional<Path> backupPreviousVersion(final Path filePath) {
+        if (!Files.exists(filePath)) {
+            return Optional.empty();
+        }
+
+        final var backupPath = Path.of(filePath + UUID.randomUUID().toString());
+        moveFile(filePath, backupPath);
+        return Optional.ofNullable(backupPath);
+    }
+
+    private void rollback(final Path backupPath, final Path filePath) {
+        try {
+            moveFile(backupPath, filePath);
+        } catch (final Exception ex) {
+            LOGGER.warn("Could not rollback the backup file '{}' to the original '{}'", backupPath, filePath, ex);
+        }
+    }
+
+    private void removePreviousVersion(final Path filePath) {
+        if (filePath == null || !Files.exists(filePath)) {
+            return;
+        }
+
+        try {
+            Files.delete(filePath);
+        } catch (final IOException e) {
+            throw new PersistentVolumeArtifactStorageException(String.format("Could not delete previous version '%s'", filePath), e);
+        }
+    }
+
+    private Path createTempFilePath(final Path destinationFolder) {
+        final var retries = 10;
+        return createTempFilePath(destinationFolder, retries).orElseThrow(() -> {
+            throw new PersistentVolumeArtifactStorageException(String.format("Could not generate upload file path after '%s' retries", retries));
+        });
+    }
+
+    private Optional<Path> createTempFilePath(final Path destinationFolder, int retries) {
+        for (var i = 0; i < retries; i++) {
+            final var filePath = destinationFolder.resolve(UUID.randomUUID().toString());
+            if (Files.notExists(filePath)) {
+                return Optional.of(filePath);
+            }
+        }
+        return Optional.empty();
+    }
+
+    private Path buildDestinationFolder(final String vspId, final String versionId) {
+        return storageConfiguration.getStoragePath().resolve(vspId).resolve(versionId);
+    }
+
+    private void persist(final InputStream artifactInputStream, final Path filePath) throws IOException {
+        try (final var inputStream = artifactInputStream;
+            final var fileOutputStream = new FileOutputStream(filePath.toFile());) {
+            inputStream.transferTo(fileOutputStream);
+        }
+    }
+
+    private void moveFile(final Path from, final Path to) {
+        try {
+            Files.move(from, to, StandardCopyOption.REPLACE_EXISTING);
+        } catch (final IOException e) {
+            throw new PersistentVolumeArtifactStorageException(String.format("Could not move file '%s' to '%s'", from, to), e);
+        }
+    }
+
+}
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/CsarSizeReducerException.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/CsarSizeReducerException.java
new file mode 100644
index 0000000000..f57666ac70
--- /dev/null
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/CsarSizeReducerException.java
@@ -0,0 +1,30 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.sdc.be.csar.storage.exception;
+
+import org.openecomp.sdc.be.exception.BusinessException;
+
+public class CsarSizeReducerException extends BusinessException {
+
+    public CsarSizeReducerException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java
new file mode 100644
index 0000000000..28fff65bb6
--- /dev/null
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java
@@ -0,0 +1,34 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.sdc.be.csar.storage.exception;
+
+import org.openecomp.sdc.be.exception.BusinessException;
+
+public class PersistentVolumeArtifactStorageException extends BusinessException {
+
+    public PersistentVolumeArtifactStorageException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+
+    public PersistentVolumeArtifactStorageException(final String message) {
+        super(message);
+    }
+}