From 707fb6d83819058d5736b2dc38bea3c2d9e07a2d Mon Sep 17 00:00:00 2001 From: vasraz Date: Fri, 8 Oct 2021 14:48:08 +0100 Subject: Large csar handling - object store Change-Id: I4e88bd7bfcc1fdbc93d67da2682f2e873ba243c6 Signed-off-by: Vasyl Razinkov Issue-ID: SDC-3754 --- common-be/pom.xml | 478 +++++++++++---------- .../sdc/be/csar/storage/CsarSizeReducer.java | 233 ---------- .../sdc/be/csar/storage/MinIoArtifactInfo.java | 38 ++ .../storage/MinIoStorageArtifactStorageConfig.java | 52 +++ .../MinIoStorageArtifactStorageManager.java | 247 +++++++++++ .../csar/storage/MinIoStorageCsarSizeReducer.java | 233 ++++++++++ .../sdc/be/csar/storage/NoneStorageManager.java | 54 +++ .../storage/PersistentStorageArtifactInfo.java | 33 -- .../PersistentVolumeArtifactStorageConfig.java | 32 -- .../PersistentVolumeArtifactStorageManager.java | 161 ------- .../sdc/be/csar/storage/StorageFactory.java | 91 ++++ .../exception/ArtifactStorageException.java | 34 ++ .../PersistentVolumeArtifactStorageException.java | 34 -- .../sdc/be/csar/storage/CsarSizeReducerTest.java | 125 ------ .../MinIoStorageArtifactStorageManagerTest.java | 120 ++++++ .../storage/MinIoStorageCsarSizeReducerTest.java | 125 ++++++ .../be/csar/storage/NoneStorageManagerTest.java | 78 ++++ ...PersistentVolumeArtifactStorageManagerTest.java | 118 ----- .../csarSizeReducer/dummyToReduce-2-files.zip | Bin 0 -> 3154 bytes .../csarSizeReducer/dummyToReduce-3-files.zip | Bin 0 -> 23905 bytes .../resources/csarSizeReducer/dummyToReduce.zip | Bin 23905 -> 0 bytes .../s3StoreArtifactStorageManager/dummy.csar | Bin 0 -> 25876 bytes 22 files changed, 1316 insertions(+), 970 deletions(-) delete mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java create mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoArtifactInfo.java create mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java create mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java create mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducer.java create mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java delete mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java delete mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java delete mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java create mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java create mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/ArtifactStorageException.java delete mode 100644 common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java delete mode 100644 common-be/src/test/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducerTest.java create mode 100644 common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java create mode 100644 common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducerTest.java create mode 100644 common-be/src/test/java/org/openecomp/sdc/be/csar/storage/NoneStorageManagerTest.java delete mode 100644 common-be/src/test/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManagerTest.java create mode 100644 common-be/src/test/resources/csarSizeReducer/dummyToReduce-2-files.zip create mode 100644 common-be/src/test/resources/csarSizeReducer/dummyToReduce-3-files.zip delete mode 100644 common-be/src/test/resources/csarSizeReducer/dummyToReduce.zip create mode 100644 common-be/src/test/resources/s3StoreArtifactStorageManager/dummy.csar (limited to 'common-be') diff --git a/common-be/pom.xml b/common-be/pom.xml index af93628eb5..d84eec5c0c 100644 --- a/common-be/pom.xml +++ b/common-be/pom.xml @@ -1,156 +1,161 @@ - 4.0.0 - - org.openecomp.sdc.be - common-be - - - org.openecomp.sdc - sdc-main - 1.10.0-SNAPSHOT - - - - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - - - org.openecomp.sdc - common-app-api - ${project.version} - - - com.fasterxml.jackson.core - jackson-core - - - commons-codec - commons-codec - - - - - - org.apache.commons - commons-lang3 - ${lang3.version} - provided - - - - ch.qos.logback - logback-classic - ${logback.version} - provided - - - - com.google.guava - guava - ${guava.version} - provided - - - org.functionaljava - functionaljava - ${functionaljava.version} - provided - - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - - - - org.bouncycastle - bcpkix-jdk15on - ${bouncycastle.version} - compile - - - - org.hamcrest - hamcrest - ${hamcrest.version} - test - - - org.hamcrest - hamcrest-library - ${hamcrest.version} - test - - - org.junit.jupiter - junit-jupiter - ${junitJupiter.version} - test - - - org.mockito - mockito-junit-jupiter - ${mockitoJupiter.version} - test - - - org.onap.sdc.common - onap-tosca-datatype - ${tosca.datatype.version} - - - com.fasterxml.jackson.core - jackson-core - - - - - org.projectlombok - lombok - ${lombok.version} - - - com.google.code.bean-matchers - bean-matchers - ${bean-matchers.version} - test - - - - - org.togglz - togglz-core - ${togglz.version} - - - - - org.togglz - togglz-servlet - ${togglz.version} - - - - - org.togglz - togglz-console - ${togglz.version} - + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + org.openecomp.sdc.be + common-be + + + org.openecomp.sdc + sdc-main + 1.10.0-SNAPSHOT + + + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + io.minio + minio + 8.3.0 + + + + + org.openecomp.sdc + common-app-api + ${project.version} + + + com.fasterxml.jackson.core + jackson-core + + + commons-codec + commons-codec + + + + + + org.apache.commons + commons-lang3 + ${lang3.version} + provided + + + + ch.qos.logback + logback-classic + ${logback.version} + provided + + + + com.google.guava + guava + ${guava.version} + provided + + + org.functionaljava + functionaljava + ${functionaljava.version} + provided + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-core + + + + + + org.bouncycastle + bcpkix-jdk15on + ${bouncycastle.version} + compile + + + + org.hamcrest + hamcrest + ${hamcrest.version} + test + + + org.hamcrest + hamcrest-library + ${hamcrest.version} + test + + + org.junit.jupiter + junit-jupiter + ${junitJupiter.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockitoJupiter.version} + test + + + org.onap.sdc.common + onap-tosca-datatype + ${tosca.datatype.version} + + + com.fasterxml.jackson.core + jackson-core + + + + + org.projectlombok + lombok + ${lombok.version} + + + com.google.code.bean-matchers + bean-matchers + ${bean-matchers.version} + test + + + + + org.togglz + togglz-core + ${togglz.version} + + + + + org.togglz + togglz-servlet + ${togglz.version} + + + + + org.togglz + togglz-console + ${togglz.version} + @@ -159,87 +164,92 @@ ${togglz.version} test - - org.springframework - spring-context - ${spring.version} - compile - - - org.springframework - spring-expression - - - org.springframework - spring-core - - - - - org.springframework - spring-core - ${spring.version} - - - org.apache.cxf - cxf-rt-frontend-jaxrs - ${cxf.version} - - - org.jboss.spec.javax.rmi - jboss-rmi-api_1.0_spec - - - - - - - - - maven-jar-plugin - ${maven-jar-plugin.version} - - - default-jar - package - - jar - test-jar - - - - - - com.github.sylvainlaurent.maven - yaml-json-validator-maven-plugin - - - validate - validate - - validate - - - - - - src/main/resources/**/*.y*ml - src/test/resources/**/*.y*ml - - - - - src/main/resources/**/*.json - src/test/resources/**/*.json - - - - ${skipYamlJsonValidator} - - - - - - + + org.springframework + spring-context + ${spring.version} + compile + + + org.springframework + spring-expression + + + org.springframework + spring-core + + + + + org.springframework + spring-core + ${spring.version} + + + org.apache.cxf + cxf-rt-frontend-jaxrs + ${cxf.version} + + + org.jboss.spec.javax.rmi + jboss-rmi-api_1.0_spec + + + + + org.openecomp.sdc.core + openecomp-common-lib + ${project.version} + + + + + + + maven-jar-plugin + ${maven-jar-plugin.version} + + + default-jar + package + + jar + test-jar + + + + + + com.github.sylvainlaurent.maven + yaml-json-validator-maven-plugin + + + validate + validate + + validate + + + + + + src/main/resources/**/*.y*ml + src/test/resources/**/*.y*ml + + + + + src/main/resources/**/*.json + src/test/resources/**/*.json + + + + ${skipYamlJsonValidator} + + + + + + diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java deleted file mode 100644 index 822acc0766..0000000000 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2021 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.openecomp.sdc.be.csar.storage; - -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; -import lombok.Getter; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FilenameUtils; -import org.openecomp.sdc.be.csar.storage.exception.CsarSizeReducerException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CsarSizeReducer implements PackageSizeReducer { - - private static final Logger LOGGER = LoggerFactory.getLogger(CsarSizeReducer.class); - private static final Set ALLOWED_SIGNATURE_EXTENSIONS = Set.of("cms"); - private static final Set ALLOWED_CERTIFICATE_EXTENSIONS = Set.of("cert", "crt"); - private static final String CSAR_EXTENSION = "csar"; - private static final String UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR = "An unexpected problem happened while reading the CSAR '%s'"; - @Getter - private final AtomicBoolean reduced = new AtomicBoolean(false); - - private final CsarPackageReducerConfiguration configuration; - - public CsarSizeReducer(final CsarPackageReducerConfiguration configuration) { - this.configuration = configuration; - } - - @Override - public byte[] reduce(final Path csarPackagePath) { - if (hasSignedPackageStructure(csarPackagePath)) { - return reduce(csarPackagePath, this::signedZipProcessingConsumer); - } else { - return reduce(csarPackagePath, this::unsignedZipProcessingConsumer); - } - } - - private byte[] reduce(final Path csarPackagePath, final ZipProcessFunction zipProcessingFunction) { - final var reducedCsarPath = Path.of(csarPackagePath + "." + UUID.randomUUID()); - - try (final var zf = new ZipFile(csarPackagePath.toString()); - final var zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(reducedCsarPath)))) { - zf.entries().asIterator().forEachRemaining(zipProcessingFunction.getProcessZipConsumer(csarPackagePath, zf, zos)); - } catch (final IOException ex1) { - rollback(reducedCsarPath); - final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath); - throw new CsarSizeReducerException(errorMsg, ex1); - } - final byte[] reducedCsarBytes; - try { - if (reduced.get()) { - reducedCsarBytes = Files.readAllBytes(reducedCsarPath); - } else { - reducedCsarBytes = Files.readAllBytes(csarPackagePath); - } - } catch (final IOException e) { - final var errorMsg = String.format("Could not read bytes of file '%s'", csarPackagePath); - throw new CsarSizeReducerException(errorMsg, e); - } - try { - Files.delete(reducedCsarPath); - } catch (final IOException e) { - final var errorMsg = String.format("Could not delete temporary file '%s'", reducedCsarPath); - throw new CsarSizeReducerException(errorMsg, e); - } - - return reducedCsarBytes; - } - - private Consumer signedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) { - final var thresholdEntries = configuration.getThresholdEntries(); - final var totalEntryArchive = new AtomicInteger(0); - return zipEntry -> { - final var entryName = zipEntry.getName(); - try { - if (totalEntryArchive.getAndIncrement() > thresholdEntries) { - // too much entries in this archive, can lead to inodes exhaustion of the system - final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); - throw new CsarSizeReducerException(errorMsg); - } - zos.putNextEntry(new ZipEntry(entryName)); - if (!zipEntry.isDirectory()) { - if (entryName.toLowerCase().endsWith(CSAR_EXTENSION)) { - final var internalCsarExtractPath = Path.of(csarPackagePath + "." + UUID.randomUUID()); - Files.copy(zf.getInputStream(zipEntry), internalCsarExtractPath, REPLACE_EXISTING); - zos.write(reduce(internalCsarExtractPath, this::unsignedZipProcessingConsumer)); - Files.delete(internalCsarExtractPath); - } else { - zos.write(zf.getInputStream(zipEntry).readAllBytes()); - } - } - zos.closeEntry(); - } catch (final IOException ei) { - final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); - throw new CsarSizeReducerException(errorMsg, ei); - } - }; - } - - private Consumer unsignedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) { - final var thresholdEntries = configuration.getThresholdEntries(); - final var totalEntryArchive = new AtomicInteger(0); - return zipEntry -> { - final var entryName = zipEntry.getName(); - if (totalEntryArchive.getAndIncrement() > thresholdEntries) { - // too much entries in this archive, can lead to inodes exhaustion of the system - final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); - throw new CsarSizeReducerException(errorMsg); - } - try { - zos.putNextEntry(new ZipEntry(entryName)); - if (!zipEntry.isDirectory()) { - if (isCandidateToRemove(zipEntry)) { - // replace with EMPTY string to avoid package description inconsistency/validation errors - zos.write("".getBytes()); - reduced.set(true); - } else { - zos.write(zf.getInputStream(zipEntry).readAllBytes()); - } - } - zos.closeEntry(); - } catch (final IOException ei) { - final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); - throw new CsarSizeReducerException(errorMsg, ei); - } - }; - } - - private void rollback(final Path reducedCsarPath) { - if (Files.exists(reducedCsarPath)) { - try { - Files.delete(reducedCsarPath); - } catch (final Exception ex2) { - LOGGER.warn("Could not delete temporary file '{}'", reducedCsarPath, ex2); - } - } - } - - private boolean isCandidateToRemove(final ZipEntry zipEntry) { - final String zipEntryName = zipEntry.getName(); - return configuration.getFoldersToStrip().stream().anyMatch(Path.of(zipEntryName)::startsWith) - || zipEntry.getSize() > configuration.getSizeLimit(); - } - - private boolean hasSignedPackageStructure(final Path csarPackagePath) { - final List packagePathList; - try (final var zf = new ZipFile(csarPackagePath.toString())) { - packagePathList = zf.stream() - .filter(zipEntry -> !zipEntry.isDirectory()) - .map(ZipEntry::getName).map(Path::of) - .collect(Collectors.toList()); - } catch (final IOException e) { - final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath); - throw new CsarSizeReducerException(errorMsg, e); - } - - if (CollectionUtils.isEmpty(packagePathList)) { - return false; - } - final int numberOfFiles = packagePathList.size(); - if (numberOfFiles == 2) { - return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList); - } - if (numberOfFiles == 3) { - return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList) && hasOneCertificateFile(packagePathList); - } - return false; - } - - private boolean hasOneInternalPackageFile(final List packagePathList) { - return packagePathList.parallelStream() - .map(Path::toString) - .map(FilenameUtils::getExtension) - .map(String::toLowerCase) - .filter(extension -> extension.endsWith(CSAR_EXTENSION)).count() == 1; - } - - private boolean hasOneSignatureFile(final List packagePathList) { - return packagePathList.parallelStream() - .map(Path::toString) - .map(FilenameUtils::getExtension) - .map(String::toLowerCase) - .filter(ALLOWED_SIGNATURE_EXTENSIONS::contains).count() == 1; - } - - private boolean hasOneCertificateFile(final List packagePathList) { - return packagePathList.parallelStream() - .map(Path::toString) - .map(FilenameUtils::getExtension) - .map(String::toLowerCase) - .filter(ALLOWED_CERTIFICATE_EXTENSIONS::contains).count() == 1; - } - - @FunctionalInterface - private interface ZipProcessFunction { - - Consumer getProcessZipConsumer(Path csarPackagePath, ZipFile zf, ZipOutputStream zos); - } - -} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoArtifactInfo.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoArtifactInfo.java new file mode 100644 index 0000000000..a193cdd6db --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoArtifactInfo.java @@ -0,0 +1,38 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +@Getter +public class MinIoArtifactInfo implements ArtifactInfo { + + private final String bucket; + private final String objectName; + + @Override + public String getInfo() { + return String.format("bucket: %s\n" + + "object: %s", bucket, objectName); + } +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java new file mode 100644 index 0000000000..6f6778f2b0 --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java @@ -0,0 +1,52 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +@Getter +public class MinIoStorageArtifactStorageConfig implements ArtifactStorageConfig { + + private final boolean isEnabled; + private final EndPoint endPoint; + private final Credentials credentials; + private final String tempPath; + + @AllArgsConstructor + @Getter + public static class EndPoint { + + private final String host; + private final int port; + private final boolean secure; + } + + @AllArgsConstructor + @Getter + public static class Credentials { + + private final String accessKey; + private final String secretKey; + } + +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java new file mode 100644 index 0000000000..0a48c2233c --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java @@ -0,0 +1,247 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import static org.openecomp.sdc.common.errors.Messages.EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING; + +import io.minio.BucketExistsArgs; +import io.minio.CopyObjectArgs; +import io.minio.CopySource; +import io.minio.GetObjectArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.MinioClient.Builder; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import java.io.InputStream; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import lombok.Getter; +import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.Credentials; +import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.EndPoint; +import org.openecomp.sdc.be.csar.storage.exception.ArtifactStorageException; +import org.openecomp.sdc.common.CommonConfigurationManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MinIoStorageArtifactStorageManager implements ArtifactStorageManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(MinIoStorageArtifactStorageManager.class); + private static final String EXTERNAL_CSAR_STORE = "externalCsarStore"; + + @Getter + private final MinIoStorageArtifactStorageConfig storageConfiguration; + private final MinioClient minioClient; + + public MinIoStorageArtifactStorageManager() { + this.storageConfiguration = readMinIoStorageArtifactStorageConfig(); + minioClient = initMinioClient(); + } + + //for testing only + MinIoStorageArtifactStorageManager(final ArtifactStorageConfig storageConfiguration) { + this.storageConfiguration = (MinIoStorageArtifactStorageConfig) storageConfiguration; + minioClient = initMinioClient(); + } + + @Override + public ArtifactInfo persist(final String vspId, final String versionId, final ArtifactInfo uploadedArtifactInfo) { + final MinIoArtifactInfo minioObjectTemp = (MinIoArtifactInfo) uploadedArtifactInfo; + try { + minioClient.getObject( + GetObjectArgs.builder() + .bucket(minioObjectTemp.getBucket()) + .object(minioObjectTemp.getObjectName()) + .build() + ); + } catch (final Exception e) { + throw new ArtifactStorageException( + String.format("Failed to retrieve uploaded artifact with bucket '%s' and name '%s' while persisting", + minioObjectTemp.getBucket(), minioObjectTemp.getObjectName()), e); + } + + final var backupPath = backupPreviousVersion(vspId, versionId).orElse(null); + try { + moveFile(minioObjectTemp, vspId, versionId); + } catch (final Exception e) { + rollback(minioObjectTemp, vspId, versionId); + final var errorMsg = String.format("Could not persist artifact for VSP '%s', version '%s'", vspId, versionId); + throw new ArtifactStorageException(errorMsg, e); + } + + removePreviousVersion(backupPath); + + return new MinIoArtifactInfo(vspId, versionId); + } + + @Override + public ArtifactInfo upload(final String vspId, final String versionId, final InputStream fileToUpload) { + + final String name = versionId + "--" + UUID.randomUUID(); + try { + // Make bucket if not exist. + final boolean found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(vspId).build()); + + if (!found) { + // Make a new bucket ${vspId} . + minioClient.makeBucket(MakeBucketArgs.builder().bucket(vspId).build()); + } else { + LOGGER.info("Bucket '{}' already exists.", vspId); + } + + minioClient.putObject( + PutObjectArgs.builder() + .bucket(vspId) + .object(name) + .stream(fileToUpload, fileToUpload.available(), -1) + .build() + ); + + } catch (final Exception e) { + throw new ArtifactStorageException("Failed to upload artifact", e); + } + + return new MinIoArtifactInfo(vspId, name); + } + + @Override + public boolean isEnabled() { + return storageConfiguration != null && storageConfiguration.isEnabled(); + } + + @Override + public InputStream get(final ArtifactInfo artifactInfo) { + final MinIoArtifactInfo minioObject = (MinIoArtifactInfo) artifactInfo; + try { + return minioClient.getObject(GetObjectArgs.builder() + .bucket(minioObject.getBucket()) + .object(minioObject.getObjectName()) + .build()); + } catch (final Exception e) { + throw new ArtifactStorageException("Failed to get Object", e); + } + } + + @Override + public void delete(final ArtifactInfo artifactInfo) { + final MinIoArtifactInfo minioObject = (MinIoArtifactInfo) artifactInfo; + try { + minioClient.removeObject(RemoveObjectArgs.builder() + .bucket(minioObject.getBucket()) + .object(minioObject.getObjectName()) + .bypassGovernanceMode(true) + .build()); + } catch (final Exception e) { + throw new ArtifactStorageException(String.format("Failed to delete '%s'", minioObject.getObjectName()), e); + } + + } + + private Optional backupPreviousVersion(final String vspId, final String versionId) { + + final String tempName = versionId + "--" + UUID.randomUUID().toString(); + try { + copy(vspId, tempName, versionId); + } catch (final Exception e) { + return Optional.empty(); + } + + return Optional.of(new MinIoArtifactInfo(vspId, tempName)); + } + + private void rollback(final MinIoArtifactInfo minioObject, final String vspId, final String versionId) { + try { + moveFile(minioObject, vspId, versionId); + } catch (final Exception ex) { + LOGGER.warn("Could not rollback the backup '{}' to the original '{}'", versionId, minioObject.getObjectName(), ex); + } + } + + private void removePreviousVersion(final MinIoArtifactInfo minioObject) { + if (minioObject == null) { + return; + } + delete(minioObject); + } + + private void moveFile(final MinIoArtifactInfo minioObject, final String vspId, final String versionId) { + try { + copy(vspId, versionId, minioObject.getObjectName()); + } catch (final Exception e) { + throw new ArtifactStorageException("Failed to move", e); + } + delete(minioObject); + } + + private void copy(final String vspId, final String versionId, final String objectName) throws Exception { + minioClient.copyObject( + CopyObjectArgs.builder() + .bucket(vspId) + .object(versionId) + .source(CopySource.builder() + .bucket(vspId) + .object(objectName) + .build()) + .build()); + } + + private MinIoStorageArtifactStorageConfig readMinIoStorageArtifactStorageConfig() { + final var commonConfigurationManager = CommonConfigurationManager.getInstance(); + + final Map endpoint = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "endpoint", null); + final Map credentials = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "credentials", null); + final String tempPath = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "tempPath", null); + LOGGER.info("ArtifactConfig.endpoint: '{}'", endpoint); + LOGGER.info("ArtifactConfig.credentials: '{}'", credentials); + LOGGER.info("ArtifactConfig.tempPath: '{}'", tempPath); + + if (endpoint == null) { + throw new RuntimeException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("endpoint")); + } + if (credentials == null) { + throw new RuntimeException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("credentials")); + } + if (tempPath == null) { + throw new RuntimeException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("tempPath")); + } + final String host = (String) endpoint.getOrDefault("host", null); + final int port = (int) endpoint.getOrDefault("port", 0); + final boolean secure = (boolean) endpoint.getOrDefault("secure", false); + + final String accessKey = (String) credentials.getOrDefault("accessKey", null); + final String secretKey = (String) credentials.getOrDefault("secretKey", null); + + return new MinIoStorageArtifactStorageConfig(true, new EndPoint(host, port, secure), new Credentials(accessKey, secretKey), tempPath); + } + + private MinioClient initMinioClient() { + final EndPoint endPoint = storageConfiguration.getEndPoint(); + final Credentials credentials = storageConfiguration.getCredentials(); + + final Builder builder = MinioClient.builder(); + return builder + .endpoint(endPoint.getHost(), endPoint.getPort(), endPoint.isSecure()) + .credentials(credentials.getAccessKey(), credentials.getSecretKey()) + .build(); + } + +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducer.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducer.java new file mode 100644 index 0000000000..3181b088c0 --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducer.java @@ -0,0 +1,233 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; +import lombok.Getter; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FilenameUtils; +import org.openecomp.sdc.be.csar.storage.exception.CsarSizeReducerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MinIoStorageCsarSizeReducer implements PackageSizeReducer { + + private static final Logger LOGGER = LoggerFactory.getLogger(MinIoStorageCsarSizeReducer.class); + private static final Set ALLOWED_SIGNATURE_EXTENSIONS = Set.of("cms"); + private static final Set ALLOWED_CERTIFICATE_EXTENSIONS = Set.of("cert", "crt"); + private static final String CSAR_EXTENSION = "csar"; + private static final String UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR = "An unexpected problem happened while reading the CSAR '%s'"; + @Getter + private final AtomicBoolean reduced = new AtomicBoolean(false); + + private final CsarPackageReducerConfiguration configuration; + + public MinIoStorageCsarSizeReducer(final CsarPackageReducerConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public byte[] reduce(final Path csarPackagePath) { + if (hasSignedPackageStructure(csarPackagePath)) { + return reduce(csarPackagePath, this::signedZipProcessingConsumer); + } else { + return reduce(csarPackagePath, this::unsignedZipProcessingConsumer); + } + } + + private byte[] reduce(final Path csarPackagePath, final ZipProcessFunction zipProcessingFunction) { + final var reducedCsarPath = Path.of(csarPackagePath + "." + UUID.randomUUID()); + + try (final var zf = new ZipFile(csarPackagePath.toString()); + final var zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(reducedCsarPath)))) { + zf.entries().asIterator().forEachRemaining(zipProcessingFunction.getProcessZipConsumer(csarPackagePath, zf, zos)); + } catch (final IOException ex1) { + rollback(reducedCsarPath); + final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath); + throw new CsarSizeReducerException(errorMsg, ex1); + } + final byte[] reducedCsarBytes; + try { + if (reduced.get()) { + reducedCsarBytes = Files.readAllBytes(reducedCsarPath); + } else { + reducedCsarBytes = Files.readAllBytes(csarPackagePath); + } + } catch (final IOException e) { + final var errorMsg = String.format("Could not read bytes of file '%s'", csarPackagePath); + throw new CsarSizeReducerException(errorMsg, e); + } + try { + Files.delete(reducedCsarPath); + } catch (final IOException e) { + final var errorMsg = String.format("Could not delete temporary file '%s'", reducedCsarPath); + throw new CsarSizeReducerException(errorMsg, e); + } + + return reducedCsarBytes; + } + + private Consumer signedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) { + final var thresholdEntries = configuration.getThresholdEntries(); + final var totalEntryArchive = new AtomicInteger(0); + return zipEntry -> { + final var entryName = zipEntry.getName(); + try { + if (totalEntryArchive.getAndIncrement() > thresholdEntries) { + // too much entries in this archive, can lead to inodes exhaustion of the system + final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); + throw new CsarSizeReducerException(errorMsg); + } + zos.putNextEntry(new ZipEntry(entryName)); + if (!zipEntry.isDirectory()) { + if (entryName.toLowerCase().endsWith(CSAR_EXTENSION)) { + final var internalCsarExtractPath = Path.of(csarPackagePath + "." + UUID.randomUUID()); + Files.copy(zf.getInputStream(zipEntry), internalCsarExtractPath, REPLACE_EXISTING); + zos.write(reduce(internalCsarExtractPath, this::unsignedZipProcessingConsumer)); + Files.delete(internalCsarExtractPath); + } else { + zos.write(zf.getInputStream(zipEntry).readAllBytes()); + } + } + zos.closeEntry(); + } catch (final IOException ei) { + final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); + throw new CsarSizeReducerException(errorMsg, ei); + } + }; + } + + private Consumer unsignedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) { + final var thresholdEntries = configuration.getThresholdEntries(); + final var totalEntryArchive = new AtomicInteger(0); + return zipEntry -> { + final var entryName = zipEntry.getName(); + if (totalEntryArchive.getAndIncrement() > thresholdEntries) { + // too much entries in this archive, can lead to inodes exhaustion of the system + final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); + throw new CsarSizeReducerException(errorMsg); + } + try { + zos.putNextEntry(new ZipEntry(entryName)); + if (!zipEntry.isDirectory()) { + if (isCandidateToRemove(zipEntry)) { + // replace with EMPTY string to avoid package description inconsistency/validation errors + zos.write("".getBytes()); + reduced.set(true); + } else { + zos.write(zf.getInputStream(zipEntry).readAllBytes()); + } + } + zos.closeEntry(); + } catch (final IOException ei) { + final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); + throw new CsarSizeReducerException(errorMsg, ei); + } + }; + } + + private void rollback(final Path reducedCsarPath) { + if (Files.exists(reducedCsarPath)) { + try { + Files.delete(reducedCsarPath); + } catch (final Exception ex2) { + LOGGER.warn("Could not delete temporary file '{}'", reducedCsarPath, ex2); + } + } + } + + private boolean isCandidateToRemove(final ZipEntry zipEntry) { + final String zipEntryName = zipEntry.getName(); + return configuration.getFoldersToStrip().stream().anyMatch(Path.of(zipEntryName)::startsWith) + || zipEntry.getSize() > configuration.getSizeLimit(); + } + + private boolean hasSignedPackageStructure(final Path csarPackagePath) { + final List packagePathList; + try (final var zf = new ZipFile(csarPackagePath.toString())) { + packagePathList = zf.stream() + .filter(zipEntry -> !zipEntry.isDirectory()) + .map(ZipEntry::getName).map(Path::of) + .collect(Collectors.toList()); + } catch (final IOException e) { + final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath); + throw new CsarSizeReducerException(errorMsg, e); + } + + if (CollectionUtils.isEmpty(packagePathList)) { + return false; + } + final int numberOfFiles = packagePathList.size(); + if (numberOfFiles == 2) { + return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList); + } + if (numberOfFiles == 3) { + return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList) && hasOneCertificateFile(packagePathList); + } + return false; + } + + private boolean hasOneInternalPackageFile(final List packagePathList) { + return packagePathList.parallelStream() + .map(Path::toString) + .map(FilenameUtils::getExtension) + .map(String::toLowerCase) + .filter(extension -> extension.endsWith(CSAR_EXTENSION)).count() == 1; + } + + private boolean hasOneSignatureFile(final List packagePathList) { + return packagePathList.parallelStream() + .map(Path::toString) + .map(FilenameUtils::getExtension) + .map(String::toLowerCase) + .filter(ALLOWED_SIGNATURE_EXTENSIONS::contains).count() == 1; + } + + private boolean hasOneCertificateFile(final List packagePathList) { + return packagePathList.parallelStream() + .map(Path::toString) + .map(FilenameUtils::getExtension) + .map(String::toLowerCase) + .filter(ALLOWED_CERTIFICATE_EXTENSIONS::contains).count() == 1; + } + + @FunctionalInterface + private interface ZipProcessFunction { + + Consumer getProcessZipConsumer(Path csarPackagePath, ZipFile zf, ZipOutputStream zos); + } + +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java new file mode 100644 index 0000000000..3fa22d41be --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/NoneStorageManager.java @@ -0,0 +1,54 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import java.io.InputStream; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +public class NoneStorageManager implements ArtifactStorageManager { + + @Override + public ArtifactInfo persist(final String vspId, final String versionId, final ArtifactInfo uploadedArtifactInfo) { + throw new UnsupportedOperationException(); + } + + @Override + public ArtifactInfo upload(final String vspId, final String versionId, final InputStream fileToUpload) { + throw new UnsupportedOperationException(); + } + + @Override + public ArtifactStorageConfig getStorageConfiguration() { + throw new UnsupportedOperationException(); + } + + @Override + public InputStream get(final ArtifactInfo artifactInfo) { + throw new UnsupportedOperationException(); + } + + @Override + public void delete(final ArtifactInfo artifactInfo) { + throw new UnsupportedOperationException(); + } + +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java deleted file mode 100644 index 0472661fd9..0000000000 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2021 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.openecomp.sdc.be.csar.storage; - -import java.nio.file.Path; -import lombok.AllArgsConstructor; -import lombok.Getter; - -@AllArgsConstructor -public class PersistentStorageArtifactInfo implements ArtifactInfo { - - @Getter - private final Path path; - -} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java deleted file mode 100644 index d3cd6fb302..0000000000 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2021 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.openecomp.sdc.be.csar.storage; - -import java.nio.file.Path; -import lombok.Data; - -@Data -public class PersistentVolumeArtifactStorageConfig implements ArtifactStorageConfig { - - private final boolean isEnabled; - private final Path storagePath; - -} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java deleted file mode 100644 index 10629b3edb..0000000000 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2021 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.openecomp.sdc.be.csar.storage; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.Optional; -import java.util.UUID; -import org.openecomp.sdc.be.csar.storage.exception.PersistentVolumeArtifactStorageException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PersistentVolumeArtifactStorageManager implements ArtifactStorageManager { - - private static final Logger LOGGER = LoggerFactory.getLogger(PersistentVolumeArtifactStorageManager.class); - - private final PersistentVolumeArtifactStorageConfig storageConfiguration; - - public PersistentVolumeArtifactStorageManager(final ArtifactStorageConfig storageConfiguration) { - this.storageConfiguration = (PersistentVolumeArtifactStorageConfig) storageConfiguration; - } - - @Override - public ArtifactInfo persist(final String vspId, final String versionId, final ArtifactInfo uploadedArtifactInfo) { - final var temporaryPath = uploadedArtifactInfo.getPath(); - if (!Files.exists(temporaryPath)) { - throw new PersistentVolumeArtifactStorageException(String.format("Given artifact does not exist '%s'", uploadedArtifactInfo.getPath())); - } - - final var filePath = buildFilePath(vspId, versionId); - final var backupPath = backupPreviousVersion(filePath).orElse(null); - try { - moveFile(temporaryPath, filePath); - } catch (final Exception e) { - rollback(backupPath, filePath); - final var errorMsg = String.format("Could not persist artifact for VSP '%s', version '%s'", vspId, versionId); - throw new PersistentVolumeArtifactStorageException(errorMsg, e); - } - - removePreviousVersion(backupPath); - - return new PersistentStorageArtifactInfo(filePath); - } - - @Override - public ArtifactInfo upload(final String vspId, final String versionId, final InputStream artifactInputStream) { - final var destinationFolder = buildDestinationFolder(vspId, versionId); - try { - Files.createDirectories(destinationFolder); - } catch (final IOException e) { - throw new PersistentVolumeArtifactStorageException(String.format("Could not create directory '%s'", destinationFolder), e); - } - - final var filePath = createTempFilePath(destinationFolder); - try { - persist(artifactInputStream, filePath); - } catch (final IOException e) { - throw new PersistentVolumeArtifactStorageException(String.format("Could not persist artifact '%s'", filePath), e); - } - - return new PersistentStorageArtifactInfo(filePath); - } - - private Path buildFilePath(final String vspId, final String versionId) { - return buildDestinationFolder(vspId, versionId).resolve(versionId); - } - - @Override - public boolean isEnabled() { - return storageConfiguration != null && storageConfiguration.isEnabled(); - } - - private Optional backupPreviousVersion(final Path filePath) { - if (!Files.exists(filePath)) { - return Optional.empty(); - } - - final var backupPath = Path.of(filePath + UUID.randomUUID().toString()); - moveFile(filePath, backupPath); - return Optional.ofNullable(backupPath); - } - - private void rollback(final Path backupPath, final Path filePath) { - try { - moveFile(backupPath, filePath); - } catch (final Exception ex) { - LOGGER.warn("Could not rollback the backup file '{}' to the original '{}'", backupPath, filePath, ex); - } - } - - private void removePreviousVersion(final Path filePath) { - if (filePath == null || !Files.exists(filePath)) { - return; - } - - try { - Files.delete(filePath); - } catch (final IOException e) { - throw new PersistentVolumeArtifactStorageException(String.format("Could not delete previous version '%s'", filePath), e); - } - } - - private Path createTempFilePath(final Path destinationFolder) { - final var retries = 10; - return createTempFilePath(destinationFolder, retries).orElseThrow(() -> { - throw new PersistentVolumeArtifactStorageException(String.format("Could not generate upload file path after '%s' retries", retries)); - }); - } - - private Optional createTempFilePath(final Path destinationFolder, int retries) { - for (var i = 0; i < retries; i++) { - final var filePath = destinationFolder.resolve(UUID.randomUUID().toString()); - if (Files.notExists(filePath)) { - return Optional.of(filePath); - } - } - return Optional.empty(); - } - - private Path buildDestinationFolder(final String vspId, final String versionId) { - return storageConfiguration.getStoragePath().resolve(vspId).resolve(versionId); - } - - private void persist(final InputStream artifactInputStream, final Path filePath) throws IOException { - try (final var inputStream = artifactInputStream; - final var fileOutputStream = new FileOutputStream(filePath.toFile());) { - inputStream.transferTo(fileOutputStream); - } - } - - private void moveFile(final Path from, final Path to) { - try { - Files.move(from, to, StandardCopyOption.REPLACE_EXISTING); - } catch (final IOException e) { - throw new PersistentVolumeArtifactStorageException(String.format("Could not move file '%s' to '%s'", from, to), e); - } - } - -} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java new file mode 100644 index 0000000000..d120b3af8d --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java @@ -0,0 +1,91 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import static org.openecomp.sdc.be.csar.storage.StorageFactory.StorageType.NONE; +import static org.openecomp.sdc.be.csar.storage.StorageFactory.StorageType.findByName; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.NoArgsConstructor; +import org.openecomp.sdc.common.CommonConfigurationManager; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; + +@NoArgsConstructor +public class StorageFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(StorageFactory.class); + private static final String EXTERNAL_CSAR_STORE = "externalCsarStore"; + + public ArtifactStorageManager createArtifactStorageManager() { + switch (getConfiguredArtifactStorageType()) { + case MINIO: // MinIoStorage enabled + return new MinIoStorageArtifactStorageManager(); + default:// all configured, nothing enabled + return new NoneStorageManager(); + } + } + + public Optional createPackageSizeReducer() { + switch (getConfiguredArtifactStorageType()) { + case MINIO: // MinIoStorage enabled + return Optional.of(new MinIoStorageCsarSizeReducer(readPackageReducerConfiguration())); + default:// all configured, nothing enabled + return Optional.empty(); + } + } + + private StorageType getConfiguredArtifactStorageType() { + final var commonConfigurationManager = CommonConfigurationManager.getInstance(); + final String storageType = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "storageType", NONE.name()); + LOGGER.info("ArtifactConfig.storageType: '{}'", storageType); + return findByName(storageType); + } + + private CsarPackageReducerConfiguration readPackageReducerConfiguration() { + final var commonConfigurationManager = CommonConfigurationManager.getInstance(); + final List foldersToStrip = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "foldersToStrip", new ArrayList<>()); + final int sizeLimit = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "sizeLimit", 1000000); + final int thresholdEntries = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "thresholdEntries", 10000); + LOGGER.info("Folders to strip: '{}'", String.join(", ", foldersToStrip)); + final Set foldersToStripPathSet = foldersToStrip.stream().map(Path::of).collect(Collectors.toSet()); + return new CsarPackageReducerConfiguration(foldersToStripPathSet, sizeLimit, thresholdEntries); + } + + public enum StorageType { + NONE, + MINIO; + + public static StorageType findByName(String name) { + for (StorageType curr : StorageType.values()) { + if (curr.name().equals(name)) { + return curr; + } + } + return null; + } + } +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/ArtifactStorageException.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/ArtifactStorageException.java new file mode 100644 index 0000000000..aa621611df --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/ArtifactStorageException.java @@ -0,0 +1,34 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage.exception; + +import org.openecomp.sdc.be.exception.BusinessException; + +public class ArtifactStorageException extends BusinessException { + + public ArtifactStorageException(final String message, final Throwable cause) { + super(message, cause); + } + + public ArtifactStorageException(final String message) { + super(message); + } +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java deleted file mode 100644 index 28fff65bb6..0000000000 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2021 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.openecomp.sdc.be.csar.storage.exception; - -import org.openecomp.sdc.be.exception.BusinessException; - -public class PersistentVolumeArtifactStorageException extends BusinessException { - - public PersistentVolumeArtifactStorageException(final String message, final Throwable cause) { - super(message, cause); - } - - public PersistentVolumeArtifactStorageException(final String message) { - super(message); - } -} diff --git a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducerTest.java b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducerTest.java deleted file mode 100644 index e9748f0a16..0000000000 --- a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducerTest.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * - - * ============LICENSE_START======================================================= - * Copyright (C) 2021 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.openecomp.sdc.be.csar.storage; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.when; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.openecomp.sdc.common.zip.ZipUtils; -import org.openecomp.sdc.common.zip.exception.ZipException; - -class CsarSizeReducerTest { - - @Mock - private CsarPackageReducerConfiguration csarPackageReducerConfiguration; - @InjectMocks - private CsarSizeReducer csarSizeReducer; - - @BeforeEach - void setUp() { - MockitoAnnotations.openMocks(this); - } - - @ParameterizedTest - @ValueSource(strings = {"dummyToReduce.zip", "dummyToReduce.csar", "dummyToNotReduce.csar"}) - void reduceByPathAndSizeTest(String fileName) throws ZipException { - final var pathToReduce1 = Path.of("Files/images"); - final var pathToReduce2 = Path.of("Files/Scripts/my_script.sh"); - final var sizeLimit = 150000L; - when(csarPackageReducerConfiguration.getSizeLimit()).thenReturn(sizeLimit); - when(csarPackageReducerConfiguration.getFoldersToStrip()).thenReturn(Set.of(pathToReduce1, pathToReduce2)); - when(csarPackageReducerConfiguration.getThresholdEntries()).thenReturn(10000); - - final var csarPath = Path.of("src/test/resources/csarSizeReducer/" + fileName); - - final Map originalCsar = ZipUtils.readZip(csarPath.toFile(), false); - - final byte[] reduce = csarSizeReducer.reduce(csarPath); - - final Map reducedCsar = ZipUtils.readZip(reduce, false); - - assertEquals(originalCsar.keySet().size(), reducedCsar.keySet().size(), "No file should be removed"); - for (final Entry originalEntry : originalCsar.entrySet()) { - final var originalFilePath = originalEntry.getKey(); - final byte[] originalBytes = originalEntry.getValue(); - assertTrue(reducedCsar.containsKey(originalFilePath), - String.format("No file should be removed, but it is missing original file '%s'", originalFilePath)); - - final String extention = fileName.substring(fileName.lastIndexOf('.') + 1); - switch (extention.toLowerCase()) { - case "zip": - verifyZIP(pathToReduce1, pathToReduce2, sizeLimit, reducedCsar, originalFilePath, originalBytes); - break; - case "csar": - verifyCSAR(pathToReduce1, pathToReduce2, sizeLimit, reducedCsar, originalFilePath, originalBytes); - break; - default: - fail("Unexpected file extention"); - break; - } - } - } - - private void verifyCSAR(final Path pathToReduce1, final Path pathToReduce2, final long sizeLimit, final Map reducedCsar, - final String originalFilePath, final byte[] originalBytes) { - if (originalFilePath.startsWith(pathToReduce1.toString()) || originalFilePath.startsWith(pathToReduce2.toString()) - || originalBytes.length > sizeLimit) { - assertArrayEquals("".getBytes(StandardCharsets.UTF_8), reducedCsar.get(originalFilePath), - String.format("File '%s' expected to be reduced to empty string", originalFilePath)); - } else { - assertArrayEquals(originalBytes, reducedCsar.get(originalFilePath), - String.format("File '%s' expected to be equal", originalFilePath)); - } - } - - private void verifyZIP(final Path pathToReduce1, final Path pathToReduce2, final long sizeLimit, final Map reducedCsar, - final String originalFilePath, final byte[] originalBytes) { - if (originalFilePath.startsWith(pathToReduce1.toString()) || originalFilePath.startsWith(pathToReduce2.toString()) - || originalBytes.length > sizeLimit) { - assertArrayEquals("".getBytes(StandardCharsets.UTF_8), reducedCsar.get(originalFilePath), - String.format("File '%s' expected to be reduced to empty string", originalFilePath)); - } else { - if (originalFilePath.endsWith(".csar") && csarSizeReducer.getReduced().get()) { - assertNotEquals(originalBytes.length, reducedCsar.get(originalFilePath).length, - String.format("File '%s' expected to be NOT equal", originalFilePath)); - } else { - assertArrayEquals(originalBytes, reducedCsar.get(originalFilePath), - String.format("File '%s' expected to be equal", originalFilePath)); - } - } - } -} diff --git a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java new file mode 100644 index 0000000000..41eed0cebb --- /dev/null +++ b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java @@ -0,0 +1,120 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +import io.minio.BucketExistsArgs; +import io.minio.MinioClient; +import java.io.IOException; +import java.io.InputStream; +import javax.activation.DataHandler; +import org.apache.cxf.jaxrs.ext.multipart.Attachment; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; +import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.Credentials; +import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.EndPoint; + +@ExtendWith(MockitoExtension.class) +class MinIoStorageArtifactStorageManagerTest { + + public static final String VSP_ID = "vsp-id"; + public static final String VERSION_ID = "version-id"; + private MinIoStorageArtifactStorageManager testSubject; + @Mock + private MinioClient minioClient; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private MinioClient.Builder builderMinio; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private BucketExistsArgs.Builder builderBucketExistsArgs; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + + try (MockedStatic utilities = Mockito.mockStatic(MinioClient.class)) { + utilities.when(MinioClient::builder).thenReturn(builderMinio); + when(builderMinio + .endpoint(anyString(), anyInt(), anyBoolean()) + .credentials(anyString(), anyString()) + .build() + ).thenReturn(minioClient); + + testSubject = new MinIoStorageArtifactStorageManager( + new MinIoStorageArtifactStorageConfig(true, new EndPoint("host", 9000, false), new Credentials("accessKey", "secretKey"), "")); + } + } + + @Test + void testUpload() throws Exception { + + when(builderBucketExistsArgs + .bucket(anyString()) + .build() + ).thenReturn(new BucketExistsArgs()); + when(minioClient.bucketExists(any(BucketExistsArgs.class))).thenReturn(true); + + final Attachment attachment = mockAttachment(); + final ArtifactInfo result = testSubject.upload(VSP_ID, VERSION_ID, attachment.getDataHandler().getInputStream()); + Assertions.assertNotNull(result); + Assertions.assertTrue(result instanceof MinIoArtifactInfo); + Assertions.assertEquals(VSP_ID, ((MinIoArtifactInfo) result).getBucket()); + Assertions.assertTrue(((MinIoArtifactInfo) result).getObjectName().startsWith(VERSION_ID + "--")); + } + + @Test + void testPersist() { + final ArtifactInfo result = testSubject.persist(VSP_ID, VERSION_ID, new MinIoArtifactInfo(VSP_ID, VERSION_ID)); + Assertions.assertNotNull(result); + Assertions.assertTrue(result instanceof MinIoArtifactInfo); + Assertions.assertEquals(VSP_ID, ((MinIoArtifactInfo) result).getBucket()); + Assertions.assertEquals(VERSION_ID, ((MinIoArtifactInfo) result).getObjectName()); + } + + @Test + void testIsEnabled() { + Assertions.assertTrue(testSubject.isEnabled()); + } + + private Attachment mockAttachment() throws IOException { + final Attachment attachment = Mockito.mock(Attachment.class); + final DataHandler dataHandler = Mockito.mock(DataHandler.class); + final InputStream inputStream = Mockito.mock(InputStream.class); + when(dataHandler.getInputStream()).thenReturn(inputStream); + when(attachment.getDataHandler()).thenReturn(dataHandler); + return attachment; + } + +} diff --git a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducerTest.java b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducerTest.java new file mode 100644 index 0000000000..6515c6fb1e --- /dev/null +++ b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducerTest.java @@ -0,0 +1,125 @@ +/* + * - + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.openecomp.sdc.common.zip.ZipUtils; +import org.openecomp.sdc.common.zip.exception.ZipException; + +class MinIoStorageCsarSizeReducerTest { + + @Mock + private CsarPackageReducerConfiguration csarPackageReducerConfiguration; + @InjectMocks + private MinIoStorageCsarSizeReducer minIoStorageCsarSizeReducer; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + } + + @ParameterizedTest + @ValueSource(strings = {"dummyToReduce-3-files.zip", "dummyToReduce.csar", "dummyToNotReduce.csar", "dummyToReduce-2-files.zip"}) + void reduceByPathAndSizeTest(String fileName) throws ZipException { + final var pathToReduce1 = Path.of("Files/images"); + final var pathToReduce2 = Path.of("Files/Scripts/my_script.sh"); + final var sizeLimit = 150000L; + when(csarPackageReducerConfiguration.getSizeLimit()).thenReturn(sizeLimit); + when(csarPackageReducerConfiguration.getFoldersToStrip()).thenReturn(Set.of(pathToReduce1, pathToReduce2)); + when(csarPackageReducerConfiguration.getThresholdEntries()).thenReturn(10000); + + final var csarPath = Path.of("src/test/resources/csarSizeReducer/" + fileName); + + final Map originalCsar = ZipUtils.readZip(csarPath.toFile(), false); + + final byte[] reduce = minIoStorageCsarSizeReducer.reduce(csarPath); + + final Map reducedCsar = ZipUtils.readZip(reduce, false); + + assertEquals(originalCsar.keySet().size(), reducedCsar.keySet().size(), "No file should be removed"); + for (final Entry originalEntry : originalCsar.entrySet()) { + final var originalFilePath = originalEntry.getKey(); + final byte[] originalBytes = originalEntry.getValue(); + assertTrue(reducedCsar.containsKey(originalFilePath), + String.format("No file should be removed, but it is missing original file '%s'", originalFilePath)); + + final String extention = fileName.substring(fileName.lastIndexOf('.') + 1); + switch (extention.toLowerCase()) { + case "zip": + verifyZIP(pathToReduce1, pathToReduce2, sizeLimit, reducedCsar, originalFilePath, originalBytes); + break; + case "csar": + verifyCSAR(pathToReduce1, pathToReduce2, sizeLimit, reducedCsar, originalFilePath, originalBytes); + break; + default: + fail("Unexpected file extention"); + break; + } + } + } + + private void verifyCSAR(final Path pathToReduce1, final Path pathToReduce2, final long sizeLimit, final Map reducedCsar, + final String originalFilePath, final byte[] originalBytes) { + if (originalFilePath.startsWith(pathToReduce1.toString()) || originalFilePath.startsWith(pathToReduce2.toString()) + || originalBytes.length > sizeLimit) { + assertArrayEquals("".getBytes(StandardCharsets.UTF_8), reducedCsar.get(originalFilePath), + String.format("File '%s' expected to be reduced to empty string", originalFilePath)); + } else { + assertArrayEquals(originalBytes, reducedCsar.get(originalFilePath), + String.format("File '%s' expected to be equal", originalFilePath)); + } + } + + private void verifyZIP(final Path pathToReduce1, final Path pathToReduce2, final long sizeLimit, final Map reducedCsar, + final String originalFilePath, final byte[] originalBytes) { + if (originalFilePath.startsWith(pathToReduce1.toString()) || originalFilePath.startsWith(pathToReduce2.toString()) + || originalBytes.length > sizeLimit) { + assertArrayEquals("".getBytes(StandardCharsets.UTF_8), reducedCsar.get(originalFilePath), + String.format("File '%s' expected to be reduced to empty string", originalFilePath)); + } else { + if (originalFilePath.endsWith(".csar") && minIoStorageCsarSizeReducer.getReduced().get()) { + assertNotEquals(originalBytes.length, reducedCsar.get(originalFilePath).length, + String.format("File '%s' expected to be NOT equal", originalFilePath)); + } else { + assertArrayEquals(originalBytes, reducedCsar.get(originalFilePath), + String.format("File '%s' expected to be equal", originalFilePath)); + } + } + } +} \ No newline at end of file diff --git a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/NoneStorageManagerTest.java b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/NoneStorageManagerTest.java new file mode 100644 index 0000000000..58394b9b76 --- /dev/null +++ b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/NoneStorageManagerTest.java @@ -0,0 +1,78 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import java.io.ByteArrayInputStream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class NoneStorageManagerTest { + + public static final MinIoArtifactInfo UPLOADED_ARTIFACT_INFO = new MinIoArtifactInfo("bucket", "object"); + private NoneStorageManager testSubject; + + @BeforeEach + void setUp() { + testSubject = new NoneStorageManager(); + } + + @Test + void testCtor() { + Assertions.assertTrue(testSubject instanceof NoneStorageManager); + } + + @Test + void testPersist() { + Assertions.assertThrows(UnsupportedOperationException.class, + () -> testSubject.persist("vspId", "versionId", UPLOADED_ARTIFACT_INFO)); + } + + @Test + void testUpload() { + Assertions.assertThrows(UnsupportedOperationException.class, + () -> testSubject.upload("vspId", "versionId", new ByteArrayInputStream(new byte[0]))); + } + + @Test + void testGetStorageConfiguration() { + Assertions.assertThrows(UnsupportedOperationException.class, () -> testSubject.getStorageConfiguration()); + } + + @Test + void testGet() { + Assertions.assertThrows(UnsupportedOperationException.class, () -> testSubject.get(UPLOADED_ARTIFACT_INFO)); + } + + @Test + void testDelete() { + Assertions.assertThrows(UnsupportedOperationException.class, () -> testSubject.delete(UPLOADED_ARTIFACT_INFO)); + } + + @Test + void testIsEnabled() { + Assertions.assertFalse(testSubject.isEnabled()); + } + +} diff --git a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManagerTest.java b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManagerTest.java deleted file mode 100644 index ab8c11c7c1..0000000000 --- a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManagerTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2021 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.openecomp.sdc.be.csar.storage; - -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Objects; -import javax.activation.DataHandler; -import org.apache.commons.io.IOUtils; -import org.apache.cxf.jaxrs.ext.multipart.Attachment; -import org.apache.cxf.jaxrs.ext.multipart.ContentDisposition; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import org.mockito.ArgumentMatchers; -import org.mockito.Mockito; - -@TestMethodOrder(OrderAnnotation.class) -class PersistentVolumeArtifactStorageManagerTest { - - private static final String SRC_TEST_RESOURCES = "src/test/resources/"; - - private PersistentVolumeArtifactStorageManager testSubject; - - @BeforeEach - void setUp() { - testSubject = new PersistentVolumeArtifactStorageManager(new PersistentVolumeArtifactStorageConfig(true, Path.of(SRC_TEST_RESOURCES))); - } - - @AfterAll - static void tearDown() throws IOException { - Files.move(Path.of(SRC_TEST_RESOURCES + "vspId/versionId/versionId"), - Path.of(SRC_TEST_RESOURCES + "persistentVolumeArtifactStorageManager/dummy.csar")); - Files.list(Path.of("src/test/resources/vspId/versionId/")).forEach(path -> { - try { - Files.deleteIfExists(path); - } catch (IOException e) { - e.printStackTrace(); - } - }); - Files.deleteIfExists(Path.of(SRC_TEST_RESOURCES + "vspId/versionId/")); - Files.deleteIfExists(Path.of(SRC_TEST_RESOURCES + "vspId/")); - } - - @Test - @Order(1) - void testUpload() throws IOException { - final Attachment attachment = mockAttachment("dummy.csar", this.getClass().getResource("/persistentVolumeArtifactStorageManager/dummy.csar")); - final ArtifactInfo result = testSubject.upload("vspId", "versionId", attachment.getDataHandler().getInputStream()); - Assertions.assertNotNull(result); - Assertions.assertNotNull(result.getPath()); - Assertions.assertTrue(result.getPath().startsWith(Path.of(SRC_TEST_RESOURCES + "vspId/versionId/"))); - } - - @Test - @Order(2) - void testPersist() { - final ArtifactInfo result = testSubject.persist("vspId", "versionId", - new PersistentStorageArtifactInfo(Path.of(SRC_TEST_RESOURCES + "persistentVolumeArtifactStorageManager/dummy.csar"))); - Assertions.assertNotNull(result); - Assertions.assertNotNull(result.getPath()); - Assertions.assertTrue(result.getPath().startsWith(Path.of(SRC_TEST_RESOURCES + "vspId/versionId/"))); - } - - @Test - void testIsEnabled() { - Assertions.assertTrue(testSubject.isEnabled()); - } - - private Attachment mockAttachment(final String fileName, final URL fileToUpload) throws IOException { - final Attachment attachment = Mockito.mock(Attachment.class); - when(attachment.getContentDisposition()).thenReturn(new ContentDisposition("test")); - final DataHandler dataHandler = Mockito.mock(DataHandler.class); - when(dataHandler.getName()).thenReturn(fileName); - final InputStream inputStream = Mockito.mock(InputStream.class); - when(dataHandler.getInputStream()).thenReturn(inputStream); - when(attachment.getDataHandler()).thenReturn(dataHandler); - byte[] bytes = "upload package Test".getBytes(); - if (Objects.nonNull(fileToUpload)) { - try { - bytes = IOUtils.toByteArray(fileToUpload); - } catch (final IOException e) { - fail("Not able to convert file to byte array"); - } - } - when(attachment.getObject(ArgumentMatchers.any())).thenReturn(bytes); - return attachment; - } - -} diff --git a/common-be/src/test/resources/csarSizeReducer/dummyToReduce-2-files.zip b/common-be/src/test/resources/csarSizeReducer/dummyToReduce-2-files.zip new file mode 100644 index 0000000000..be48e8a674 Binary files /dev/null and b/common-be/src/test/resources/csarSizeReducer/dummyToReduce-2-files.zip differ diff --git a/common-be/src/test/resources/csarSizeReducer/dummyToReduce-3-files.zip b/common-be/src/test/resources/csarSizeReducer/dummyToReduce-3-files.zip new file mode 100644 index 0000000000..fecb45aaaf Binary files /dev/null and b/common-be/src/test/resources/csarSizeReducer/dummyToReduce-3-files.zip differ diff --git a/common-be/src/test/resources/csarSizeReducer/dummyToReduce.zip b/common-be/src/test/resources/csarSizeReducer/dummyToReduce.zip deleted file mode 100644 index fecb45aaaf..0000000000 Binary files a/common-be/src/test/resources/csarSizeReducer/dummyToReduce.zip and /dev/null differ diff --git a/common-be/src/test/resources/s3StoreArtifactStorageManager/dummy.csar b/common-be/src/test/resources/s3StoreArtifactStorageManager/dummy.csar new file mode 100644 index 0000000000..73b28f52fd Binary files /dev/null and b/common-be/src/test/resources/s3StoreArtifactStorageManager/dummy.csar differ -- cgit 1.2.3-korg