diff options
Diffstat (limited to 'common-be/src/main')
7 files changed, 432 insertions, 0 deletions
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarPackageReducerConfiguration.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarPackageReducerConfiguration.java new file mode 100644 index 0000000000..a14222ab17 --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarPackageReducerConfiguration.java @@ -0,0 +1,33 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import java.nio.file.Path; +import java.util.Set; +import lombok.Data; + +@Data +public class CsarPackageReducerConfiguration implements PackageSizeReducerConfig { + + private final Set<Path> foldersToStrip; + private final long sizeLimit; + +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java new file mode 100644 index 0000000000..cf35c8c4d7 --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java @@ -0,0 +1,109 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; +import org.openecomp.sdc.be.csar.storage.exception.CsarSizeReducerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CsarSizeReducer implements PackageSizeReducer { + + private static final Logger LOGGER = LoggerFactory.getLogger(CsarSizeReducer.class); + + private final CsarPackageReducerConfiguration configuration; + + public CsarSizeReducer(final CsarPackageReducerConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public byte[] reduce(final Path csarPackagePath) { + final var reducedCsarPath = Path.of(csarPackagePath + "." + UUID.randomUUID()); + + try (final var zf = new ZipFile(csarPackagePath.toString()); + final var zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(reducedCsarPath)))) { + + zf.entries().asIterator().forEachRemaining(entry -> { + final var entryName = entry.getName(); + try { + if (!entry.isDirectory()) { + zos.putNextEntry(new ZipEntry(entryName)); + if (isCandidateToRemove(entry)) { + // replace with EMPTY string to avoid package description inconsistency/validation errors + zos.write("".getBytes()); + } else { + zos.write(zf.getInputStream(entry).readAllBytes()); + } + } + zos.closeEntry(); + } catch (final IOException ei) { + final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); + throw new CsarSizeReducerException(errorMsg, ei); + } + }); + + } catch (final IOException ex1) { + rollback(reducedCsarPath); + final var errorMsg = String.format("An unexpected problem happened while reading the CSAR '%s'", csarPackagePath); + throw new CsarSizeReducerException(errorMsg, ex1); + } + final byte[] reducedCsarBytes; + try { + reducedCsarBytes = Files.readAllBytes(reducedCsarPath); + } catch (final IOException e) { + final var errorMsg = String.format("Could not read bytes of file '%s'", csarPackagePath); + throw new CsarSizeReducerException(errorMsg, e); + } + try { + Files.delete(reducedCsarPath); + } catch (final IOException e) { + final var errorMsg = String.format("Could not delete temporary file '%s'", reducedCsarPath); + throw new CsarSizeReducerException(errorMsg, e); + } + + return reducedCsarBytes; + } + + private void rollback(final Path reducedCsarPath) { + if (Files.exists(reducedCsarPath)) { + try { + Files.delete(reducedCsarPath); + } catch (final Exception ex2) { + LOGGER.warn("Could not delete temporary file '{}'", reducedCsarPath, ex2); + } + } + } + + private boolean isCandidateToRemove(final ZipEntry zipEntry) { + final String zipEntryName = zipEntry.getName(); + return configuration.getFoldersToStrip().stream().anyMatch(Path.of(zipEntryName)::startsWith) + || zipEntry.getSize() > configuration.getSizeLimit(); + } + +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java new file mode 100644 index 0000000000..0472661fd9 --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentStorageArtifactInfo.java @@ -0,0 +1,33 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import java.nio.file.Path; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +public class PersistentStorageArtifactInfo implements ArtifactInfo { + + @Getter + private final Path path; + +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java new file mode 100644 index 0000000000..d3cd6fb302 --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageConfig.java @@ -0,0 +1,32 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import java.nio.file.Path; +import lombok.Data; + +@Data +public class PersistentVolumeArtifactStorageConfig implements ArtifactStorageConfig { + + private final boolean isEnabled; + private final Path storagePath; + +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java new file mode 100644 index 0000000000..10629b3edb --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/PersistentVolumeArtifactStorageManager.java @@ -0,0 +1,161 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Optional; +import java.util.UUID; +import org.openecomp.sdc.be.csar.storage.exception.PersistentVolumeArtifactStorageException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PersistentVolumeArtifactStorageManager implements ArtifactStorageManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(PersistentVolumeArtifactStorageManager.class); + + private final PersistentVolumeArtifactStorageConfig storageConfiguration; + + public PersistentVolumeArtifactStorageManager(final ArtifactStorageConfig storageConfiguration) { + this.storageConfiguration = (PersistentVolumeArtifactStorageConfig) storageConfiguration; + } + + @Override + public ArtifactInfo persist(final String vspId, final String versionId, final ArtifactInfo uploadedArtifactInfo) { + final var temporaryPath = uploadedArtifactInfo.getPath(); + if (!Files.exists(temporaryPath)) { + throw new PersistentVolumeArtifactStorageException(String.format("Given artifact does not exist '%s'", uploadedArtifactInfo.getPath())); + } + + final var filePath = buildFilePath(vspId, versionId); + final var backupPath = backupPreviousVersion(filePath).orElse(null); + try { + moveFile(temporaryPath, filePath); + } catch (final Exception e) { + rollback(backupPath, filePath); + final var errorMsg = String.format("Could not persist artifact for VSP '%s', version '%s'", vspId, versionId); + throw new PersistentVolumeArtifactStorageException(errorMsg, e); + } + + removePreviousVersion(backupPath); + + return new PersistentStorageArtifactInfo(filePath); + } + + @Override + public ArtifactInfo upload(final String vspId, final String versionId, final InputStream artifactInputStream) { + final var destinationFolder = buildDestinationFolder(vspId, versionId); + try { + Files.createDirectories(destinationFolder); + } catch (final IOException e) { + throw new PersistentVolumeArtifactStorageException(String.format("Could not create directory '%s'", destinationFolder), e); + } + + final var filePath = createTempFilePath(destinationFolder); + try { + persist(artifactInputStream, filePath); + } catch (final IOException e) { + throw new PersistentVolumeArtifactStorageException(String.format("Could not persist artifact '%s'", filePath), e); + } + + return new PersistentStorageArtifactInfo(filePath); + } + + private Path buildFilePath(final String vspId, final String versionId) { + return buildDestinationFolder(vspId, versionId).resolve(versionId); + } + + @Override + public boolean isEnabled() { + return storageConfiguration != null && storageConfiguration.isEnabled(); + } + + private Optional<Path> backupPreviousVersion(final Path filePath) { + if (!Files.exists(filePath)) { + return Optional.empty(); + } + + final var backupPath = Path.of(filePath + UUID.randomUUID().toString()); + moveFile(filePath, backupPath); + return Optional.ofNullable(backupPath); + } + + private void rollback(final Path backupPath, final Path filePath) { + try { + moveFile(backupPath, filePath); + } catch (final Exception ex) { + LOGGER.warn("Could not rollback the backup file '{}' to the original '{}'", backupPath, filePath, ex); + } + } + + private void removePreviousVersion(final Path filePath) { + if (filePath == null || !Files.exists(filePath)) { + return; + } + + try { + Files.delete(filePath); + } catch (final IOException e) { + throw new PersistentVolumeArtifactStorageException(String.format("Could not delete previous version '%s'", filePath), e); + } + } + + private Path createTempFilePath(final Path destinationFolder) { + final var retries = 10; + return createTempFilePath(destinationFolder, retries).orElseThrow(() -> { + throw new PersistentVolumeArtifactStorageException(String.format("Could not generate upload file path after '%s' retries", retries)); + }); + } + + private Optional<Path> createTempFilePath(final Path destinationFolder, int retries) { + for (var i = 0; i < retries; i++) { + final var filePath = destinationFolder.resolve(UUID.randomUUID().toString()); + if (Files.notExists(filePath)) { + return Optional.of(filePath); + } + } + return Optional.empty(); + } + + private Path buildDestinationFolder(final String vspId, final String versionId) { + return storageConfiguration.getStoragePath().resolve(vspId).resolve(versionId); + } + + private void persist(final InputStream artifactInputStream, final Path filePath) throws IOException { + try (final var inputStream = artifactInputStream; + final var fileOutputStream = new FileOutputStream(filePath.toFile());) { + inputStream.transferTo(fileOutputStream); + } + } + + private void moveFile(final Path from, final Path to) { + try { + Files.move(from, to, StandardCopyOption.REPLACE_EXISTING); + } catch (final IOException e) { + throw new PersistentVolumeArtifactStorageException(String.format("Could not move file '%s' to '%s'", from, to), e); + } + } + +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/CsarSizeReducerException.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/CsarSizeReducerException.java new file mode 100644 index 0000000000..f57666ac70 --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/CsarSizeReducerException.java @@ -0,0 +1,30 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage.exception; + +import org.openecomp.sdc.be.exception.BusinessException; + +public class CsarSizeReducerException extends BusinessException { + + public CsarSizeReducerException(final String message, final Throwable cause) { + super(message, cause); + } +} diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java new file mode 100644 index 0000000000..28fff65bb6 --- /dev/null +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/exception/PersistentVolumeArtifactStorageException.java @@ -0,0 +1,34 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.csar.storage.exception; + +import org.openecomp.sdc.be.exception.BusinessException; + +public class PersistentVolumeArtifactStorageException extends BusinessException { + + public PersistentVolumeArtifactStorageException(final String message, final Throwable cause) { + super(message, cause); + } + + public PersistentVolumeArtifactStorageException(final String message) { + super(message); + } +} |