From 36ff777984fbd728737b264d7aa3933794716519 Mon Sep 17 00:00:00 2001 From: vasraz Date: Thu, 29 Jul 2021 14:41:18 +0100 Subject: Implement 'Signed Large CSAR' support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I33cc381b86c6a10e20d521c0d3dcc76c28344b8f Signed-off-by: Vasyl Razinkov Issue-ID: SDC-3652 Issue-ID: SDC-3653 Signed-off-by: André Schmid --- .../sdc/be/csar/storage/CsarSizeReducer.java | 153 ++++++++++++++++++--- .../sdc/be/csar/storage/CsarSizeReducerTest.java | 55 ++++++-- .../src/test/resources/csarSizeReducer/dummy.csar | Bin 25876 -> 0 bytes .../csarSizeReducer/dummyToNotReduce.csar | Bin 0 -> 24301 bytes .../resources/csarSizeReducer/dummyToReduce.csar | Bin 0 -> 25876 bytes .../resources/csarSizeReducer/dummyToReduce.zip | Bin 0 -> 23905 bytes 6 files changed, 177 insertions(+), 31 deletions(-) delete mode 100644 common-be/src/test/resources/csarSizeReducer/dummy.csar create mode 100644 common-be/src/test/resources/csarSizeReducer/dummyToNotReduce.csar create mode 100644 common-be/src/test/resources/csarSizeReducer/dummyToReduce.csar create mode 100644 common-be/src/test/resources/csarSizeReducer/dummyToReduce.zip (limited to 'common-be') diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java index cf35c8c4d7..1fef373362 100644 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducer.java @@ -20,14 +20,24 @@ package org.openecomp.sdc.be.csar.storage; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + import java.io.BufferedOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; +import lombok.Getter; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FilenameUtils; import org.openecomp.sdc.be.csar.storage.exception.CsarSizeReducerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +45,12 @@ import org.slf4j.LoggerFactory; public class CsarSizeReducer implements PackageSizeReducer { private static final Logger LOGGER = LoggerFactory.getLogger(CsarSizeReducer.class); + private static final Set ALLOWED_SIGNATURE_EXTENSIONS = Set.of("cms"); + private static final Set ALLOWED_CERTIFICATE_EXTENSIONS = Set.of("cert", "crt"); + private static final String CSAR_EXTENSION = "csar"; + private static final String UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR = "An unexpected problem happened while reading the CSAR '%s'"; + @Getter + private final AtomicBoolean reduced = new AtomicBoolean(false); private final CsarPackageReducerConfiguration configuration; @@ -44,38 +60,31 @@ public class CsarSizeReducer implements PackageSizeReducer { @Override public byte[] reduce(final Path csarPackagePath) { + if (hasSignedPackageStructure(csarPackagePath)) { + return reduce(csarPackagePath, this::signedZipProcessingConsumer); + } else { + return reduce(csarPackagePath, this::unsignedZipProcessingConsumer); + } + } + + private byte[] reduce(final Path csarPackagePath, final ZipProcessFunction zipProcessingFunction) { final var reducedCsarPath = Path.of(csarPackagePath + "." + UUID.randomUUID()); try (final var zf = new ZipFile(csarPackagePath.toString()); final var zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(reducedCsarPath)))) { - - zf.entries().asIterator().forEachRemaining(entry -> { - final var entryName = entry.getName(); - try { - if (!entry.isDirectory()) { - zos.putNextEntry(new ZipEntry(entryName)); - if (isCandidateToRemove(entry)) { - // replace with EMPTY string to avoid package description inconsistency/validation errors - zos.write("".getBytes()); - } else { - zos.write(zf.getInputStream(entry).readAllBytes()); - } - } - zos.closeEntry(); - } catch (final IOException ei) { - final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); - throw new CsarSizeReducerException(errorMsg, ei); - } - }); - + zf.entries().asIterator().forEachRemaining(zipProcessingFunction.getProcessZipConsumer(csarPackagePath, zf, zos)); } catch (final IOException ex1) { rollback(reducedCsarPath); - final var errorMsg = String.format("An unexpected problem happened while reading the CSAR '%s'", csarPackagePath); + final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath); throw new CsarSizeReducerException(errorMsg, ex1); } final byte[] reducedCsarBytes; try { - reducedCsarBytes = Files.readAllBytes(reducedCsarPath); + if (reduced.get()) { + reducedCsarBytes = Files.readAllBytes(reducedCsarPath); + } else { + reducedCsarBytes = Files.readAllBytes(csarPackagePath); + } } catch (final IOException e) { final var errorMsg = String.format("Could not read bytes of file '%s'", csarPackagePath); throw new CsarSizeReducerException(errorMsg, e); @@ -90,6 +99,51 @@ public class CsarSizeReducer implements PackageSizeReducer { return reducedCsarBytes; } + private Consumer signedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) { + return zipEntry -> { + final var entryName = zipEntry.getName(); + try { + zos.putNextEntry(new ZipEntry(entryName)); + if (!zipEntry.isDirectory()) { + if (entryName.toLowerCase().endsWith(CSAR_EXTENSION)) { + final var internalCsarExtractPath = Path.of(csarPackagePath + "." + UUID.randomUUID()); + Files.copy(zf.getInputStream(zipEntry), internalCsarExtractPath, REPLACE_EXISTING); + zos.write(reduce(internalCsarExtractPath, this::unsignedZipProcessingConsumer)); + Files.delete(internalCsarExtractPath); + } else { + zos.write(zf.getInputStream(zipEntry).readAllBytes()); + } + } + zos.closeEntry(); + } catch (final IOException ei) { + final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); + throw new CsarSizeReducerException(errorMsg, ei); + } + }; + } + + private Consumer unsignedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) { + return zipEntry -> { + final var entryName = zipEntry.getName(); + try { + zos.putNextEntry(new ZipEntry(entryName)); + if (!zipEntry.isDirectory()) { + if (isCandidateToRemove(zipEntry)) { + // replace with EMPTY string to avoid package description inconsistency/validation errors + zos.write("".getBytes()); + reduced.set(true); + } else { + zos.write(zf.getInputStream(zipEntry).readAllBytes()); + } + } + zos.closeEntry(); + } catch (final IOException ei) { + final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath); + throw new CsarSizeReducerException(errorMsg, ei); + } + }; + } + private void rollback(final Path reducedCsarPath) { if (Files.exists(reducedCsarPath)) { try { @@ -106,4 +160,59 @@ public class CsarSizeReducer implements PackageSizeReducer { || zipEntry.getSize() > configuration.getSizeLimit(); } + private boolean hasSignedPackageStructure(final Path csarPackagePath) { + final List packagePathList; + try (final var zf = new ZipFile(csarPackagePath.toString())) { + packagePathList = zf.stream() + .filter(zipEntry -> !zipEntry.isDirectory()) + .map(ZipEntry::getName).map(Path::of) + .collect(Collectors.toList()); + } catch (final IOException e) { + final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath); + throw new CsarSizeReducerException(errorMsg, e); + } + + if (CollectionUtils.isEmpty(packagePathList)) { + return false; + } + final int numberOfFiles = packagePathList.size(); + if (numberOfFiles == 2) { + return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList); + } + if (numberOfFiles == 3) { + return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList) && hasOneCertificateFile(packagePathList); + } + return false; + } + + private boolean hasOneInternalPackageFile(final List packagePathList) { + return packagePathList.parallelStream() + .map(Path::toString) + .map(FilenameUtils::getExtension) + .map(String::toLowerCase) + .filter(extension -> extension.endsWith(CSAR_EXTENSION)).count() == 1; + } + + private boolean hasOneSignatureFile(final List packagePathList) { + return packagePathList.parallelStream() + .map(Path::toString) + .map(FilenameUtils::getExtension) + .map(String::toLowerCase) + .filter(ALLOWED_SIGNATURE_EXTENSIONS::contains).count() == 1; + } + + private boolean hasOneCertificateFile(final List packagePathList) { + return packagePathList.parallelStream() + .map(Path::toString) + .map(FilenameUtils::getExtension) + .map(String::toLowerCase) + .filter(ALLOWED_CERTIFICATE_EXTENSIONS::contains).count() == 1; + } + + @FunctionalInterface + private interface ZipProcessFunction { + + Consumer getProcessZipConsumer(Path csarPackagePath, ZipFile zf, ZipOutputStream zos); + } + } diff --git a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducerTest.java b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducerTest.java index c7586446f7..eaa5ffeda2 100644 --- a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducerTest.java +++ b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/CsarSizeReducerTest.java @@ -23,7 +23,9 @@ package org.openecomp.sdc.be.csar.storage; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.when; import java.nio.charset.StandardCharsets; @@ -32,7 +34,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -51,15 +54,16 @@ class CsarSizeReducerTest { MockitoAnnotations.openMocks(this); } - @Test - void reduceByPathAndSizeTest() throws ZipException { + @ParameterizedTest + @ValueSource(strings = {"dummyToReduce.zip", "dummyToReduce.csar", "dummyToNotReduce.csar"}) + void reduceByPathAndSizeTest(String fileName) throws ZipException { final var pathToReduce1 = Path.of("Files/images"); final var pathToReduce2 = Path.of("Files/Scripts/my_script.sh"); final var sizeLimit = 150000L; when(csarPackageReducerConfiguration.getSizeLimit()).thenReturn(sizeLimit); when(csarPackageReducerConfiguration.getFoldersToStrip()).thenReturn(Set.of(pathToReduce1, pathToReduce2)); - final var csarPath = Path.of("src/test/resources/csarSizeReducer/dummy.csar"); + final var csarPath = Path.of("src/test/resources/csarSizeReducer/" + fileName); final Map originalCsar = ZipUtils.readZip(csarPath.toFile(), false); @@ -74,14 +78,47 @@ class CsarSizeReducerTest { assertTrue(reducedCsar.containsKey(originalFilePath), String.format("No file should be removed, but it is missing original file '%s'", originalFilePath)); - if (originalFilePath.startsWith(pathToReduce1.toString()) || originalFilePath.startsWith(pathToReduce2.toString()) - || originalBytes.length > sizeLimit) { - assertArrayEquals("".getBytes(StandardCharsets.UTF_8), reducedCsar.get(originalFilePath), - String.format("File '%s' expected to be reduced to empty string", originalFilePath)); + final String extention = fileName.substring(fileName.lastIndexOf('.') + 1); + switch (extention.toLowerCase()) { + case "zip": + verifyZIP(pathToReduce1, pathToReduce2, sizeLimit, reducedCsar, originalFilePath, originalBytes); + break; + case "csar": + verifyCSAR(pathToReduce1, pathToReduce2, sizeLimit, reducedCsar, originalFilePath, originalBytes); + break; + default: + fail("Unexpected file extention"); + break; + } + } + } + + private void verifyCSAR(final Path pathToReduce1, final Path pathToReduce2, final long sizeLimit, final Map reducedCsar, + final String originalFilePath, final byte[] originalBytes) { + if (originalFilePath.startsWith(pathToReduce1.toString()) || originalFilePath.startsWith(pathToReduce2.toString()) + || originalBytes.length > sizeLimit) { + assertArrayEquals("".getBytes(StandardCharsets.UTF_8), reducedCsar.get(originalFilePath), + String.format("File '%s' expected to be reduced to empty string", originalFilePath)); + } else { + assertArrayEquals(originalBytes, reducedCsar.get(originalFilePath), + String.format("File '%s' expected to be equal", originalFilePath)); + } + } + + private void verifyZIP(final Path pathToReduce1, final Path pathToReduce2, final long sizeLimit, final Map reducedCsar, + final String originalFilePath, final byte[] originalBytes) { + if (originalFilePath.startsWith(pathToReduce1.toString()) || originalFilePath.startsWith(pathToReduce2.toString()) + || originalBytes.length > sizeLimit) { + assertArrayEquals("".getBytes(StandardCharsets.UTF_8), reducedCsar.get(originalFilePath), + String.format("File '%s' expected to be reduced to empty string", originalFilePath)); + } else { + if (originalFilePath.endsWith(".csar") && csarSizeReducer.getReduced().get()) { + assertNotEquals(originalBytes.length, reducedCsar.get(originalFilePath).length, + String.format("File '%s' expected to be NOT equal", originalFilePath)); } else { assertArrayEquals(originalBytes, reducedCsar.get(originalFilePath), String.format("File '%s' expected to be equal", originalFilePath)); } } } -} \ No newline at end of file +} diff --git a/common-be/src/test/resources/csarSizeReducer/dummy.csar b/common-be/src/test/resources/csarSizeReducer/dummy.csar deleted file mode 100644 index 73b28f52fd..0000000000 Binary files a/common-be/src/test/resources/csarSizeReducer/dummy.csar and /dev/null differ diff --git a/common-be/src/test/resources/csarSizeReducer/dummyToNotReduce.csar b/common-be/src/test/resources/csarSizeReducer/dummyToNotReduce.csar new file mode 100644 index 0000000000..d44041382e Binary files /dev/null and b/common-be/src/test/resources/csarSizeReducer/dummyToNotReduce.csar differ diff --git a/common-be/src/test/resources/csarSizeReducer/dummyToReduce.csar b/common-be/src/test/resources/csarSizeReducer/dummyToReduce.csar new file mode 100644 index 0000000000..73b28f52fd Binary files /dev/null and b/common-be/src/test/resources/csarSizeReducer/dummyToReduce.csar differ diff --git a/common-be/src/test/resources/csarSizeReducer/dummyToReduce.zip b/common-be/src/test/resources/csarSizeReducer/dummyToReduce.zip new file mode 100644 index 0000000000..fecb45aaaf Binary files /dev/null and b/common-be/src/test/resources/csarSizeReducer/dummyToReduce.zip differ -- cgit 1.2.3-korg