summaryrefslogtreecommitdiffstats
path: root/openecomp-be/api
diff options
context:
space:
mode:
authorvasraz <vasyl.razinkov@est.tech>2021-10-08 14:48:08 +0100
committerAndr� Schmid <andre.schmid@est.tech>2021-11-05 20:28:52 +0000
commit707fb6d83819058d5736b2dc38bea3c2d9e07a2d (patch)
treed319572a4514f9bdc86619a53eba22a8c0d0abd0 /openecomp-be/api
parent90ff4221ee13c227cd6b2d4566f785602781f7b0 (diff)
Large csar handling - object store
Change-Id: I4e88bd7bfcc1fdbc93d67da2682f2e873ba243c6 Signed-off-by: Vasyl Razinkov <vasyl.razinkov@est.tech> Issue-ID: SDC-3754
Diffstat (limited to 'openecomp-be/api')
-rw-r--r--openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java87
-rw-r--r--openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/test/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImplTest.java36
2 files changed, 60 insertions, 63 deletions
diff --git a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java
index 23930ed640..6fe7f9dd0a 100644
--- a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java
+++ b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java
@@ -28,13 +28,14 @@ import static javax.ws.rs.core.Response.Status.NOT_FOUND;
import static org.openecomp.core.validation.errors.ErrorMessagesFormatBuilder.getErrorWithParameters;
import static org.openecomp.sdc.common.errors.Messages.ERROR_HAS_OCCURRED_WHILE_PERSISTING_THE_ARTIFACT;
import static org.openecomp.sdc.common.errors.Messages.ERROR_HAS_OCCURRED_WHILE_REDUCING_THE_ARTIFACT_SIZE;
-import static org.openecomp.sdc.common.errors.Messages.EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING_FULL_PATH;
import static org.openecomp.sdc.common.errors.Messages.NO_FILE_WAS_UPLOADED_OR_FILE_NOT_EXIST;
import static org.openecomp.sdc.common.errors.Messages.PACKAGE_PROCESS_ERROR;
import static org.openecomp.sdc.common.errors.Messages.UNEXPECTED_PROBLEM_HAPPENED_WHILE_GETTING;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
@@ -42,8 +43,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.UUID;
+import javax.activation.DataHandler;
import javax.inject.Named;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.tuple.Pair;
@@ -55,13 +56,8 @@ import org.openecomp.sdc.activitylog.dao.type.ActivityType;
import org.openecomp.sdc.be.csar.storage.ArtifactInfo;
import org.openecomp.sdc.be.csar.storage.ArtifactStorageConfig;
import org.openecomp.sdc.be.csar.storage.ArtifactStorageManager;
-import org.openecomp.sdc.be.csar.storage.CsarPackageReducerConfiguration;
-import org.openecomp.sdc.be.csar.storage.CsarSizeReducer;
import org.openecomp.sdc.be.csar.storage.PackageSizeReducer;
-import org.openecomp.sdc.be.csar.storage.PersistentVolumeArtifactStorageConfig;
-import org.openecomp.sdc.be.csar.storage.PersistentVolumeArtifactStorageManager;
-import org.openecomp.sdc.be.exception.BusinessException;
-import org.openecomp.sdc.common.CommonConfigurationManager;
+import org.openecomp.sdc.be.csar.storage.StorageFactory;
import org.openecomp.sdc.common.util.ValidationUtils;
import org.openecomp.sdc.common.utils.SdcCommon;
import org.openecomp.sdc.datatypes.error.ErrorLevel;
@@ -98,7 +94,6 @@ import org.springframework.stereotype.Service;
public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplateCandidate {
private static final Logger LOGGER = LoggerFactory.getLogger(OrchestrationTemplateCandidateImpl.class);
- private static final String EXTERNAL_CSAR_STORE = "externalCsarStore";
private final OrchestrationTemplateCandidateManager candidateManager;
private final VendorSoftwareProductManager vendorSoftwareProductManager;
private final ActivityLogManager activityLogManager;
@@ -110,9 +105,10 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate
this.vendorSoftwareProductManager = VspManagerFactory.getInstance().createInterface();
this.activityLogManager = ActivityLogManagerFactory.getInstance().createInterface();
LOGGER.info("Instantiating artifactStorageManager");
- this.artifactStorageManager = new PersistentVolumeArtifactStorageManager(readArtifactStorageConfiguration());
+ final StorageFactory storageFactory = new StorageFactory();
+ this.artifactStorageManager = storageFactory.createArtifactStorageManager();
LOGGER.info("Instantiating packageSizeReducer");
- this.packageSizeReducer = new CsarSizeReducer(readPackageReducerConfiguration());
+ this.packageSizeReducer = storageFactory.createPackageSizeReducer().orElse(null);
}
// Constructor used in test to avoid mock static
@@ -128,56 +124,45 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate
this.packageSizeReducer = packageSizeReducer;
}
- private CsarPackageReducerConfiguration readPackageReducerConfiguration() {
- final var commonConfigurationManager = CommonConfigurationManager.getInstance();
- final List<String> foldersToStrip = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "foldersToStrip", new ArrayList<>());
- final int sizeLimit = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "sizeLimit", 1000000);
- final int thresholdEntries = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "thresholdEntries", 10000);
- LOGGER.info("Folders to strip: '{}'", String.join(", ", foldersToStrip));
- final Set<Path> foldersToStripPathSet = foldersToStrip.stream().map(Path::of).collect(Collectors.toSet());
- return new CsarPackageReducerConfiguration(foldersToStripPathSet, sizeLimit, thresholdEntries);
- }
-
- private ArtifactStorageConfig readArtifactStorageConfiguration() {
- final var commonConfigurationManager = CommonConfigurationManager.getInstance();
- final boolean isEnabled = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "storeCsarsExternally", false);
- LOGGER.info("ArtifactConfig.isEnabled: '{}'", isEnabled);
- final String storagePathString = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "fullPath", null);
- LOGGER.info("ArtifactConfig.storagePath: '{}'", storagePathString);
- if (isEnabled && storagePathString == null) {
- throw new OrchestrationTemplateCandidateException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING_FULL_PATH.getErrorMessage());
- }
- final var storagePath = storagePathString == null ? null : Path.of(storagePathString);
- return new PersistentVolumeArtifactStorageConfig(isEnabled, storagePath);
- }
-
@Override
public Response upload(String vspId, String versionId, final Attachment fileToUpload, final String user) {
vspId = ValidationUtils.sanitizeInputString(vspId);
versionId = ValidationUtils.sanitizeInputString(versionId);
final byte[] fileToUploadBytes;
- final var filename = ValidationUtils.sanitizeInputString(fileToUpload.getDataHandler().getName());
+ final DataHandler dataHandler = fileToUpload.getDataHandler();
+ final var filename = ValidationUtils.sanitizeInputString(dataHandler.getName());
ArtifactInfo artifactInfo = null;
if (artifactStorageManager.isEnabled()) {
- final InputStream packageInputStream;
+ final Path tempArtifactPath;
try {
- packageInputStream = fileToUpload.getDataHandler().getInputStream();
- } catch (final IOException e) {
+ final ArtifactStorageConfig storageConfiguration = artifactStorageManager.getStorageConfiguration();
+
+ final Path folder = Path.of(storageConfiguration.getTempPath()).resolve(vspId).resolve(versionId);
+ tempArtifactPath = folder.resolve(UUID.randomUUID().toString());
+ Files.createDirectories(folder);
+ try (final InputStream packageInputStream = dataHandler.getInputStream();
+ final var fileOutputStream = new FileOutputStream(tempArtifactPath.toFile())) {
+ packageInputStream.transferTo(fileOutputStream);
+ }
+ } catch (final Exception e) {
return Response.status(INTERNAL_SERVER_ERROR).entity(buildUploadResponseWithError(
new ErrorMessage(ErrorLevel.ERROR, UNEXPECTED_PROBLEM_HAPPENED_WHILE_GETTING.formatMessage(filename)))).build();
}
- try {
- artifactInfo = artifactStorageManager.upload(vspId, versionId, packageInputStream);
- } catch (final BusinessException e) {
+ try (final InputStream inputStream = Files.newInputStream(tempArtifactPath)) {
+ artifactInfo = artifactStorageManager.upload(vspId, versionId, inputStream);
+ } catch (final Exception e) {
+ LOGGER.error("Package Size Reducer not configured", e);
return Response.status(INTERNAL_SERVER_ERROR).entity(buildUploadResponseWithError(
new ErrorMessage(ErrorLevel.ERROR, ERROR_HAS_OCCURRED_WHILE_PERSISTING_THE_ARTIFACT.formatMessage(filename)))).build();
}
try {
- fileToUploadBytes = packageSizeReducer.reduce(artifactInfo.getPath());
- } catch (final BusinessException e) {
+ fileToUploadBytes = packageSizeReducer.reduce(tempArtifactPath);
+ Files.delete(tempArtifactPath);
+ } catch (final Exception e) {
+ LOGGER.error("Package Size Reducer not configured", e);
return Response.status(INTERNAL_SERVER_ERROR).entity(buildUploadResponseWithError(
- new ErrorMessage(ErrorLevel.ERROR, ERROR_HAS_OCCURRED_WHILE_REDUCING_THE_ARTIFACT_SIZE.formatMessage(artifactInfo.getPath()))))
- .build();
+ new ErrorMessage(ErrorLevel.ERROR,
+ ERROR_HAS_OCCURRED_WHILE_REDUCING_THE_ARTIFACT_SIZE.formatMessage(tempArtifactPath.toString())))).build();
}
} else {
fileToUploadBytes = fileToUpload.getObject(byte[].class);
@@ -192,12 +177,16 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate
if (onboardPackageInfo == null) {
final UploadFileResponseDto uploadFileResponseDto = buildUploadResponseWithError(
new ErrorMessage(ErrorLevel.ERROR, PACKAGE_PROCESS_ERROR.formatMessage(filename)));
- return Response.ok(uploadFileResponseDto)
- .build();
+ return Response.ok(uploadFileResponseDto).build();
}
final var version = new Version(versionId);
final var vspDetails = vendorSoftwareProductManager.getVsp(vspId, version);
- return processOnboardPackage(onboardPackageInfo, vspDetails, errorMessages);
+ final Response response = processOnboardPackage(onboardPackageInfo, vspDetails, errorMessages);
+ final UploadFileResponseDto entity = (UploadFileResponseDto) response.getEntity();
+ if (artifactStorageManager.isEnabled() && !entity.getErrors().isEmpty()) {
+ artifactStorageManager.delete(artifactInfo);
+ }
+ return response;
}
private Response processOnboardPackage(final OnboardPackageInfo onboardPackageInfo, final VspDetails vspDetails,
diff --git a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/test/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImplTest.java b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/test/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImplTest.java
index edf29b75c5..2d2c30865a 100644
--- a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/test/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImplTest.java
+++ b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/test/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImplTest.java
@@ -32,6 +32,8 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -49,14 +51,18 @@ import org.apache.cxf.jaxrs.ext.multipart.ContentDisposition;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum;
import org.openecomp.sdc.activitylog.ActivityLogManager;
import org.openecomp.sdc.be.csar.storage.ArtifactStorageManager;
+import org.openecomp.sdc.be.csar.storage.MinIoArtifactInfo;
+import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig;
+import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.Credentials;
+import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.EndPoint;
import org.openecomp.sdc.be.csar.storage.PackageSizeReducer;
-import org.openecomp.sdc.be.csar.storage.PersistentStorageArtifactInfo;
import org.openecomp.sdc.logging.api.Logger;
import org.openecomp.sdc.logging.api.LoggerFactory;
import org.openecomp.sdc.vendorsoftwareproduct.OrchestrationTemplateCandidateManager;
@@ -88,6 +94,7 @@ class OrchestrationTemplateCandidateImplTest {
private ArtifactStorageManager artifactStorageManager;
@Mock
private PackageSizeReducer packageSizeReducer;
+ @InjectMocks
private OrchestrationTemplateCandidateImpl orchestrationTemplateCandidate;
@BeforeEach
@@ -132,20 +139,15 @@ class OrchestrationTemplateCandidateImplTest {
ArgumentMatchers.eq(candidateId),
ArgumentMatchers.any())).thenReturn(Optional.of(fds));
- orchestrationTemplateCandidate =
- new OrchestrationTemplateCandidateImpl(candidateManager, vendorSoftwareProductManager, activityLogManager,
- artifactStorageManager, packageSizeReducer);
-
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@Test
- void uploadSignedTest() {
+ void uploadSignedTest() throws IOException {
Response response = orchestrationTemplateCandidate
- .upload("1", "1", mockAttachment("filename.zip", this.getClass().getResource("/files/sample-signed.zip")),
- "1");
+ .upload("1", "1", mockAttachment("filename.zip", this.getClass().getResource("/files/sample-signed.zip")), user);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
assertTrue(((UploadFileResponseDto) response.getEntity()).getErrors().isEmpty());
}
@@ -153,7 +155,7 @@ class OrchestrationTemplateCandidateImplTest {
@Test
void uploadNotSignedTest() throws IOException {
Response response = orchestrationTemplateCandidate.upload("1", "1",
- mockAttachment("filename.csar", this.getClass().getResource("/files/sample-not-signed.csar")), "1");
+ mockAttachment("filename.csar", this.getClass().getResource("/files/sample-not-signed.csar")), user);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
assertTrue(((UploadFileResponseDto) response.getEntity()).getErrors().isEmpty());
}
@@ -161,23 +163,29 @@ class OrchestrationTemplateCandidateImplTest {
@Test
void uploadNotSignedArtifactStorageManagerIsEnabledTest() throws IOException {
when(artifactStorageManager.isEnabled()).thenReturn(true);
+ when(artifactStorageManager.getStorageConfiguration()).thenReturn(
+ new MinIoStorageArtifactStorageConfig(true, new EndPoint("host", 9000, false), new Credentials("accessKey", "secretKey"), "tempPath"));
+
final Path path = Path.of("src/test/resources/files/sample-not-signed.csar");
- when(artifactStorageManager.upload(anyString(), anyString(), any())).thenReturn(new PersistentStorageArtifactInfo(path));
+ when(artifactStorageManager.upload(anyString(), anyString(), any())).thenReturn(new MinIoArtifactInfo("vspId", "name"));
final byte[] bytes = Files.readAllBytes(path);
when(packageSizeReducer.reduce(any())).thenReturn(bytes);
Response response = orchestrationTemplateCandidate.upload("1", "1",
- mockAttachment("filename.csar", this.getClass().getResource("/files/sample-not-signed.csar")), "1");
+ mockAttachment("filename.csar", this.getClass().getResource("/files/sample-not-signed.csar")), user);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
assertTrue(((UploadFileResponseDto) response.getEntity()).getErrors().isEmpty());
}
- private Attachment mockAttachment(final String fileName, final URL fileToUpload) {
+ private Attachment mockAttachment(final String fileName, final URL fileToUpload) throws IOException {
final Attachment attachment = Mockito.mock(Attachment.class);
+ final InputStream inputStream = Mockito.mock(InputStream.class);
when(attachment.getContentDisposition()).thenReturn(new ContentDisposition("test"));
final DataHandler dataHandler = Mockito.mock(DataHandler.class);
when(dataHandler.getName()).thenReturn(fileName);
when(attachment.getDataHandler()).thenReturn(dataHandler);
+ when(dataHandler.getInputStream()).thenReturn(inputStream);
+ when(inputStream.transferTo(any(OutputStream.class))).thenReturn(0L);
byte[] bytes = "upload package Test".getBytes();
if (Objects.nonNull(fileToUpload)) {
try {
@@ -192,9 +200,9 @@ class OrchestrationTemplateCandidateImplTest {
}
@Test
- void uploadSignNotValidTest() {
+ void uploadSignNotValidTest() throws IOException {
Response response = orchestrationTemplateCandidate
- .upload("1", "1", mockAttachment("filename.zip", null), "1");
+ .upload("1", "1", mockAttachment("filename.zip", null), user);
assertEquals(Status.NOT_ACCEPTABLE.getStatusCode(), response.getStatus());
assertFalse(((UploadFileResponseDto) response.getEntity()).getErrors().isEmpty());
}