diff options
16 files changed, 91 insertions, 122 deletions
diff --git a/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb b/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb index 5a025548f7..0ec99e74c7 100644 --- a/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb +++ b/catalog-be/src/main/docker/backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb @@ -1276,3 +1276,4 @@ externalCsarStore: accessKey: "login" secretKey: "password" tempPath: "/home/onap/temp/" + uploadPartSize: 200000000 diff --git a/catalog-fe/sdc-frontend/chef-repo/cookbooks/sdc-catalog-fe/templates/default/FE-configuration.yaml.erb b/catalog-fe/sdc-frontend/chef-repo/cookbooks/sdc-catalog-fe/templates/default/FE-configuration.yaml.erb index 4b371959d5..8dc3c51df3 100644 --- a/catalog-fe/sdc-frontend/chef-repo/cookbooks/sdc-catalog-fe/templates/default/FE-configuration.yaml.erb +++ b/catalog-fe/sdc-frontend/chef-repo/cookbooks/sdc-catalog-fe/templates/default/FE-configuration.yaml.erb @@ -24,7 +24,7 @@ beSslPort: <%= @ssl_port %> threadpoolSize: 50 # request processing timeout (seconds) -requestTimeout: 10 +requestTimeout: 1200 # catalog ms (the host-port values need to be changed once it is deployed) catalogFacadeMs: diff --git a/catalog-fe/src/main/resources/config/configuration.yaml b/catalog-fe/src/main/resources/config/configuration.yaml index 2eff183e1a..f5d2e66f65 100644 --- a/catalog-fe/src/main/resources/config/configuration.yaml +++ b/catalog-fe/src/main/resources/config/configuration.yaml @@ -20,7 +20,7 @@ beSslPort: 8443 threadpoolSize: 50 # request processing timeout (seconds) -requestTimeout: 10 +requestTimeout: 1200 # catalog ms (the host-port values need to be changed once it is deployed) catalogFacadeMs: diff --git a/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java b/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java index 5d9f74005b..69e863b4fd 100644 --- a/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java +++ b/common-app-api/src/main/java/org/openecomp/sdc/be/config/Configuration.java @@ -495,6 +495,7 @@ public class Configuration extends BasicConfiguration { private Endpoint endpoint; private Credentials credentials; private String tempPath; + private int uploadPartSize; @Getter @Setter diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java index 6f6778f2b0..cfbbc4fe4e 100644 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java @@ -31,6 +31,7 @@ public class MinIoStorageArtifactStorageConfig implements ArtifactStorageConfig private final EndPoint endPoint; private final Credentials credentials; private final String tempPath; + private final int uploadPartSize; @AllArgsConstructor @Getter diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java index 54a71d9b2a..7b9cf087a6 100644 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java @@ -23,18 +23,14 @@ package org.openecomp.sdc.be.csar.storage; import static org.openecomp.sdc.common.errors.Messages.EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING; import io.minio.BucketExistsArgs; -import io.minio.CopyObjectArgs; -import io.minio.CopySource; import io.minio.GetObjectArgs; import io.minio.MakeBucketArgs; import io.minio.MinioClient; -import io.minio.MinioClient.Builder; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.StatObjectArgs; import java.io.InputStream; import java.util.Map; -import java.util.Optional; -import java.util.UUID; import lombok.Getter; import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.Credentials; import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.EndPoint; @@ -46,33 +42,38 @@ import org.slf4j.LoggerFactory; public class MinIoStorageArtifactStorageManager implements ArtifactStorageManager { private static final Logger LOGGER = LoggerFactory.getLogger(MinIoStorageArtifactStorageManager.class); + private static final String ENDPOINT = "endpoint"; + private static final String CREDENTIALS = "credentials"; + private static final String TEMP_PATH = "tempPath"; private static final String EXTERNAL_CSAR_STORE = "externalCsarStore"; - @Getter private final MinIoStorageArtifactStorageConfig storageConfiguration; private final MinioClient minioClient; public MinIoStorageArtifactStorageManager() { this.storageConfiguration = readMinIoStorageArtifactStorageConfig(); - minioClient = initMinioClient(); + this.minioClient = initMinioClient(); } //for testing only MinIoStorageArtifactStorageManager(final ArtifactStorageConfig storageConfiguration) { this.storageConfiguration = (MinIoStorageArtifactStorageConfig) storageConfiguration; - minioClient = initMinioClient(); + this.minioClient = initMinioClient(); } @Override public ArtifactInfo persist(final String vspId, final String versionId, final ArtifactInfo uploadedArtifactInfo) { final MinIoArtifactInfo minioObjectTemp = (MinIoArtifactInfo) uploadedArtifactInfo; + LOGGER.debug("PERSIST - bucket: '{}', object: '{}'", minioObjectTemp.getBucket(), minioObjectTemp.getObjectName()); try { - minioClient.getObject( - GetObjectArgs.builder() + // Get information of an object. + minioClient.statObject( + StatObjectArgs.builder() .bucket(minioObjectTemp.getBucket()) .object(minioObjectTemp.getObjectName()) .build() ); + } catch (final Exception e) { LOGGER.error("Failed to retrieve uploaded artifact with bucket '{}' and name '{}' while persisting", minioObjectTemp.getBucket(), minioObjectTemp.getObjectName(), e); @@ -80,26 +81,12 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage String.format("Failed to retrieve uploaded artifact with bucket '%s' and name '%s' while persisting", minioObjectTemp.getBucket(), minioObjectTemp.getObjectName()), e); } - - final var backupPath = backupPreviousVersion(vspId, versionId).orElse(null); - try { - moveFile(minioObjectTemp, vspId, versionId); - } catch (final Exception e) { - rollback(minioObjectTemp, vspId, versionId); - LOGGER.error("Could not persist artifact for bucket '{}', object '{}'", vspId, versionId, e); - final var errorMsg = String.format("Could not persist artifact for VSP '%s', version '%s'", vspId, versionId); - throw new ArtifactStorageException(errorMsg, e); - } - - removePreviousVersion(backupPath); - return new MinIoArtifactInfo(vspId, versionId); } @Override public ArtifactInfo upload(final String vspId, final String versionId, final InputStream fileToUpload) { - final String name = versionId + "--" + UUID.randomUUID(); try { // Make bucket if not exist. final boolean found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(vspId).build()); @@ -111,30 +98,32 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage LOGGER.info("Bucket '{}' already exists.", vspId); } - put(vspId, name, fileToUpload); + put(vspId, versionId, fileToUpload); } catch (final Exception e) { - LOGGER.error("Failed to upload artifact - bucket: '{}', object: '{}'", vspId, name, e); + LOGGER.error("Failed to upload artifact - bucket: '{}', object: '{}'", vspId, versionId, e); throw new ArtifactStorageException("Failed to upload artifact", e); } - return new MinIoArtifactInfo(vspId, name); + return new MinIoArtifactInfo(vspId, versionId); } @Override public void put(final String vspId, final String name, final InputStream fileToUpload) { + LOGGER.debug("BEGIN -> PUT - bucket: '{}', object: '{}'", vspId, name); try { minioClient.putObject( PutObjectArgs.builder() .bucket(vspId) .object(name) - .stream(fileToUpload, fileToUpload.available(), -1) + .stream(fileToUpload, -1, storageConfiguration.getUploadPartSize()) .build() ); } catch (final Exception e) { LOGGER.error("Failed to put - bucket: '{}', object: '{}'", vspId, name, e); throw new ArtifactStorageException("Failed to upload artifact", e); } + LOGGER.debug("SUCCESS -> PUT - bucket: '{}', object: '{}'", vspId, name); } @Override @@ -155,6 +144,7 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage @Override public InputStream get(final String bucketID, final String objectID) { + LOGGER.debug("GET - bucket: '{}', object: '{}'", bucketID, objectID); try { return minioClient.getObject(GetObjectArgs.builder() .bucket(bucketID) @@ -169,6 +159,7 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage @Override public void delete(final ArtifactInfo artifactInfo) { final MinIoArtifactInfo minioObject = (MinIoArtifactInfo) artifactInfo; + LOGGER.debug("DELETE - bucket: '{}', object: '{}'", minioObject.getBucket(), minioObject.getObjectName()); try { minioClient.removeObject(RemoveObjectArgs.builder() .bucket(minioObject.getBucket()) @@ -182,97 +173,48 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage } - private Optional<MinIoArtifactInfo> backupPreviousVersion(final String vspId, final String versionId) { - - final String tempName = versionId + "--" + UUID.randomUUID().toString(); - try { - copy(vspId, tempName, versionId); - } catch (final Exception e) { - LOGGER.error("Failed to copy - bucket: '{}', object: '{}'", vspId, versionId, e); - return Optional.empty(); - } - - return Optional.of(new MinIoArtifactInfo(vspId, tempName)); - } - - private void rollback(final MinIoArtifactInfo minioObject, final String vspId, final String versionId) { - try { - moveFile(minioObject, vspId, versionId); - } catch (final Exception ex) { - LOGGER.warn("Could not rollback the backup '{}' to the original '{}'", versionId, minioObject.getObjectName(), ex); - } - } - - private void removePreviousVersion(final MinIoArtifactInfo minioObject) { - if (minioObject == null) { - return; - } - delete(minioObject); - } - - private void moveFile(final MinIoArtifactInfo minioObject, final String vspId, final String versionId) { - try { - copy(vspId, versionId, minioObject.getObjectName()); - } catch (final Exception e) { - LOGGER.error("Failed to copy - bucket: '{}', object: '{}'", vspId, versionId, e); - throw new ArtifactStorageException("Failed to move", e); - } - delete(minioObject); - } - - private void copy(final String vspId, final String versionId, final String objectName) throws Exception { - minioClient.copyObject( - CopyObjectArgs.builder() - .bucket(vspId) - .object(versionId) - .source(CopySource.builder() - .bucket(vspId) - .object(objectName) - .build()) - .build()); - } - private MinIoStorageArtifactStorageConfig readMinIoStorageArtifactStorageConfig() { final var commonConfigurationManager = CommonConfigurationManager.getInstance(); - - final Map<String, Object> endpoint = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "endpoint", null); - final Map<String, Object> credentials = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "credentials", null); - final String tempPath = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "tempPath", null); + final Map<String, Object> endpoint = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, ENDPOINT, null); + final Map<String, Object> creds = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, CREDENTIALS, null); + final String tempPath = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, TEMP_PATH, null); + final int uploadPartSize = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "uploadPartSize", 50_000_000); if (endpoint == null) { - LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("endpoint")); - throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("endpoint")); + LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(ENDPOINT)); + throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(ENDPOINT)); } - if (credentials == null) { - LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("credentials")); - throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("credentials")); + if (creds == null) { + LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(CREDENTIALS)); + throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(CREDENTIALS)); } if (tempPath == null) { - LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("tempPath")); - throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("tempPath")); + LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(TEMP_PATH)); + throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(TEMP_PATH)); } LOGGER.info("ArtifactConfig.endpoint: '{}'", endpoint); - LOGGER.info("ArtifactConfig.credentials: '{}'", credentials); + LOGGER.info("ArtifactConfig.credentials: '{}'", creds); LOGGER.info("ArtifactConfig.tempPath: '{}'", tempPath); + LOGGER.info("ArtifactConfig.uploadPartSize: '{}'", uploadPartSize); final String host = (String) endpoint.getOrDefault("host", null); final int port = (int) endpoint.getOrDefault("port", 0); final boolean secure = (boolean) endpoint.getOrDefault("secure", false); - final String accessKey = (String) credentials.getOrDefault("accessKey", null); - final String secretKey = (String) credentials.getOrDefault("secretKey", null); + final String accessKey = (String) creds.getOrDefault("accessKey", null); + final String secretKey = (String) creds.getOrDefault("secretKey", null); - return new MinIoStorageArtifactStorageConfig(true, new EndPoint(host, port, secure), new Credentials(accessKey, secretKey), tempPath); + return new MinIoStorageArtifactStorageConfig + (true, new EndPoint(host, port, secure), new Credentials(accessKey, secretKey), tempPath, uploadPartSize); } private MinioClient initMinioClient() { - final EndPoint endPoint = storageConfiguration.getEndPoint(); - final Credentials credentials = storageConfiguration.getCredentials(); + final EndPoint storageConfigurationEndPoint = storageConfiguration.getEndPoint(); + final Credentials storageConfigurationCredentials = storageConfiguration.getCredentials(); - final Builder builder = MinioClient.builder(); - return builder - .endpoint(endPoint.getHost(), endPoint.getPort(), endPoint.isSecure()) - .credentials(credentials.getAccessKey(), credentials.getSecretKey()) + return MinioClient.builder() + .endpoint(storageConfigurationEndPoint.getHost(), storageConfigurationEndPoint.getPort(), storageConfigurationEndPoint.isSecure()) + .credentials(storageConfigurationCredentials.getAccessKey(), storageConfigurationCredentials.getSecretKey()) .build(); } diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java index 2be7b10118..8c86917e01 100644 --- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java +++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java @@ -60,6 +60,7 @@ public class StorageFactory { private StorageType getConfiguredArtifactStorageType() { final var commonConfigurationManager = CommonConfigurationManager.getInstance(); + commonConfigurationManager.reload(); final String storageType = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "storageType", NONE.name()); LOGGER.info("ArtifactConfig.storageType: '{}'", storageType); return findByName(storageType); diff --git a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java index 41eed0cebb..fa577913fa 100644 --- a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java +++ b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java @@ -72,8 +72,8 @@ class MinIoStorageArtifactStorageManagerTest { .build() ).thenReturn(minioClient); - testSubject = new MinIoStorageArtifactStorageManager( - new MinIoStorageArtifactStorageConfig(true, new EndPoint("host", 9000, false), new Credentials("accessKey", "secretKey"), "")); + testSubject = new MinIoStorageArtifactStorageManager(new MinIoStorageArtifactStorageConfig + (true, new EndPoint("host", 9000, false), new Credentials("accessKey", "secretKey"), "", 10_000_000)); } } @@ -91,7 +91,7 @@ class MinIoStorageArtifactStorageManagerTest { Assertions.assertNotNull(result); Assertions.assertTrue(result instanceof MinIoArtifactInfo); Assertions.assertEquals(VSP_ID, ((MinIoArtifactInfo) result).getBucket()); - Assertions.assertTrue(((MinIoArtifactInfo) result).getObjectName().startsWith(VERSION_ID + "--")); + Assertions.assertTrue(((MinIoArtifactInfo) result).getObjectName().startsWith(VERSION_ID)); } @Test diff --git a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java index b51583980d..01d2a59e45 100644 --- a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java +++ b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/main/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImpl.java @@ -32,6 +32,7 @@ import static org.openecomp.sdc.common.errors.Messages.PACKAGE_PROCESS_ERROR; import static org.openecomp.sdc.common.errors.Messages.UNEXPECTED_PROBLEM_HAPPENED_WHILE_GETTING; import java.io.ByteArrayInputStream; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -102,6 +103,7 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate private final VendorSoftwareProductManager vendorSoftwareProductManager; private final ActivityLogManager activityLogManager; private final ArtifactStorageManager artifactStorageManager; + private final StorageFactory storageFactory; private final PackageSizeReducer packageSizeReducer; private final OrchestrationTemplateCandidateUploadManager orchestrationTemplateCandidateUploadManager; @@ -111,7 +113,7 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate this.vendorSoftwareProductManager = VspManagerFactory.getInstance().createInterface(); this.activityLogManager = ActivityLogManagerFactory.getInstance().createInterface(); LOGGER.info("Instantiating artifactStorageManager"); - final StorageFactory storageFactory = new StorageFactory(); + this.storageFactory = new StorageFactory(); this.artifactStorageManager = storageFactory.createArtifactStorageManager(); LOGGER.info("Instantiating packageSizeReducer"); this.packageSizeReducer = storageFactory.createPackageSizeReducer().orElse(null); @@ -129,12 +131,14 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate this.vendorSoftwareProductManager = vendorSoftwareProductManager; this.activityLogManager = activityLogManager; this.artifactStorageManager = artifactStorageManager; + this.storageFactory = new StorageFactory(); this.packageSizeReducer = packageSizeReducer; this.orchestrationTemplateCandidateUploadManager = orchestrationTemplateCandidateUploadManager; } @Override public Response upload(String vspId, String versionId, final Attachment fileToUpload, final String user) { + LOGGER.debug("STARTED -> OrchestrationTemplateCandidateImpl.upload"); vspId = ValidationUtils.sanitizeInputString(vspId); versionId = ValidationUtils.sanitizeInputString(versionId); final Response response; @@ -145,6 +149,7 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate final DataHandler dataHandler = fileToUpload.getDataHandler(); final var filename = ValidationUtils.sanitizeInputString(dataHandler.getName()); ArtifactInfo artifactInfo = null; + final ArtifactStorageManager artifactStorageManager = storageFactory.createArtifactStorageManager(); if (artifactStorageManager.isEnabled()) { artifactInfo = handleArtifactStorage(vspId, versionId, filename, dataHandler); fileToUploadBytes = artifactInfo.getBytes(); @@ -190,6 +195,7 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate } throw ex; } + LOGGER.debug("FINISHED -> OrchestrationTemplateCandidateImpl.upload"); return response; } @@ -202,22 +208,26 @@ public class OrchestrationTemplateCandidateImpl implements OrchestrationTemplate final Path folder = Path.of(storageConfiguration.getTempPath()).resolve(vspId).resolve(versionId); tempArtifactPath = folder.resolve(UUID.randomUUID().toString()); Files.createDirectories(folder); + LOGGER.debug("STARTED -> Transfer to '{}'", tempArtifactPath.toString()); try (final InputStream packageInputStream = artifactDataHandler.getInputStream(); final var fileOutputStream = new FileOutputStream(tempArtifactPath.toFile())) { packageInputStream.transferTo(fileOutputStream); } + LOGGER.debug("FINISHED -> Transfer to '{}'", tempArtifactPath.toString()); } catch (final Exception e) { throw new ArtifactStorageException(UNEXPECTED_PROBLEM_HAPPENED_WHILE_GETTING.formatMessage(filename)); } final ArtifactInfo artifactInfo; - try (final InputStream inputStream = Files.newInputStream(tempArtifactPath)) { + try (final InputStream inputStream = new FileInputStream(tempArtifactPath.toFile())) { artifactInfo = artifactStorageManager.upload(vspId, versionId, inputStream); } catch (final Exception e) { LOGGER.error("Package Size Reducer not configured", e); throw new ArtifactStorageException(ERROR_HAS_OCCURRED_WHILE_PERSISTING_THE_ARTIFACT.formatMessage(filename)); } try { + LOGGER.debug("STARTED -> reducing '{}'", tempArtifactPath.toString()); artifactInfo.setBytes(packageSizeReducer.reduce(tempArtifactPath)); + LOGGER.debug("FINISHED -> reducing '{}'", tempArtifactPath.toString()); Files.delete(tempArtifactPath); } catch (final Exception e) { LOGGER.error("Package Size Reducer not configured", e); diff --git a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/test/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImplTest.java b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/test/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImplTest.java index 6c21cc471e..802d6d885a 100644 --- a/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/test/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImplTest.java +++ b/openecomp-be/api/openecomp-sdc-rest-webapp/vendor-software-products-rest/vendor-software-products-rest-services/src/test/java/org/openecomp/sdcrests/vsp/rest/services/OrchestrationTemplateCandidateImplTest.java @@ -65,6 +65,7 @@ import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig; import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.Credentials; import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.EndPoint; import org.openecomp.sdc.be.csar.storage.PackageSizeReducer; +import org.openecomp.sdc.be.csar.storage.StorageFactory; import org.openecomp.sdc.logging.api.Logger; import org.openecomp.sdc.logging.api.LoggerFactory; import org.openecomp.sdc.vendorsoftwareproduct.OrchestrationTemplateCandidateManager; @@ -100,6 +101,10 @@ class OrchestrationTemplateCandidateImplTest { private PackageSizeReducer packageSizeReducer; @Mock private OrchestrationTemplateCandidateUploadManager orchestrationTemplateCandidateUploadManager; + @Mock + private StorageFactory storageFactory; + @Mock + private Attachment fileToUpload; @InjectMocks private OrchestrationTemplateCandidateImpl orchestrationTemplateCandidate; @@ -178,9 +183,10 @@ class OrchestrationTemplateCandidateImplTest { @Test void uploadNotSignedArtifactStorageManagerIsEnabledTest() throws IOException { + when(storageFactory.createArtifactStorageManager()).thenReturn(artifactStorageManager); when(artifactStorageManager.isEnabled()).thenReturn(true); - when(artifactStorageManager.getStorageConfiguration()).thenReturn( - new MinIoStorageArtifactStorageConfig(true, new EndPoint("host", 9000, false), new Credentials("accessKey", "secretKey"), "tempPath")); + when(artifactStorageManager.getStorageConfiguration()).thenReturn(new MinIoStorageArtifactStorageConfig + (true, new EndPoint("host", 9000, false), new Credentials("accessKey", "secretKey"), "tempPath", 10_000_000)); final Path path = Path.of("src/test/resources/files/sample-not-signed.csar"); final String vspId = "vspId"; @@ -316,12 +322,10 @@ class OrchestrationTemplateCandidateImplTest { vspUploadStatusDto.setLockId(UUID.randomUUID()); when(orchestrationTemplateCandidateUploadManager.putUploadInProgress(candidateId, versionId, user)).thenReturn(vspUploadStatusDto); final RuntimeException forcedException = new RuntimeException(); - when(artifactStorageManager.isEnabled()).thenThrow(forcedException); - final Attachment mock = Mockito.mock(Attachment.class); - when(mock.getDataHandler()).thenReturn(Mockito.mock(DataHandler.class)); + when(fileToUpload.getDataHandler()).thenThrow(forcedException); //when final RuntimeException actualException = assertThrows(RuntimeException.class, - () -> orchestrationTemplateCandidate.upload(candidateId, versionId, mock, user)); + () -> orchestrationTemplateCandidate.upload(candidateId, versionId, fileToUpload, user)); //then assertEquals(forcedException, actualException); verify(orchestrationTemplateCandidateUploadManager) diff --git a/openecomp-be/backend/openecomp-sdc-vendor-software-product-manager/src/test/java/org/openecomp/sdc/vendorsoftwareproduct/impl/orchestration/csar/validation/CsarSecurityValidatorTest.java b/openecomp-be/backend/openecomp-sdc-vendor-software-product-manager/src/test/java/org/openecomp/sdc/vendorsoftwareproduct/impl/orchestration/csar/validation/CsarSecurityValidatorTest.java index 5f880701f3..25b9ff0f68 100644 --- a/openecomp-be/backend/openecomp-sdc-vendor-software-product-manager/src/test/java/org/openecomp/sdc/vendorsoftwareproduct/impl/orchestration/csar/validation/CsarSecurityValidatorTest.java +++ b/openecomp-be/backend/openecomp-sdc-vendor-software-product-manager/src/test/java/org/openecomp/sdc/vendorsoftwareproduct/impl/orchestration/csar/validation/CsarSecurityValidatorTest.java @@ -150,6 +150,7 @@ class CsarSecurityValidatorTest { when(commonConfigurationManager.getConfigValue("externalCsarStore", "credentials", null)).thenReturn(credentials); when(commonConfigurationManager.getConfigValue("externalCsarStore", "tempPath", null)).thenReturn("cert/2-file-signed-package"); when(commonConfigurationManager.getConfigValue(eq("externalCsarStore"), eq("storageType"), any())).thenReturn(MINIO.name()); + when(commonConfigurationManager.getConfigValue(eq("externalCsarStore"), eq("uploadPartSize"), any())).thenReturn(50_000_000); final byte[] packageBytes = getFileBytesOrFail("signed-package-tampered-data.zip"); diff --git a/openecomp-be/backend/openecomp-sdc-vendor-software-product-manager/src/test/java/org/openecomp/sdc/vendorsoftwareproduct/security/SecurityManagerTest.java b/openecomp-be/backend/openecomp-sdc-vendor-software-product-manager/src/test/java/org/openecomp/sdc/vendorsoftwareproduct/security/SecurityManagerTest.java index afc43967c9..b9f3d8707f 100644 --- a/openecomp-be/backend/openecomp-sdc-vendor-software-product-manager/src/test/java/org/openecomp/sdc/vendorsoftwareproduct/security/SecurityManagerTest.java +++ b/openecomp-be/backend/openecomp-sdc-vendor-software-product-manager/src/test/java/org/openecomp/sdc/vendorsoftwareproduct/security/SecurityManagerTest.java @@ -180,6 +180,7 @@ class SecurityManagerTest { when(commonConfigurationManager.getConfigValue("externalCsarStore", "credentials", null)).thenReturn(credentials); when(commonConfigurationManager.getConfigValue("externalCsarStore", "tempPath", null)).thenReturn("cert/2-file-signed-package"); when(commonConfigurationManager.getConfigValue(eq("externalCsarStore"), eq("storageType"), any())).thenReturn(MINIO.name()); + when(commonConfigurationManager.getConfigValue(eq("externalCsarStore"), eq("uploadPartSize"), any())).thenReturn(50_000_000); prepareCertFiles("/cert/rootCA.cert", cerDirPath + "root.cert"); byte[] fileToUploadBytes = readAllBytes("/cert/2-file-signed-package/2-file-signed-package.zip"); @@ -248,6 +249,7 @@ class SecurityManagerTest { when(commonConfigurationManager.getConfigValue("externalCsarStore", "credentials", null)).thenReturn(credentials); when(commonConfigurationManager.getConfigValue("externalCsarStore", "tempPath", null)).thenReturn("tempPath"); when(commonConfigurationManager.getConfigValue(eq("externalCsarStore"), eq("storageType"), any())).thenReturn(MINIO.name()); + when(commonConfigurationManager.getConfigValue(eq("externalCsarStore"), eq("uploadPartSize"), any())).thenReturn(50_000_000); prepareCertFiles("/cert/rootCA.cert", cerDirPath + "root.cert"); byte[] fileToUploadBytes = readAllBytes("/cert/3-file-signed-package/3-file-signed-package.zip"); diff --git a/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/chef-repo/cookbooks/sdc-onboard-backend/files/default/logback.xml b/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/chef-repo/cookbooks/sdc-onboard-backend/files/default/logback.xml index 34dd8e3e6c..49ae967d7f 100644 --- a/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/chef-repo/cookbooks/sdc-onboard-backend/files/default/logback.xml +++ b/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/chef-repo/cookbooks/sdc-onboard-backend/files/default/logback.xml @@ -211,4 +211,5 @@ </root> <logger name="org.openecomp.sdc" level="INFO"/> + <logger name="org.openecomp.sdcrests" level="INFO"/> </configuration>
\ No newline at end of file diff --git a/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/chef-repo/cookbooks/sdc-onboard-backend/templates/default/configuration.yaml.erb b/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/chef-repo/cookbooks/sdc-onboard-backend/templates/default/configuration.yaml.erb index 3b02114334..ae07857585 100644 --- a/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/chef-repo/cookbooks/sdc-onboard-backend/templates/default/configuration.yaml.erb +++ b/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/chef-repo/cookbooks/sdc-onboard-backend/templates/default/configuration.yaml.erb @@ -68,3 +68,4 @@ externalCsarStore: sizeLimit: 10000000 thresholdEntries: 10000 tempPath: "/home/onap/temp/" + uploadPartSize: 200000000 diff --git a/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/startup.sh b/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/startup.sh index dc68d6f3a5..0829fba3ae 100644 --- a/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/startup.sh +++ b/openecomp-be/dist/sdc-onboard-backend-docker/artifacts/startup.sh @@ -7,6 +7,7 @@ JAVA_OPTIONS="$JAVA_OPTIONS \ -Dlogback.configurationFile=$JETTY_BASE/config/onboarding-be/logback.xml \ -Dconfiguration.yaml=$JETTY_BASE/config/onboarding-be/onboarding_configuration.yaml \ -Dfeatures.properties=$JETTY_BASE/config/onboarding-be/features.properties \ + -XX:+HeapDumpOnOutOfMemoryError \ -Dconfig.location=$JETTY_BASE/config/onboarding-be/." cd $JETTY_BASE diff --git a/openecomp-be/lib/openecomp-common-lib/src/main/java/org/openecomp/sdc/common/CommonConfigurationManager.java b/openecomp-be/lib/openecomp-common-lib/src/main/java/org/openecomp/sdc/common/CommonConfigurationManager.java index 46747fb881..92ace445da 100644 --- a/openecomp-be/lib/openecomp-common-lib/src/main/java/org/openecomp/sdc/common/CommonConfigurationManager.java +++ b/openecomp-be/lib/openecomp-common-lib/src/main/java/org/openecomp/sdc/common/CommonConfigurationManager.java @@ -19,21 +19,21 @@ */ package org.openecomp.sdc.common; -import org.onap.sdc.tosca.services.YamlUtil; -import org.openecomp.sdc.logging.api.Logger; -import org.openecomp.sdc.logging.api.LoggerFactory; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.LinkedHashMap; import java.util.Map; import java.util.function.BiConsumer; +import org.onap.sdc.tosca.services.YamlUtil; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; /** - * This is a common class that can access the config file given in input to the JVM with the parameter - * -Dconfiguration.yaml=file.yaml. + * This is a common class that can access the config file given in input to the JVM with the parameter -Dconfiguration.yaml=file.yaml. */ public class CommonConfigurationManager { + public static final String JVM_PARAM_CONFIGURATION_FILE = "configuration.yaml"; private static final Logger LOGGER = LoggerFactory.getLogger(CommonConfigurationManager.class); private static CommonConfigurationManager singletonInstance; @@ -57,6 +57,10 @@ public class CommonConfigurationManager { return singletonInstance; } + public void reload() { + initConfiguration(); + } + private void initConfiguration() { YamlUtil yamlUtil = new YamlUtil(); readConfigurationFromStream(yamlUtil, (filename, stream) -> { @@ -106,7 +110,7 @@ public class CommonConfigurationManager { Map<String, Object> section = this.configuration.get(yamlSection); if (section == null) { LOGGER.error("Section " + yamlSection + " is missing in configuration file '" + configFilename + - "'. Using defaults"); + "'. Using defaults"); return defaultValue; } Object value = section.get(name); @@ -114,15 +118,14 @@ public class CommonConfigurationManager { return value == null ? defaultValue : (T) value; } catch (ClassCastException e) { LOGGER.warn( - String.format("Failed to read configuration property '%s' as requested type. Using default '%s'", - name, defaultValue), e); + String.format("Failed to read configuration property '%s' as requested type. Using default '%s'", + name, defaultValue), e); return defaultValue; } } /** - * This method can be used to access a specific configuration parameter in the configuration in the - * yamlSection predefined in the constructor. + * This method can be used to access a specific configuration parameter in the configuration in the yamlSection predefined in the constructor. * * @param name The name of the config * @param defaultValue A default value |