aboutsummaryrefslogtreecommitdiffstats
path: root/common-be
diff options
context:
space:
mode:
authorvasraz <vasyl.razinkov@est.tech>2022-02-26 17:22:58 +0000
committerVasyl Razinkov <vasyl.razinkov@est.tech>2022-02-26 18:25:12 +0000
commit4b978c9cc115c4f4032d5f3dbc4d3cde002449cc (patch)
treeb3986d499eb989c5210a25522258c6678b691dd3 /common-be
parentd0c2403f1f7088d60b135976c40917302daf8b9e (diff)
Implement improved MinIo client
Change-Id: Ic9abd6b0bdaa17e9deff2279a64416d81f7ad606 Signed-off-by: Vasyl Razinkov <vasyl.razinkov@est.tech> Issue-ID: SDC-3886
Diffstat (limited to 'common-be')
-rw-r--r--common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java1
-rw-r--r--common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java140
-rw-r--r--common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java1
-rw-r--r--common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java6
4 files changed, 46 insertions, 102 deletions
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java
index 6f6778f2b0..cfbbc4fe4e 100644
--- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageConfig.java
@@ -31,6 +31,7 @@ public class MinIoStorageArtifactStorageConfig implements ArtifactStorageConfig
private final EndPoint endPoint;
private final Credentials credentials;
private final String tempPath;
+ private final int uploadPartSize;
@AllArgsConstructor
@Getter
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java
index 54a71d9b2a..7b9cf087a6 100644
--- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManager.java
@@ -23,18 +23,14 @@ package org.openecomp.sdc.be.csar.storage;
import static org.openecomp.sdc.common.errors.Messages.EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING;
import io.minio.BucketExistsArgs;
-import io.minio.CopyObjectArgs;
-import io.minio.CopySource;
import io.minio.GetObjectArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
-import io.minio.MinioClient.Builder;
import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs;
+import io.minio.StatObjectArgs;
import java.io.InputStream;
import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
import lombok.Getter;
import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.Credentials;
import org.openecomp.sdc.be.csar.storage.MinIoStorageArtifactStorageConfig.EndPoint;
@@ -46,33 +42,38 @@ import org.slf4j.LoggerFactory;
public class MinIoStorageArtifactStorageManager implements ArtifactStorageManager {
private static final Logger LOGGER = LoggerFactory.getLogger(MinIoStorageArtifactStorageManager.class);
+ private static final String ENDPOINT = "endpoint";
+ private static final String CREDENTIALS = "credentials";
+ private static final String TEMP_PATH = "tempPath";
private static final String EXTERNAL_CSAR_STORE = "externalCsarStore";
-
@Getter
private final MinIoStorageArtifactStorageConfig storageConfiguration;
private final MinioClient minioClient;
public MinIoStorageArtifactStorageManager() {
this.storageConfiguration = readMinIoStorageArtifactStorageConfig();
- minioClient = initMinioClient();
+ this.minioClient = initMinioClient();
}
//for testing only
MinIoStorageArtifactStorageManager(final ArtifactStorageConfig storageConfiguration) {
this.storageConfiguration = (MinIoStorageArtifactStorageConfig) storageConfiguration;
- minioClient = initMinioClient();
+ this.minioClient = initMinioClient();
}
@Override
public ArtifactInfo persist(final String vspId, final String versionId, final ArtifactInfo uploadedArtifactInfo) {
final MinIoArtifactInfo minioObjectTemp = (MinIoArtifactInfo) uploadedArtifactInfo;
+ LOGGER.debug("PERSIST - bucket: '{}', object: '{}'", minioObjectTemp.getBucket(), minioObjectTemp.getObjectName());
try {
- minioClient.getObject(
- GetObjectArgs.builder()
+ // Get information of an object.
+ minioClient.statObject(
+ StatObjectArgs.builder()
.bucket(minioObjectTemp.getBucket())
.object(minioObjectTemp.getObjectName())
.build()
);
+
} catch (final Exception e) {
LOGGER.error("Failed to retrieve uploaded artifact with bucket '{}' and name '{}' while persisting", minioObjectTemp.getBucket(),
minioObjectTemp.getObjectName(), e);
@@ -80,26 +81,12 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage
String.format("Failed to retrieve uploaded artifact with bucket '%s' and name '%s' while persisting",
minioObjectTemp.getBucket(), minioObjectTemp.getObjectName()), e);
}
-
- final var backupPath = backupPreviousVersion(vspId, versionId).orElse(null);
- try {
- moveFile(minioObjectTemp, vspId, versionId);
- } catch (final Exception e) {
- rollback(minioObjectTemp, vspId, versionId);
- LOGGER.error("Could not persist artifact for bucket '{}', object '{}'", vspId, versionId, e);
- final var errorMsg = String.format("Could not persist artifact for VSP '%s', version '%s'", vspId, versionId);
- throw new ArtifactStorageException(errorMsg, e);
- }
-
- removePreviousVersion(backupPath);
-
return new MinIoArtifactInfo(vspId, versionId);
}
@Override
public ArtifactInfo upload(final String vspId, final String versionId, final InputStream fileToUpload) {
- final String name = versionId + "--" + UUID.randomUUID();
try {
// Make bucket if not exist.
final boolean found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(vspId).build());
@@ -111,30 +98,32 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage
LOGGER.info("Bucket '{}' already exists.", vspId);
}
- put(vspId, name, fileToUpload);
+ put(vspId, versionId, fileToUpload);
} catch (final Exception e) {
- LOGGER.error("Failed to upload artifact - bucket: '{}', object: '{}'", vspId, name, e);
+ LOGGER.error("Failed to upload artifact - bucket: '{}', object: '{}'", vspId, versionId, e);
throw new ArtifactStorageException("Failed to upload artifact", e);
}
- return new MinIoArtifactInfo(vspId, name);
+ return new MinIoArtifactInfo(vspId, versionId);
}
@Override
public void put(final String vspId, final String name, final InputStream fileToUpload) {
+ LOGGER.debug("BEGIN -> PUT - bucket: '{}', object: '{}'", vspId, name);
try {
minioClient.putObject(
PutObjectArgs.builder()
.bucket(vspId)
.object(name)
- .stream(fileToUpload, fileToUpload.available(), -1)
+ .stream(fileToUpload, -1, storageConfiguration.getUploadPartSize())
.build()
);
} catch (final Exception e) {
LOGGER.error("Failed to put - bucket: '{}', object: '{}'", vspId, name, e);
throw new ArtifactStorageException("Failed to upload artifact", e);
}
+ LOGGER.debug("SUCCESS -> PUT - bucket: '{}', object: '{}'", vspId, name);
}
@Override
@@ -155,6 +144,7 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage
@Override
public InputStream get(final String bucketID, final String objectID) {
+ LOGGER.debug("GET - bucket: '{}', object: '{}'", bucketID, objectID);
try {
return minioClient.getObject(GetObjectArgs.builder()
.bucket(bucketID)
@@ -169,6 +159,7 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage
@Override
public void delete(final ArtifactInfo artifactInfo) {
final MinIoArtifactInfo minioObject = (MinIoArtifactInfo) artifactInfo;
+ LOGGER.debug("DELETE - bucket: '{}', object: '{}'", minioObject.getBucket(), minioObject.getObjectName());
try {
minioClient.removeObject(RemoveObjectArgs.builder()
.bucket(minioObject.getBucket())
@@ -182,97 +173,48 @@ public class MinIoStorageArtifactStorageManager implements ArtifactStorageManage
}
- private Optional<MinIoArtifactInfo> backupPreviousVersion(final String vspId, final String versionId) {
-
- final String tempName = versionId + "--" + UUID.randomUUID().toString();
- try {
- copy(vspId, tempName, versionId);
- } catch (final Exception e) {
- LOGGER.error("Failed to copy - bucket: '{}', object: '{}'", vspId, versionId, e);
- return Optional.empty();
- }
-
- return Optional.of(new MinIoArtifactInfo(vspId, tempName));
- }
-
- private void rollback(final MinIoArtifactInfo minioObject, final String vspId, final String versionId) {
- try {
- moveFile(minioObject, vspId, versionId);
- } catch (final Exception ex) {
- LOGGER.warn("Could not rollback the backup '{}' to the original '{}'", versionId, minioObject.getObjectName(), ex);
- }
- }
-
- private void removePreviousVersion(final MinIoArtifactInfo minioObject) {
- if (minioObject == null) {
- return;
- }
- delete(minioObject);
- }
-
- private void moveFile(final MinIoArtifactInfo minioObject, final String vspId, final String versionId) {
- try {
- copy(vspId, versionId, minioObject.getObjectName());
- } catch (final Exception e) {
- LOGGER.error("Failed to copy - bucket: '{}', object: '{}'", vspId, versionId, e);
- throw new ArtifactStorageException("Failed to move", e);
- }
- delete(minioObject);
- }
-
- private void copy(final String vspId, final String versionId, final String objectName) throws Exception {
- minioClient.copyObject(
- CopyObjectArgs.builder()
- .bucket(vspId)
- .object(versionId)
- .source(CopySource.builder()
- .bucket(vspId)
- .object(objectName)
- .build())
- .build());
- }
-
private MinIoStorageArtifactStorageConfig readMinIoStorageArtifactStorageConfig() {
final var commonConfigurationManager = CommonConfigurationManager.getInstance();
-
- final Map<String, Object> endpoint = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "endpoint", null);
- final Map<String, Object> credentials = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "credentials", null);
- final String tempPath = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "tempPath", null);
+ final Map<String, Object> endpoint = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, ENDPOINT, null);
+ final Map<String, Object> creds = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, CREDENTIALS, null);
+ final String tempPath = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, TEMP_PATH, null);
+ final int uploadPartSize = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "uploadPartSize", 50_000_000);
if (endpoint == null) {
- LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("endpoint"));
- throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("endpoint"));
+ LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(ENDPOINT));
+ throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(ENDPOINT));
}
- if (credentials == null) {
- LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("credentials"));
- throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("credentials"));
+ if (creds == null) {
+ LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(CREDENTIALS));
+ throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(CREDENTIALS));
}
if (tempPath == null) {
- LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("tempPath"));
- throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage("tempPath"));
+ LOGGER.error(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(TEMP_PATH));
+ throw new ArtifactStorageException(EXTERNAL_CSAR_STORE_CONFIGURATION_FAILURE_MISSING.formatMessage(TEMP_PATH));
}
LOGGER.info("ArtifactConfig.endpoint: '{}'", endpoint);
- LOGGER.info("ArtifactConfig.credentials: '{}'", credentials);
+ LOGGER.info("ArtifactConfig.credentials: '{}'", creds);
LOGGER.info("ArtifactConfig.tempPath: '{}'", tempPath);
+ LOGGER.info("ArtifactConfig.uploadPartSize: '{}'", uploadPartSize);
final String host = (String) endpoint.getOrDefault("host", null);
final int port = (int) endpoint.getOrDefault("port", 0);
final boolean secure = (boolean) endpoint.getOrDefault("secure", false);
- final String accessKey = (String) credentials.getOrDefault("accessKey", null);
- final String secretKey = (String) credentials.getOrDefault("secretKey", null);
+ final String accessKey = (String) creds.getOrDefault("accessKey", null);
+ final String secretKey = (String) creds.getOrDefault("secretKey", null);
- return new MinIoStorageArtifactStorageConfig(true, new EndPoint(host, port, secure), new Credentials(accessKey, secretKey), tempPath);
+ return new MinIoStorageArtifactStorageConfig
+ (true, new EndPoint(host, port, secure), new Credentials(accessKey, secretKey), tempPath, uploadPartSize);
}
private MinioClient initMinioClient() {
- final EndPoint endPoint = storageConfiguration.getEndPoint();
- final Credentials credentials = storageConfiguration.getCredentials();
+ final EndPoint storageConfigurationEndPoint = storageConfiguration.getEndPoint();
+ final Credentials storageConfigurationCredentials = storageConfiguration.getCredentials();
- final Builder builder = MinioClient.builder();
- return builder
- .endpoint(endPoint.getHost(), endPoint.getPort(), endPoint.isSecure())
- .credentials(credentials.getAccessKey(), credentials.getSecretKey())
+ return MinioClient.builder()
+ .endpoint(storageConfigurationEndPoint.getHost(), storageConfigurationEndPoint.getPort(), storageConfigurationEndPoint.isSecure())
+ .credentials(storageConfigurationCredentials.getAccessKey(), storageConfigurationCredentials.getSecretKey())
.build();
}
diff --git a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java
index 2be7b10118..8c86917e01 100644
--- a/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java
+++ b/common-be/src/main/java/org/openecomp/sdc/be/csar/storage/StorageFactory.java
@@ -60,6 +60,7 @@ public class StorageFactory {
private StorageType getConfiguredArtifactStorageType() {
final var commonConfigurationManager = CommonConfigurationManager.getInstance();
+ commonConfigurationManager.reload();
final String storageType = commonConfigurationManager.getConfigValue(EXTERNAL_CSAR_STORE, "storageType", NONE.name());
LOGGER.info("ArtifactConfig.storageType: '{}'", storageType);
return findByName(storageType);
diff --git a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java
index 41eed0cebb..fa577913fa 100644
--- a/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java
+++ b/common-be/src/test/java/org/openecomp/sdc/be/csar/storage/MinIoStorageArtifactStorageManagerTest.java
@@ -72,8 +72,8 @@ class MinIoStorageArtifactStorageManagerTest {
.build()
).thenReturn(minioClient);
- testSubject = new MinIoStorageArtifactStorageManager(
- new MinIoStorageArtifactStorageConfig(true, new EndPoint("host", 9000, false), new Credentials("accessKey", "secretKey"), ""));
+ testSubject = new MinIoStorageArtifactStorageManager(new MinIoStorageArtifactStorageConfig
+ (true, new EndPoint("host", 9000, false), new Credentials("accessKey", "secretKey"), "", 10_000_000));
}
}
@@ -91,7 +91,7 @@ class MinIoStorageArtifactStorageManagerTest {
Assertions.assertNotNull(result);
Assertions.assertTrue(result instanceof MinIoArtifactInfo);
Assertions.assertEquals(VSP_ID, ((MinIoArtifactInfo) result).getBucket());
- Assertions.assertTrue(((MinIoArtifactInfo) result).getObjectName().startsWith(VERSION_ID + "--"));
+ Assertions.assertTrue(((MinIoArtifactInfo) result).getObjectName().startsWith(VERSION_ID));
}
@Test