summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichaelMorris <michael.morris@est.tech>2021-11-11 10:35:02 +0000
committerMichael Morris <michael.morris@est.tech>2021-11-11 15:32:01 +0000
commitda996fd16101fba9eb739adb49679b4b1e3ee70a (patch)
tree5dce59effb1cb6af05292fb75d4f0b60f3a5c7d9
parent0286f013bcf29c206a1a178f82d5478067807750 (diff)
Fix stability test issue
Failures in stability tests are observed on attempts to get distribution status from SDC backend. These errors are caused by timeouts when reading from the DB (5 sec) and also the reads exceeding the tombstone threshold (100,000). The fix implemented in this change is two fold: 1. Stop writing null values in empty columns in DB writes (a tombstone is created for each null column value) 2. Change the query to the DB to remove the unnecessary filtering which greatly increases the time taken for the read. Failures in stability tests are also observed due to failures in parsing YAML files from VSP. The files are parsed using snakeYAML. The class used is not thread safe however it is possible for it to be invoked in multiple threads. This is most likely the cause of the parsing failures The fix implemented here includes changes to ensure an instance of the Yaml class is not used across multiple threads Signed-off-by: MichaelMorris <michael.morris@est.tech> Issue-ID: SDC-3773 Change-Id: I4067ff0032bb67ab114db2f05accce5d7b42d188
-rw-r--r--catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/AuditAccessor.java2
-rw-r--r--catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/CassandraClient.java2
-rw-r--r--common-app-api/src/main/java/org/openecomp/sdc/common/util/YamlToObjectConverter.java40
3 files changed, 19 insertions, 25 deletions
diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/AuditAccessor.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/AuditAccessor.java
index 482ece7f10..ad9d01e77d 100644
--- a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/AuditAccessor.java
+++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/AuditAccessor.java
@@ -32,7 +32,7 @@ import org.openecomp.sdc.be.resources.data.auditing.ResourceAdminEvent;
public interface AuditAccessor {
// ***** distributionstatusevent table
- @Query("SELECT * FROM sdcaudit.distributionstatusevent WHERE DID = :did AND ACTION = 'DStatus' ALLOW FILTERING")
+ @Query("SELECT * FROM sdcaudit.distributionstatusevent WHERE DID = :did")
Result<DistributionStatusEvent> getListOfDistributionStatuses(@Param("did") String did);
// ***** resourceadminevent table
diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/CassandraClient.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/CassandraClient.java
index 25a6b49bba..3bd318c7b8 100644
--- a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/CassandraClient.java
+++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/CassandraClient.java
@@ -154,7 +154,7 @@ public class CassandraClient {
}
try {
Mapper<T> mapper = manager.mapper(clazz);
- mapper.save(entity);
+ mapper.save(entity, Mapper.Option.saveNullFields(false));
} catch (Exception e) {
logger.error(EcompLoggerErrorCode.DATA_ERROR, CassandraClient.class.getName(), "Failed to save entity [{}], error :", entity, e);
return CassandraOperationStatus.GENERAL_ERROR;
diff --git a/common-app-api/src/main/java/org/openecomp/sdc/common/util/YamlToObjectConverter.java b/common-app-api/src/main/java/org/openecomp/sdc/common/util/YamlToObjectConverter.java
index 2475b77705..b1f68c9b82 100644
--- a/common-app-api/src/main/java/org/openecomp/sdc/common/util/YamlToObjectConverter.java
+++ b/common-app-api/src/main/java/org/openecomp/sdc/common/util/YamlToObjectConverter.java
@@ -48,17 +48,16 @@ import org.openecomp.sdc.exception.YamlConversionException;
import org.openecomp.sdc.fe.config.Configuration.FeMonitoringConfig;
import org.yaml.snakeyaml.TypeDescription;
import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.Constructor;
import org.yaml.snakeyaml.introspector.PropertyUtils;
public class YamlToObjectConverter {
private static Logger log = Logger.getLogger(YamlToObjectConverter.class.getName());
- private static HashMap<String, Yaml> yamls = new HashMap<>();
- private static Yaml defaultYaml = new Yaml();
+ private static HashMap<String, Constructor> yamlConstructors = new HashMap<>();
static {
- org.yaml.snakeyaml.constructor.Constructor deConstructor = new org.yaml.snakeyaml.constructor.Constructor(
- DistributionEngineConfiguration.class);
+ Constructor deConstructor = new Constructor(DistributionEngineConfiguration.class);
TypeDescription deDescription = new TypeDescription(DistributionEngineConfiguration.class);
deDescription.putListPropertyType("distributionStatusTopic", DistributionStatusTopicConfig.class);
deDescription.putListPropertyType("distribNotifServiceArtifactTypes", ComponentArtifactTypesConfig.class);
@@ -66,18 +65,15 @@ public class YamlToObjectConverter {
deDescription.putListPropertyType("createTopic", CreateTopicConfig.class);
deDescription.putListPropertyType("distributionNotificationTopic", DistributionNotificationTopicConfig.class);
deConstructor.addTypeDescription(deDescription);
- Yaml yaml = new Yaml(deConstructor);
- yamls.put(DistributionEngineConfiguration.class.getName(), yaml);
+ yamlConstructors.put(DistributionEngineConfiguration.class.getName(), deConstructor);
// FE conf
- org.yaml.snakeyaml.constructor.Constructor feConfConstructor = new org.yaml.snakeyaml.constructor.Constructor(
- org.openecomp.sdc.fe.config.Configuration.class);
+ Constructor feConfConstructor = new Constructor(org.openecomp.sdc.fe.config.Configuration.class);
TypeDescription feConfDescription = new TypeDescription(org.openecomp.sdc.fe.config.Configuration.class);
feConfDescription.putListPropertyType("systemMonitoring", FeMonitoringConfig.class);
feConfConstructor.addTypeDescription(feConfDescription);
- yamls.put(org.openecomp.sdc.fe.config.Configuration.class.getName(), new Yaml(feConfConstructor));
+ yamlConstructors.put(org.openecomp.sdc.fe.config.Configuration.class.getName(), feConfConstructor);
// BE conf
- org.yaml.snakeyaml.constructor.Constructor beConfConstructor = new org.yaml.snakeyaml.constructor.Constructor(
- org.openecomp.sdc.be.config.Configuration.class);
+ Constructor beConfConstructor = new Constructor(org.openecomp.sdc.be.config.Configuration.class);
TypeDescription beConfDescription = new TypeDescription(org.openecomp.sdc.be.config.Configuration.class);
beConfConstructor.addTypeDescription(beConfDescription);
// systemMonitoring
@@ -97,23 +93,19 @@ public class YamlToObjectConverter {
beConfDescription.putListPropertyType("applicationL2Cache", ApplicationL2CacheConfig.class);
// tosca validators config
beConfDescription.putListPropertyType("toscaValidators", ToscaValidatorsConfig.class);
- yamls.put(org.openecomp.sdc.be.config.Configuration.class.getName(), new Yaml(beConfConstructor));
+ yamlConstructors.put(org.openecomp.sdc.be.config.Configuration.class.getName(), beConfConstructor);
// HEAT deployment artifact
- org.yaml.snakeyaml.constructor.Constructor depArtHeatConstructor = new org.yaml.snakeyaml.constructor.Constructor(
- DeploymentArtifactHeatConfiguration.class);
+ Constructor depArtHeatConstructor = new Constructor(DeploymentArtifactHeatConfiguration.class);
PropertyUtils propertyUtils = new PropertyUtils();
// Skip properties which are found in YAML but not found in POJO
propertyUtils.setSkipMissingProperties(true);
depArtHeatConstructor.setPropertyUtils(propertyUtils);
- yamls.put(org.openecomp.sdc.be.config.validation.DeploymentArtifactHeatConfiguration.class.getName(), new Yaml(depArtHeatConstructor));
+ yamlConstructors.put(org.openecomp.sdc.be.config.validation.DeploymentArtifactHeatConfiguration.class.getName(), depArtHeatConstructor);
}
private static <T> Yaml getYamlByClassName(Class<T> className) {
- Yaml yaml = yamls.get(className.getName());
- if (yaml == null) {
- yaml = defaultYaml;
- }
- return yaml;
+ Constructor yamlConstructor = yamlConstructors.get(className.getName());
+ return yamlConstructor == null ? new Yaml() : new Yaml(yamlConstructor);
}
public <T> T convert(final String dirPath, final Class<T> className, final String configFileName) throws YamlConversionException {
@@ -164,13 +156,15 @@ public class YamlToObjectConverter {
@SuppressWarnings("unchecked")
public boolean isValidYaml(byte[] fileContents) {
try {
- Iterable<Object> mappedToscaTemplateIt = defaultYaml.loadAll(new ByteArrayInputStream(fileContents));
+ Iterable<Object> mappedToscaTemplateIt = new Yaml().loadAll(new ByteArrayInputStream(fileContents));
for (Object o : mappedToscaTemplateIt) {
- System.out.println("Loaded object type:" + o.getClass());
+ log.debug("Loaded object type:" + o.getClass());
Map<String, Object> map = (Map<String, Object>) o;
}
} catch (Exception e) {
- log.error(EcompLoggerErrorCode.UNKNOWN_ERROR, "", "", "Failed to convert yaml file to object - yaml is invalid", e);
+ log.error(EcompLoggerErrorCode.UNKNOWN_ERROR, "", "", "Failed to convert yaml file to object - yaml is invalid. Exception: {}, message: {}",
+ e.getClass().getName(), e.getMessage());
+ log.error("Failed to convert yaml file to object - yaml is invalid", e);
return false;
}
return true;