diff options
5 files changed, 171 insertions, 41 deletions
diff --git a/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java b/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java index 4c9a9b15..575f8191 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java @@ -23,12 +23,13 @@ package org.onap.aai.serialization.db; import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import com.google.common.base.CaseFormat; -import com.google.common.collect.Multimap; import org.apache.commons.collections.IteratorUtils; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; -import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree; -import org.apache.tinkerpop.gremlin.structure.*; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.janusgraph.core.SchemaViolationException; import org.javatuples.Triplet; import org.onap.aai.concurrent.AaiCallable; @@ -62,7 +63,6 @@ import org.onap.aai.serialization.db.exceptions.MultipleEdgeRuleFoundException; import org.onap.aai.serialization.db.exceptions.NoEdgeRuleFoundException; import org.onap.aai.serialization.engines.TransactionalGraphEngine; import org.onap.aai.serialization.engines.query.QueryEngine; -import org.onap.aai.serialization.tinkerpop.TreeBackedVertex; import org.onap.aai.setup.SchemaVersion; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConfig; @@ -81,15 +81,15 @@ import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; public class DBSerializer { private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DBSerializer.class); + private static final String IMPLICIT_DELETE = "Implicit DELETE"; + private static final String MISSING_REQUIRED_NODE_PROPERTY = "Vertex missing required aai-node-type property"; private final TransactionalGraphEngine engine; @@ -417,7 +417,57 @@ public class DBSerializer { for (Vertex toBeRemoved : processedVertexes) { dependentVertexes.remove(toBeRemoved); } - this.deleteItemsWithTraversal(dependentVertexes); + + // If the dependent vertices are not empty, then with + // the current behaviour, it should remove the vertices implicitly + // We are updating the code to properly log which call + // is doing this so the SE can work with the clients making the call to + // tell them not to call this API and can hopefully deprecate this + // functionality in the future releases + if(!dependentVertexes.isEmpty()){ + + LoggingContext.responseDescription(IMPLICIT_DELETE); + + // Find all the deletable vertices from the dependent vertices that should be deleted + // So for each of the following dependent vertices, + // we will use the edge properties and do the cascade delete + List<Vertex> impliedDeleteVertices = this.engine.getQueryEngine().findDeletable(dependentVertexes); + int impliedDeleteCount = impliedDeleteVertices.size(); + + LOGGER.warn( + "For the vertex with id {}, doing an implicit delete on update will delete total of {} vertexes", + v.id(), + impliedDeleteCount + ); + + String impliedDeleteLogEnabled = AAIConfig.get(AAIConstants.AAI_IMPLIED_DELETE_LOG_ENABLED, "true"); + + int impliedDeleteLogLimit = AAIConfig.getInt(AAIConstants.AAI_IMPLIED_DELETE_LOG_LIMIT, "-1"); + + if(impliedDeleteLogLimit == -1){ + impliedDeleteLogLimit = Integer.MAX_VALUE; + } + + // If the logging is enabled for implied delete + // then log the payload in the latest format + if("true".equals(impliedDeleteLogEnabled) && + impliedDeleteCount <= impliedDeleteLogLimit){ + for(Vertex vertex : impliedDeleteVertices){ + Introspector introspector = null; + try { + introspector = getLatestVersionView(vertex); + if(LOGGER.isInfoEnabled()){ + LOGGER.info("Implied delete object in json format {}", introspector.marshal(false)); + } + } catch(Exception ex){ + LOGGER.warn("Encountered an exception during retrieval of vertex properties with vertex-id {} -> {}", v.id(), LogFormatTools.getStackTop(ex)); + } + } + } + + // After all the appropriate logging, calling the delete to delete the affected vertices + this.delete(impliedDeleteVertices); + } this.executePostSideEffects(obj, v); return processedVertexes; @@ -910,9 +960,7 @@ public class DBSerializer { String cleanUp = "false"; boolean nodeOnly = false; StopWatch.conditionalStart(); - Tree<Element> tree = this.engine.getQueryEngine().findSubGraph(v, depth, nodeOnly); - TreeBackedVertex treeVertex = new TreeBackedVertex(v, tree); - this.dbToObject(obj, treeVertex, seen, depth, nodeOnly, cleanUp); + this.dbToObject(obj, v, seen, depth, nodeOnly, cleanUp); dbTimeMsecs += StopWatch.stopIfStarted(); return obj; } @@ -1608,7 +1656,6 @@ public class DBSerializer { public void deleteItemsWithTraversal(List<Vertex> vertexes) throws IllegalStateException { for (Vertex v : vertexes) { - LOGGER.debug("About to delete the vertex with id: " + v.id()); deleteWithTraversal(v); } @@ -1624,10 +1671,39 @@ public class DBSerializer { List<Vertex> results = this.engine.getQueryEngine().findDeletable(startVertex); for (Vertex v : results) { - LOGGER.warn("Removing vertex " + v.id().toString()); + LOGGER.debug("Removing vertex {} with label {}", v.id(), v.label()); + v.remove(); + } + dbTimeMsecs += StopWatch.stopIfStarted(); + } + /** + * Removes the list of vertexes from the graph + * <p> + * Current the vertex label will just be vertex but + * in the future the aai-node-type property will be replaced + * by using the vertex label as when retrieving an vertex + * and retrieving an single property on an vertex will pre-fetch + * all the properties of that vertex and this is due to the following property + * <p> + * <code> + * query.fast-property=true + * </code> + * <p> + * JanusGraph doesn't provide the capability to override that + * at a transaction level and there is a plan to move to vertex label + * so it is best to utilize this for now and when the change is applied + * + * @param vertices - list of vertices to delete from the graph + */ + void delete(List<Vertex> vertices){ + StopWatch.conditionalStart(); + + for (Vertex v : vertices) { + LOGGER.debug("Removing vertex {} with label {}", v.id(), v.label()); v.remove(); } + dbTimeMsecs += StopWatch.stopIfStarted(); } diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java index d072db55..94557b08 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java @@ -58,7 +58,7 @@ public class GraphTraversalQueryEngine extends QueryEngine { /** * Instantiates a new graph traversal query engine. * - * @param graphEngine the graph engine + * @param g graph traversal source to traverse the graph */ public GraphTraversalQueryEngine(GraphTraversalSource g) { super(g); @@ -146,18 +146,47 @@ public class GraphTraversalQueryEngine extends QueryEngine { */ @Override public List<Vertex> findDeletable(Vertex start) { - @SuppressWarnings("unchecked") - GraphTraversal<Vertex, Vertex> pipe = this.g - .V(start).emit(v -> true).repeat( - __.union( - __.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(), - __.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV() - ) - ).dedup(); + try { + StopWatch.conditionalStart(); + @SuppressWarnings("unchecked") + GraphTraversal<Vertex, Vertex> pipe = this.g + .V(start).emit(v -> true).repeat( + __.union( + __.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(), + __.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV() + ) + ).dedup(); - return pipe.toList(); + return pipe.toList(); + } finally { + dbTimeMsecs += StopWatch.stopIfStarted(); + } } + /** + * {@inheritDoc} + */ + @Override + public List<Vertex> findDeletable(List<Vertex> startVertexes) { + try { + StopWatch.conditionalStart(); + Vertex[] vertices = new Vertex[startVertexes.size()]; + vertices = startVertexes.toArray(vertices); + GraphTraversal<Vertex, Vertex> pipe = this.g + .V(vertices).emit(v -> true).repeat( + __.union( + __.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(), + __.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV() + ) + ).dedup(); + + return pipe.toList(); + } + finally { + dbTimeMsecs += StopWatch.stopIfStarted(); + } + } + /** * {@inheritDoc} */ diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java index 110f8628..1e2da4b8 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java @@ -37,7 +37,7 @@ public abstract class QueryEngine { /** * Instantiates a new query engine. * - * @param graphEngine the graph engine + * @param g graph traversal source to traverse the graph */ public QueryEngine (GraphTraversalSource g) { this.g = g; @@ -100,6 +100,15 @@ public abstract class QueryEngine { */ public abstract List<Vertex> findDeletable(Vertex start); + /** + * Find all vertices that should be deleted in a cascade from a delete of start vertexes + * + * @param startVertexes Specifies the list of start vertexes + * + * @return the list of vertices to be deleted when start list of vertexes is deleted + */ + public abstract List<Vertex> findDeletable(List<Vertex> startVertexes); + /** * Finds the subgraph under start, including cousins as well as start's children/grandchildren/etc. * More specifically, this includes start, all its descendants, start's cousins, and start's diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java b/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java index 5b48127c..86cb635e 100644 --- a/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java +++ b/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java @@ -24,10 +24,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; -import java.util.ArrayList; -import java.util.HashMap; import java.util.Properties; -import java.util.Timer; import org.onap.aai.logging.LoggingContext; import org.onap.aai.logging.LoggingContext.StatusCode; @@ -47,7 +44,7 @@ public class AAIConfig { private static final String GLOBAL_PROP_FILE_NAME = AAIConstants.AAI_CONFIG_FILENAME; private static Properties serverProps; private static boolean propsInitialized = false; - + /** * Instantiates a new AAI config. */ @@ -71,11 +68,11 @@ public class AAIConfig { LoggingContext.statusCode(StatusCode.COMPLETE); LOGGER.info("Initializing AAIConfig"); - + AAIConfig.getConfigFile(); AAIConfig.reloadConfig(); - - if (AAIConstants.AAI_NODENAME == null || AAIConstants.AAI_NODENAME == "") { + + if (AAIConstants.AAI_NODENAME == null || AAIConstants.AAI_NODENAME == "") { ErrorLogHelper.logError("AAI_4005", " AAI_NODENAME is not defined"); } else { LOGGER.info("A&AI Server Node Name = " + AAIConstants.AAI_NODENAME); @@ -99,9 +96,9 @@ public class AAIConfig { String propFileName = GLOBAL_PROP_FILE_NAME; Properties newServerProps = null; - + LOGGER.debug("Reloading config from " + propFileName); - + try(InputStream is = new FileInputStream(propFileName)) { newServerProps = new Properties(); newServerProps.load(is); @@ -113,7 +110,7 @@ public class AAIConfig { ErrorLogHelper.logError("AAI_4002", " " + propFileName + ". IOException: "+e.getMessage()); } } - + /** * Gets the. * @@ -142,7 +139,7 @@ public class AAIConfig { */ public static String get(String key) throws AAIException { String response = null; - + if (key.equals(AAIConstants.AAI_NODENAME)) { // Get this from InetAddress rather than the properties file String nodeName = getNodeName(); @@ -151,16 +148,16 @@ public class AAIConfig { } // else get from property file } - + if (!propsInitialized || (serverProps == null)) { reloadConfig(); } - + if ((key.endsWith("password") || key.endsWith("passwd") || key.endsWith("apisecret")) && serverProps.containsKey(key+".x")) { String valx = serverProps.getProperty(key+".x"); return Password.deobfuscate(valx); } - + if (!serverProps.containsKey(key)) { throw new AAIException("AAI_4005", "Property key "+key+" cannot be found"); } else { @@ -180,9 +177,19 @@ public class AAIConfig { * @throws AAIException the AAI exception */ public static int getInt(String key) throws AAIException{ - return Integer.valueOf(AAIConfig.get(key)); + return Integer.parseInt(AAIConfig.get(key)); } + /** + * Gets the int. + * + * @param key the key + * @return the int + */ + public static int getInt(String key, String value) { + return Integer.parseInt(AAIConfig.get(key, value)); + } + /** * Gets the server props. * @@ -191,7 +198,7 @@ public class AAIConfig { public static Properties getServerProps() { return serverProps; } - + /** * Gets the node name. * @@ -211,8 +218,8 @@ public class AAIConfig { } return null; } - - + + /** * Check if a null or an Empty string is passed in. * diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java b/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java index b735f722..b9fb08f8 100644 --- a/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java +++ b/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java @@ -69,6 +69,15 @@ public final class AAIConstants { public static final String AAI_NODENAME = "aai.config.nodename"; + /* + * Logs the objects being deleted when an client deletes objects via implied delete during PUT operation + */ + public static final String AAI_IMPLIED_DELETE_LOG_ENABLED = "aai.implied.delete.log.enabled"; + /* + * Specifies how many objects maximum to log + */ + public static final String AAI_IMPLIED_DELETE_LOG_LIMIT = "aai.implied.delete.log.limit"; + public static final String AAI_BULKCONSUMER_LIMIT = "aai.bulkconsumer.payloadlimit"; public static final String AAI_BULKCONSUMER_OVERRIDE_LIMIT = "aai.bulkconsumer.payloadoverride"; |