Add the logging statements for when

application is doing implicit delete so if an
user did a PUT on cloud region in which they implicitly
delete 100 tenants, it would log the specific tenants that got deleted with depth all on
tenants because a cloud region could also have thousands of other child
objects and there is no reason to log that if we can just log the
objects that were lost in the implicit delete process

Issue-ID: AAI-2332
Change-Id: I8cd0c876670f8687b6485d3735f9f47e84c38286
Signed-off-by: Kajur, Harish (vk250x) <vk250x@att.com>

5 files changed, 171 insertions, 41 deletions

diff --git a/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java b/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java
index 4c9a9b15..575f8191 100644
--- a/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java
+++ b/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java
@@ -23,12 +23,13 @@ package org.onap.aai.serialization.db;
 import com.att.eelf.configuration.EELFLogger;
 import com.att.eelf.configuration.EELFManager;
 import com.google.common.base.CaseFormat;
-import com.google.common.collect.Multimap;
 import org.apache.commons.collections.IteratorUtils;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
-import org.apache.tinkerpop.gremlin.structure.*;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
 import org.janusgraph.core.SchemaViolationException;
 import org.javatuples.Triplet;
 import org.onap.aai.concurrent.AaiCallable;
@@ -62,7 +63,6 @@ import org.onap.aai.serialization.db.exceptions.MultipleEdgeRuleFoundException;
 import org.onap.aai.serialization.db.exceptions.NoEdgeRuleFoundException;
 import org.onap.aai.serialization.engines.TransactionalGraphEngine;
 import org.onap.aai.serialization.engines.query.QueryEngine;
-import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
 import org.onap.aai.setup.SchemaVersion;
 import org.onap.aai.setup.SchemaVersions;
 import org.onap.aai.util.AAIConfig;
@@ -81,15 +81,15 @@ import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 public class DBSerializer {
 
     private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DBSerializer.class);
 
+    private static final String IMPLICIT_DELETE = "Implicit DELETE";
+
     private static final String MISSING_REQUIRED_NODE_PROPERTY = "Vertex missing required aai-node-type property";
 
     private final TransactionalGraphEngine engine;
@@ -417,7 +417,57 @@ public class DBSerializer {
         for (Vertex toBeRemoved : processedVertexes) {
             dependentVertexes.remove(toBeRemoved);
         }
-        this.deleteItemsWithTraversal(dependentVertexes);
+
+        // If the dependent vertices are not empty, then with
+        // the current behaviour, it should remove the vertices implicitly
+        // We are updating the code to properly log which call
+        // is doing this so the SE can work with the clients making the call to
+        // tell them not to call this API and can hopefully deprecate this
+        // functionality in the future releases
+        if(!dependentVertexes.isEmpty()){
+
+            LoggingContext.responseDescription(IMPLICIT_DELETE);
+
+            // Find all the deletable vertices from the dependent vertices that should be deleted
+            // So for each of the following dependent vertices,
+            // we will use the edge properties and do the cascade delete
+            List<Vertex> impliedDeleteVertices = this.engine.getQueryEngine().findDeletable(dependentVertexes);
+            int impliedDeleteCount = impliedDeleteVertices.size();
+
+            LOGGER.warn(
+                "For the vertex with id {}, doing an implicit delete on update will delete total of {} vertexes",
+                v.id(),
+                impliedDeleteCount
+            );
+
+            String impliedDeleteLogEnabled = AAIConfig.get(AAIConstants.AAI_IMPLIED_DELETE_LOG_ENABLED, "true");
+
+            int impliedDeleteLogLimit = AAIConfig.getInt(AAIConstants.AAI_IMPLIED_DELETE_LOG_LIMIT, "-1");
+
+            if(impliedDeleteLogLimit == -1){
+                impliedDeleteLogLimit = Integer.MAX_VALUE;
+            }
+
+            // If the logging is enabled for implied delete
+            // then log the payload in the latest format
+            if("true".equals(impliedDeleteLogEnabled) &&
+                impliedDeleteCount <= impliedDeleteLogLimit){
+                for(Vertex vertex : impliedDeleteVertices){
+                    Introspector introspector = null;
+                    try {
+                        introspector = getLatestVersionView(vertex);
+                        if(LOGGER.isInfoEnabled()){
+                            LOGGER.info("Implied delete object in json format {}", introspector.marshal(false));
+                        }
+                    } catch(Exception ex){
+                        LOGGER.warn("Encountered an exception during retrieval of vertex properties with vertex-id {} -> {}", v.id(), LogFormatTools.getStackTop(ex));
+                    }
+                }
+            }
+
+            // After all the appropriate logging, calling the delete to delete the affected vertices
+            this.delete(impliedDeleteVertices);
+        }
 
         this.executePostSideEffects(obj, v);
         return processedVertexes;
@@ -910,9 +960,7 @@ public class DBSerializer {
         String cleanUp = "false";
         boolean nodeOnly = false;
         StopWatch.conditionalStart();
-        Tree<Element> tree = this.engine.getQueryEngine().findSubGraph(v, depth, nodeOnly);
-        TreeBackedVertex treeVertex = new TreeBackedVertex(v, tree);
-        this.dbToObject(obj, treeVertex, seen, depth, nodeOnly, cleanUp);
+        this.dbToObject(obj, v, seen, depth, nodeOnly, cleanUp);
         dbTimeMsecs += StopWatch.stopIfStarted();
         return obj;
     }
@@ -1608,7 +1656,6 @@ public class DBSerializer {
     public void deleteItemsWithTraversal(List<Vertex> vertexes) throws IllegalStateException {
 
         for (Vertex v : vertexes) {
-            LOGGER.debug("About to delete the vertex with id: " + v.id());
             deleteWithTraversal(v);
         }
 
@@ -1624,10 +1671,39 @@ public class DBSerializer {
         List<Vertex> results = this.engine.getQueryEngine().findDeletable(startVertex);
 
         for (Vertex v : results) {
-            LOGGER.warn("Removing vertex " + v.id().toString());
+            LOGGER.debug("Removing vertex {} with label {}", v.id(), v.label());
+            v.remove();
+        }
+        dbTimeMsecs += StopWatch.stopIfStarted();
+    }
 
+    /**
+     * Removes the list of vertexes from the graph
+     * <p>
+     * Current the vertex label will just be vertex but
+     * in the future the aai-node-type property will be replaced
+     * by using the vertex label as when retrieving an vertex
+     * and retrieving an single property on an vertex will pre-fetch
+     * all the properties of that vertex and this is due to the following property
+     * <p>
+     * <code>
+     * query.fast-property=true
+     * </code>
+     * <p>
+     * JanusGraph doesn't provide the capability to override that
+     * at a transaction level and there is a plan to move to vertex label
+     * so it is best to utilize this for now and when the change is applied
+     *
+     * @param vertices - list of vertices to delete from the graph
+     */
+    void delete(List<Vertex> vertices){
+        StopWatch.conditionalStart();
+
+        for (Vertex v : vertices) {
+            LOGGER.debug("Removing vertex {} with label {}", v.id(), v.label());
             v.remove();
         }
+
         dbTimeMsecs += StopWatch.stopIfStarted();
     }
 
diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java
index d072db55..94557b08 100644
--- a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java
+++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java
@@ -58,7 +58,7 @@ public class GraphTraversalQueryEngine extends QueryEngine {
 	/**
 	 * Instantiates a new graph traversal query engine.
 	 *
-	 * @param graphEngine the graph engine
+	 * @param g     graph traversal source to traverse the graph
 	 */
 	public GraphTraversalQueryEngine(GraphTraversalSource g) {
 		super(g);
@@ -146,18 +146,47 @@ public class GraphTraversalQueryEngine extends QueryEngine {
 	 */
 	@Override
 	public List<Vertex> findDeletable(Vertex start) {
-		@SuppressWarnings("unchecked")
-		GraphTraversal<Vertex, Vertex> pipe = this.g
-				.V(start).emit(v -> true).repeat(
-					__.union(
-						__.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(),
-						__.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV()
-					)
-				).dedup();
+        try {
+            StopWatch.conditionalStart();
+            @SuppressWarnings("unchecked")
+            GraphTraversal<Vertex, Vertex> pipe = this.g
+                    .V(start).emit(v -> true).repeat(
+                        __.union(
+                            __.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(),
+                            __.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV()
+                        )
+                    ).dedup();
 
-		return pipe.toList();
+            return pipe.toList();
+        } finally {
+            dbTimeMsecs += StopWatch.stopIfStarted();
+        }
 	}
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public List<Vertex> findDeletable(List<Vertex> startVertexes) {
+        try {
+            StopWatch.conditionalStart();
+            Vertex[] vertices = new Vertex[startVertexes.size()];
+            vertices = startVertexes.toArray(vertices);
+            GraphTraversal<Vertex, Vertex> pipe = this.g
+                .V(vertices).emit(v -> true).repeat(
+                    __.union(
+                        __.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(),
+                        __.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV()
+                    )
+                ).dedup();
+
+            return pipe.toList();
+        }
+        finally {
+            dbTimeMsecs += StopWatch.stopIfStarted();
+        }
+    }
+
 	/**
 	 * {@inheritDoc}
 	 */
diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java
index 110f8628..1e2da4b8 100644
--- a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java
+++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java
@@ -37,7 +37,7 @@ public abstract class QueryEngine {
 	/**
 	 * Instantiates a new query engine.
 	 *
-	 * @param graphEngine the graph engine
+     * @param g     graph traversal source to traverse the graph
 	 */
 	public QueryEngine (GraphTraversalSource g) {
 		this.g = g;
@@ -100,6 +100,15 @@ public abstract class QueryEngine {
 	 */
 	public abstract List<Vertex> findDeletable(Vertex start);
 
+    /**
+     * Find all vertices that should be deleted in a cascade from a delete of start vertexes
+     *
+     * @param   startVertexes   Specifies the list of start vertexes
+     *
+     * @return  the list of vertices to be deleted when start list of vertexes is deleted
+     */
+    public abstract List<Vertex> findDeletable(List<Vertex> startVertexes);
+
 	/**
 	 * Finds the subgraph under start, including cousins as well as start's children/grandchildren/etc.
 	 * More specifically, this includes start, all its descendants, start's cousins, and start's
diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java b/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java
index 5b48127c..86cb635e 100644
--- a/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java
+++ b/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java
@@ -24,10 +24,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.Properties;
-import java.util.Timer;
 import org.onap.aai.logging.LoggingContext;
 import org.onap.aai.logging.LoggingContext.StatusCode;
 
@@ -47,7 +44,7 @@ public class AAIConfig {
 	private static final String GLOBAL_PROP_FILE_NAME = AAIConstants.AAI_CONFIG_FILENAME;
 	private static Properties serverProps;
     private static boolean propsInitialized = false;
-    
+
     /**
      * Instantiates a new AAI config.
      */
@@ -71,11 +68,11 @@ public class AAIConfig {
 		LoggingContext.statusCode(StatusCode.COMPLETE);
 
 		LOGGER.info("Initializing AAIConfig");
-		
+
         AAIConfig.getConfigFile();
         AAIConfig.reloadConfig();
-        
-        if (AAIConstants.AAI_NODENAME == null || AAIConstants.AAI_NODENAME == "") {      
+
+        if (AAIConstants.AAI_NODENAME == null || AAIConstants.AAI_NODENAME == "") {
             ErrorLogHelper.logError("AAI_4005", " AAI_NODENAME is not defined");
         } else {
             LOGGER.info("A&AI Server Node Name = " + AAIConstants.AAI_NODENAME);
@@ -99,9 +96,9 @@ public class AAIConfig {
 
         String propFileName = GLOBAL_PROP_FILE_NAME;
         Properties newServerProps = null;
-        
+
         LOGGER.debug("Reloading config from " + propFileName);
-        
+
         try(InputStream is = new FileInputStream(propFileName)) {
             newServerProps = new Properties();
             newServerProps.load(is);
@@ -113,7 +110,7 @@ public class AAIConfig {
         	ErrorLogHelper.logError("AAI_4002", " " + propFileName + ". IOException: "+e.getMessage());
         }
     }
-    
+
     /**
      * Gets the.
      *
@@ -142,7 +139,7 @@ public class AAIConfig {
      */
     public static String get(String key) throws AAIException {
     	String response = null;
-    	
+
     	if (key.equals(AAIConstants.AAI_NODENAME)) {
     		// Get this from InetAddress rather than the properties file
     		String nodeName = getNodeName();
@@ -151,16 +148,16 @@ public class AAIConfig {
     		}
     		// else get from property file
     	}
-    	
+
     	if (!propsInitialized || (serverProps == null)) {
     		reloadConfig();
     	}
-    	
+
     	if ((key.endsWith("password") || key.endsWith("passwd") || key.endsWith("apisecret")) && serverProps.containsKey(key+".x")) {
     		String valx = serverProps.getProperty(key+".x");
     		return Password.deobfuscate(valx);
     	}
-    	
+
     	if (!serverProps.containsKey(key)) {
     		throw new AAIException("AAI_4005", "Property key "+key+" cannot be found");
     	} else {
@@ -180,9 +177,19 @@ public class AAIConfig {
      * @throws AAIException the AAI exception
      */
     public static int getInt(String key) throws AAIException{
-    	return Integer.valueOf(AAIConfig.get(key));
+    	return Integer.parseInt(AAIConfig.get(key));
 	}
 
+    /**
+     * Gets the int.
+     *
+     * @param key the key
+     * @return the int
+     */
+    public static int getInt(String key, String value) {
+        return Integer.parseInt(AAIConfig.get(key, value));
+    }
+
 	/**
 	 * Gets the server props.
 	 *
@@ -191,7 +198,7 @@ public class AAIConfig {
 	public static Properties getServerProps() {
 		return serverProps;
 	}
-	
+
 	/**
 	 * Gets the node name.
 	 *
@@ -211,8 +218,8 @@ public class AAIConfig {
 		}
 		return null;
 	}
-	
-	
+
+
 	/**
 	 * Check if a null or an Empty string is passed in.
 	 *
diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java b/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java
index b735f722..b9fb08f8 100644
--- a/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java
+++ b/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java
@@ -69,6 +69,15 @@ public final class AAIConstants {
 
     public static final String AAI_NODENAME = "aai.config.nodename";
 
+    /*
+     * Logs the objects being deleted when an client deletes objects via implied delete during PUT operation
+     */
+    public static final String AAI_IMPLIED_DELETE_LOG_ENABLED = "aai.implied.delete.log.enabled";
+    /*
+     * Specifies how many objects maximum to log
+     */
+    public static final String AAI_IMPLIED_DELETE_LOG_LIMIT = "aai.implied.delete.log.limit";
+
     public static final String AAI_BULKCONSUMER_LIMIT = "aai.bulkconsumer.payloadlimit";
     public static final String AAI_BULKCONSUMER_OVERRIDE_LIMIT = "aai.bulkconsumer.payloadoverride";