aboutsummaryrefslogtreecommitdiffstats
path: root/src/main/java/org/onap/aai/dbgen/DupeTool.java
diff options
context:
space:
mode:
Diffstat (limited to 'src/main/java/org/onap/aai/dbgen/DupeTool.java')
-rw-r--r--src/main/java/org/onap/aai/dbgen/DupeTool.java205
1 files changed, 162 insertions, 43 deletions
diff --git a/src/main/java/org/onap/aai/dbgen/DupeTool.java b/src/main/java/org/onap/aai/dbgen/DupeTool.java
index 7b7ef99..fd5ae00 100644
--- a/src/main/java/org/onap/aai/dbgen/DupeTool.java
+++ b/src/main/java/org/onap/aai/dbgen/DupeTool.java
@@ -19,22 +19,19 @@
*/
package org.onap.aai.dbgen;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.util.*;
-import java.util.Map.Entry;
-
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.onap.aai.db.props.AAIProperties;
-import org.onap.aai.dbmap.AAIGraphConfig;
+import org.apache.tinkerpop.gremlin.structure.*;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.onap.aai.config.PropertyPasswordConfiguration;
import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.edges.enums.AAIDirection;
import org.onap.aai.edges.enums.EdgeProperty;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.Introspector;
@@ -45,19 +42,19 @@ import org.onap.aai.logging.ErrorLogHelper;
import org.onap.aai.logging.LogFormatTools;
import org.onap.aai.logging.LoggingContext;
import org.onap.aai.logging.LoggingContext.StatusCode;
-import org.onap.aai.edges.enums.AAIDirection;
import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.ExceptionTranslator;
+import org.onap.aai.util.GraphAdminConstants;
import org.slf4j.MDC;
-
-import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-import org.janusgraph.core.JanusGraphFactory;
-import org.janusgraph.core.JanusGraph;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.*;
+import java.util.Map.Entry;
+
public class DupeTool {
private static final EELFLogger logger = EELFManager.getInstance().getLogger(DupeTool.class.getSimpleName());
@@ -76,6 +73,7 @@ public class DupeTool {
}
private LoaderFactory loaderFactory;
+ private int dupeGroupCount = 0;
public DupeTool(LoaderFactory loaderFactory, SchemaVersions schemaVersions){
this(loaderFactory, schemaVersions, true);
@@ -89,7 +87,7 @@ public class DupeTool {
public void execute(String[] args){
- String defVersion = "v12";
+ String defVersion = "v15";
try {
defVersion = AAIConfig.get(AAIConstants.AAI_DEFAULT_API_VERSION_PROP);
} catch (AAIException ae) {
@@ -101,7 +99,7 @@ public class DupeTool {
exit(0);
}
-
+ dupeGroupCount = 0;
Loader loader = null;
try {
loader = loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion());
@@ -120,8 +118,8 @@ public class DupeTool {
try {
AAIConfig.init();
- int maxRecordsToFix = AAIConstants.AAI_DUPETOOL_DEFAULT_MAX_FIX;
- int sleepMinutes = AAIConstants.AAI_DUPETOOL_DEFAULT_SLEEP_MINUTES;
+ int maxRecordsToFix = GraphAdminConstants.AAI_DUPETOOL_DEFAULT_MAX_FIX;
+ int sleepMinutes = GraphAdminConstants.AAI_DUPETOOL_DEFAULT_SLEEP_MINUTES;
int timeWindowMinutes = 0; // A value of 0 means that we will not have a time-window -- we will look
// at all nodes of the passed-in nodeType.
long windowStartTime = 0; // Translation of the window into a starting timestamp
@@ -137,7 +135,7 @@ public class DupeTool {
}
} catch (Exception e) {
// Don't worry, we'll just use the defaults that we got from AAIConstants
- logger.warn("WARNING - could not pick up aai.dupeTool values from aaiconfig.properties file. Will use defaults. ");
+ logger.warn("WARNING - could not pick up aai.dupeTool values from aaiconfig.properties file. Will use defaults. " + e.getMessage());
}
String nodeTypeVal = "";
@@ -364,7 +362,7 @@ public class DupeTool {
showNodeDetailsForADupeSet(gt1, firstPassDupeSets.get(x), logger);
}
}
-
+ dupeGroupCount = firstPassDupeSets.size();
boolean didSomeDeletesFlag = false;
ArrayList<String> dupeSetsToFix = new ArrayList<String>();
if (autoFix && firstPassDupeSets.size() == 0) {
@@ -405,6 +403,7 @@ public class DupeTool {
+ " sets of duplicates that we think can be deleted. ";
logger.info(msg);
System.out.println(msg);
+
if (dupeSetsToFix.size() > 0) {
msg = " Here is what the sets look like: ";
logger.info(msg);
@@ -492,7 +491,7 @@ public class DupeTool {
*
* @param args the arguments
*/
- public static void main(String[] args) {
+ public static void main(String[] args) throws AAIException {
System.setProperty("aai.service.name", DupeTool.class.getSimpleName());
// Set the logging file properties to be used by EELFManager
@@ -511,11 +510,23 @@ public class DupeTool {
LoggingContext.statusCode(StatusCode.COMPLETE);
LoggingContext.responseCode(LoggingContext.SUCCESS);
- AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
- "org.onap.aai.config",
- "org.onap.aai.setup"
- );
-
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
+ PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
+ initializer.initialize(ctx);
+ try {
+ ctx.scan(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+ ctx.refresh();
+ } catch (Exception e) {
+ AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
+ logger.error("Problems running DupeTool "+aai.getMessage());
+ LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
+ throw aai;
+ }
LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
DupeTool dupeTool = new DupeTool(loaderFactory, schemaVersions);
@@ -1080,10 +1091,12 @@ public class DupeTool {
Boolean specialTenantRule, Loader loader, EELFLogger logger)
throws AAIException {
- // This method assumes that it is being passed a List of vertex objects
- // which violate our uniqueness constraints.
-
+ // This method assumes that it is being passed a List of
+ // vertex objects which violate our uniqueness constraints.
+ // Note - returning a null vertex means we could not
+ // safely pick one to keep (Ie. safely know which to delete.)
Vertex nullVtx = null;
+ GraphTraversalSource gts = g.traversal();
if (dupeVertexList == null) {
return nullVtx;
@@ -1095,12 +1108,35 @@ public class DupeTool {
if (listSize == 1) {
return (dupeVertexList.get(0));
}
+
+ // If they don't all have the same aai-uri, then we will not
+ // choose between them - we'll need someone to manually
+ // check to pick which one makes sense to keep.
+ Object uriOb = dupeVertexList.get(0).<Object>property("aai-uri").orElse(null);
+ if( uriOb == null || uriOb.toString().equals("") ){
+ // this is a bad node - hopefully will be picked up by phantom checker
+ return nullVtx;
+ }
+ String thisUri = uriOb.toString();
+ for (int i = 1; i < listSize; i++) {
+ uriOb = dupeVertexList.get(i).<Object>property("aai-uri").orElse(null);
+ if( uriOb == null || uriOb.toString().equals("") ){
+ // this is a bad node - hopefully will be picked up by phantom checker
+ return nullVtx;
+ }
+ String nextUri = uriOb.toString();
+ if( !thisUri.equals(nextUri)){
+ // there are different URI's on these - so we can't pick
+ // a dupe to keep. Someone will need to look at it.
+ return nullVtx;
+ }
+ }
Vertex vtxPreferred = null;
Vertex currentFaveVtx = dupeVertexList.get(0);
for (int i = 1; i < listSize; i++) {
Vertex vtxB = dupeVertexList.get(i);
- vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
+ vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, gts,
currentFaveVtx, vtxB, ver, specialTenantRule, loader, logger);
if (vtxPreferred == null) {
// We couldn't choose one
@@ -1110,7 +1146,14 @@ public class DupeTool {
}
}
- return (currentFaveVtx);
+ if( currentFaveVtx != null && checkAaiUriOk(gts, currentFaveVtx, logger) ){
+ return (currentFaveVtx);
+ }
+ else {
+ // We had a preferred vertex, but its aai-uri was bad, so
+ // we will not recommend one to keep.
+ return nullVtx;
+ }
} // end of getPreferredDupe()
@@ -1120,7 +1163,7 @@ public class DupeTool {
*
* @param transId the trans id
* @param fromAppId the from app id
- * @param g the g
+ * @param g the graphTraversalSource
* @param vtxA the vtx A
* @param vtxB the vtx B
* @param ver the ver
@@ -1130,7 +1173,7 @@ public class DupeTool {
* @throws AAIException the AAI exception
*/
public Vertex pickOneOfTwoDupes(String transId,
- String fromAppId, Graph g, Vertex vtxA,
+ String fromAppId, GraphTraversalSource gts, Vertex vtxA,
Vertex vtxB, String ver, Boolean specialTenantRule, Loader loader, EELFLogger logger) throws AAIException {
Vertex nullVtx = null;
@@ -1289,11 +1332,13 @@ public class DupeTool {
}
if (allTheSame) {
- if (vidA < vidB) {
- preferredVtx = vtxA;
- } else {
- preferredVtx = vtxB;
- }
+ if ( checkAaiUriOk(gts, vtxA, logger) ) {
+ preferredVtx = vtxA;
+ }
+ else if ( checkAaiUriOk(gts, vtxB, logger) ) {
+ preferredVtx = vtxB;
+ }
+ // else we're picking neither because neither one had a working aai-uri index property
} else if (specialTenantRule) {
// They asked us to apply a special rule if it applies
if (vtxIdsConn2A.size() == 2 && vtxANodeType.equals("tenant")) {
@@ -1575,6 +1620,71 @@ public class DupeTool {
}// End of getNodeKeyVals()
+
+
+ /**
+ * makes sure aai-uri exists and can be used to get this node back
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param graph the graph
+ * @param vtx
+ * @param EELFLogger
+ * @return true if aai-uri is populated and the aai-uri-index points to this vtx
+ * @throws AAIException the AAI exception
+ */
+ private Boolean checkAaiUriOk( GraphTraversalSource graph, Vertex origVtx, EELFLogger eLogger )
+ throws AAIException{
+ String aaiUriStr = "";
+ try {
+ Object ob = origVtx.<Object>property("aai-uri").orElse(null);
+ String origVid = origVtx.id().toString();
+ if (ob == null || ob.toString().equals("")) {
+ // It is missing its aai-uri
+ eLogger.debug("DEBUG No [aai-uri] property found for vid = ["
+ + origVid + "] " );
+ return false;
+ }
+ else {
+ aaiUriStr = ob.toString();
+ Iterator <Vertex> verts = graph.V().has("aai-uri",aaiUriStr);
+ int count = 0;
+ while( verts.hasNext() ){
+ count++;
+ Vertex foundV = verts.next();
+ String foundVid = foundV.id().toString();
+ if( !origVid.equals(foundVid) ){
+ eLogger.debug("DEBUG aai-uri key property ["
+ + aaiUriStr + "] for vid = ["
+ + origVid + "] brought back different vertex with vid = ["
+ + foundVid + "]." );
+ return false;
+ }
+ }
+ if( count == 0 ){
+ eLogger.debug("DEBUG aai-uri key property ["
+ + aaiUriStr + "] for vid = ["
+ + origVid + "] could not be used to query for that vertex. ");
+ return false;
+ }
+ else if( count > 1 ){
+ eLogger.debug("DEBUG aai-uri key property ["
+ + aaiUriStr + "] for vid = ["
+ + origVid + "] brought back multiple ("
+ + count + ") vertices instead of just one. ");
+ return false;
+ }
+ }
+ }
+ catch( Exception ex ){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ eLogger.error(" ERROR trying to get node with aai-uri: [" + aaiUriStr + "]" + LogFormatTools.getStackTop(ex));
+ }
+ return true;
+
+ }// End of checkAaiUriOk()
+
/**
* Get values of the key properties for a node as a single string
@@ -1850,5 +1960,14 @@ public class DupeTool {
logger.warn("WARNING from final graph.shutdown()", ex);
}
}
+
+ public int getDupeGroupCount() {
+ return dupeGroupCount;
+ }
+
+ public void setDupeGroupCount(int dgCount) {
+ this.dupeGroupCount = dgCount;
+ }
+
}