aboutsummaryrefslogtreecommitdiffstats
path: root/src/main/java/org/onap/aai/datagrooming/DataGrooming.java
diff options
context:
space:
mode:
Diffstat (limited to 'src/main/java/org/onap/aai/datagrooming/DataGrooming.java')
-rw-r--r--src/main/java/org/onap/aai/datagrooming/DataGrooming.java128
1 files changed, 110 insertions, 18 deletions
diff --git a/src/main/java/org/onap/aai/datagrooming/DataGrooming.java b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java
index 7e667f4..167ec21 100644
--- a/src/main/java/org/onap/aai/datagrooming/DataGrooming.java
+++ b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java
@@ -91,6 +91,8 @@ public class DataGrooming {
HashMap<String, Vertex> orphanNodeHash ;
HashMap<String, Vertex> missingAaiNtNodeHash ;
+ HashMap<String, Vertex> badUriNodeHash ;
+ HashMap<String, Vertex> badIndexNodeHash ;
HashMap<String, Edge> oneArmedEdgeHash ;
HashMap<String, Vertex> ghostNodeHash ;
ArrayList<String> dupeGroups;
@@ -356,6 +358,7 @@ public class DataGrooming {
JanusGraph graph2 = null;
deleteCount = 0;
int dummyUpdCount = 0;
+ int indexUpdCount = 0;
boolean executeFinalCommit = false;
deleteCandidateList = new LinkedHashSet<>();
Set<String> processedVertices = new LinkedHashSet<>();
@@ -445,6 +448,8 @@ public class DataGrooming {
HashMap<String, String> misMatchedHash = new HashMap<String, String>();
orphanNodeHash = new HashMap<String, Vertex>();
missingAaiNtNodeHash = new HashMap<String, Vertex>();
+ badUriNodeHash = new HashMap<String, Vertex>();
+ badIndexNodeHash = new HashMap<String, Vertex>();
oneArmedEdgeHash = new HashMap<String, Edge>();
HashMap<String, String> emptyVertexHash = new HashMap<String, String>();
ghostNodeHash = new HashMap<String, Vertex>();
@@ -696,19 +701,24 @@ public class DataGrooming {
&& depNodeOk){
aaiKeysOk = false;
}
-
- if( (!aaiKeysOk || !aaiUriOk)
- && !deleteCandidateList.contains(thisVid)
- && !skipIndexUpdateFix ){
- // Either the aaiKeys or aaiUri was bad. This may
- // be a problem with the indexes so we'll try to reset
- // them since this node is not on the delete list from
- // a previous run.
- tryToReSetIndexedProps(thisVtx, thisVid, indexedProps);
- }
-
- if( !aaiKeysOk || !aaiUriOk ){
- // We could not get the node back using it's own key info or aai-uri.
+
+ boolean bothKeysAreBad = false;
+ if( !aaiKeysOk && !aaiUriOk ) {
+ bothKeysAreBad = true;
+ }
+ else if ( !aaiKeysOk ){
+ // Just the key-index is bad
+ // We will not be putting this on the Auto-Delete list, just logging it (AAI-16252)
+ badIndexNodeHash.put(thisVid, thisVtx);
+ }
+ else if ( !aaiUriOk ){
+ // Just the aai-uri is bad
+ // We will not be putting this on the Auto-Delete list, just logging it (AAI-16252)
+ badUriNodeHash.put(thisVid, thisVtx);
+ }
+
+ if( bothKeysAreBad ){
+ // Neither the aai-uri nor key info could retrieve this node - BOTH are bad.
// So, it's a PHANTOM
if (deleteCandidateList.contains(thisVid)) {
@@ -854,7 +864,7 @@ public class DataGrooming {
ArrayList<Vertex> vertList = new ArrayList<>();
Iterator<Vertex> vItor3 = g.traversal().V();
- // Gotta hold these in a List - or else HBase times out as you cycle
+ // Gotta hold these in a List - or else the DB times out as you cycle
// through these
while (vItor3.hasNext()) {
Vertex v = vItor3.next();
@@ -888,7 +898,7 @@ public class DataGrooming {
continue;
}
if (ghostNodeHash.containsKey(thisVertId)) {
- // This is a phantom node, so don't try to use it
+ // We already know that this is a phantom node, so don't bother checking it
LOGGER.info(" >> Skipping edge check for edges from vertexId = "
+ thisVertId
+ ", since that guy is a Phantom Node");
@@ -1168,7 +1178,7 @@ public class DataGrooming {
deleteCount = deleteCount + dupeGrpsDeleted;
- if (deleteCount > 0 || dummyUpdCount > 0){
+ if (deleteCount > 0 || dummyUpdCount > 0 || indexUpdCount > 0){
executeFinalCommit = true;
}
@@ -1176,6 +1186,8 @@ public class DataGrooming {
int orphanNodeCount = orphanNodeHash.size();
int oneArmedEdgeCount = oneArmedEdgeHash.size();
int missingAaiNtNodeCount = missingAaiNtNodeHash.size();
+ int badUriNodeCount = badUriNodeHash.size();
+ int badIndexNodeCount = badIndexNodeHash.size();
int dupeCount = dupeGroups.size();
deleteCount = deleteCount + dupeGrpsDeleted;
@@ -1198,12 +1210,16 @@ public class DataGrooming {
+ "\n");
bw.write("Dummy-index-update to delete candidates = " + dummyUpdCount
+ "\n");
+ bw.write("index-update-Fix Attempts to phantom nodes = " + indexUpdCount
+ + "\n");
bw.write("Total number of nodes looked at = " + totalNodeCount
+ "\n");
bw.write("Ghost Nodes identified = " + ghostNodeCount + "\n");
bw.write("Orphan Nodes identified = " + orphanNodeCount + "\n");
bw.write("Missing aai-node-type Nodes identified = " + missingAaiNtNodeCount + "\n");
bw.write("Bad Edges identified = " + oneArmedEdgeCount + "\n");
+ bw.write("Bad aai-uri property Nodes identified = " + badUriNodeCount + "\n");
+ bw.write("Bad index property Nodes identified = " + badIndexNodeCount + "\n");
bw.write("Duplicate Groups count = " + dupeCount + "\n");
bw.write("MisMatching Label/aai-node-type count = "
+ misMatchedHash.size() + "\n");
@@ -1234,6 +1250,8 @@ public class DataGrooming {
bw.write("DeleteCandidate: Bad EDGE Edge-id = [" + eid + "]\n");
cleanupCandidateCount++;
}
+
+
bw.write("\n-- NOTE - To see DeleteCandidates for Duplicates, you need to look in the Duplicates Detail section below.\n");
@@ -1260,7 +1278,7 @@ public class DataGrooming {
}
}
- bw.write("\n ------------- Missing aai-node-type NODES - detail: ");
+ bw.write("\n ------------- Missing aai-node-type NODES - detail ");
for (Map.Entry<String, Vertex> entry : missingAaiNtNodeHash
.entrySet()) {
try {
@@ -1283,6 +1301,54 @@ public class DataGrooming {
LOGGER.error("error trying to print detail info for a node missing its aai-node-type " + LogFormatTools.getStackTop(dex));
}
}
+
+ bw.write("\n ------------- Nodes where aai-uri property is bad - detail ");
+ for (Map.Entry<String, Vertex> entry : badUriNodeHash
+ .entrySet()) {
+ try {
+ String vid = entry.getKey();
+ bw.write("\n> Has Bad aai-uri - Vid = " + vid + "\n");
+ ArrayList<String> retArr = showPropertiesForNode(
+ TRANSID, FROMAPPID, entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+
+ retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
+ entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+ } catch (Exception dex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("error trying to print detail info for a node with a bad aai-uri " + LogFormatTools.getStackTop(dex));
+ }
+ }
+
+ bw.write("\n ------------- Nodes where an indexed property is bad - detail: ");
+ for (Map.Entry<String, Vertex> entry : badIndexNodeHash
+ .entrySet()) {
+ try {
+ String vid = entry.getKey();
+ bw.write("\n> Node with bad index - Vid = " + vid + "\n");
+ ArrayList<String> retArr = showPropertiesForNode(
+ TRANSID, FROMAPPID, entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+
+ retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
+ entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+ } catch (Exception dex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("error trying to print detail info for a node with bad index " + LogFormatTools.getStackTop(dex));
+ }
+ }
bw.write("\n ------------- Missing Dependent Edge ORPHAN NODES - detail: ");
for (Map.Entry<String, Vertex> entry : orphanNodeHash
@@ -1430,7 +1496,7 @@ public class DataGrooming {
LOGGER.info("\n ------------- Done doing all the checks ------------ ");
LOGGER.info("Output will be written to " + fullOutputFileName);
- if (cleanupCandidateCount > 0) {
+ if (cleanupCandidateCount > 0 || badUriNodeCount > 0 || badIndexNodeCount > 0) {
// Technically, this is not an error -- but we're throwing this
// error so that hopefully a
// monitoring system will pick it up and do something with it.
@@ -1560,6 +1626,8 @@ public class DataGrooming {
// We will only deal with properties that are indexed and have a value - and for those,
// we will re-set them to the same value they already have, so that hopefully if their
// index was broken, it may get re-set.
+
+ // NOTE -- as of 1902-P2, this is deprecated --------------
LOGGER.info(" We will try to re-set the indexed properties for this node without changing any property values. VID = " + thisVidStr );
// These reserved-prop-names are all indexed for all nodes
@@ -3011,7 +3079,31 @@ class CommandLineArgs {
public int getMissingAaiNtNodeCount(){
return getMissingAaiNtNodeHash().size();
}
+
+ public HashMap<String, Vertex> getBadUriNodeHash() {
+ return badUriNodeHash;
+ }
+ public void setBadUriNodeHash(HashMap<String, Vertex> badUriNodeHash) {
+ this.badUriNodeHash = badUriNodeHash;
+ }
+
+ public int getBadUriNodeCount(){
+ return getBadUriNodeHash().size();
+ }
+
+ public HashMap<String, Vertex> getBadIndexNodeHash() {
+ return badIndexNodeHash;
+ }
+
+ public void setBadIndexNodeHash(HashMap<String, Vertex> badIndexNodeHash) {
+ this.badIndexNodeHash = badIndexNodeHash;
+ }
+
+ public int getBadIndexNodeCount(){
+ return getBadIndexNodeHash().size();
+ }
+
public HashMap<String, Edge> getOneArmedEdgeHash() {
return oneArmedEdgeHash;
}