diff options
Diffstat (limited to 'aai-resources/src/main/java/org/onap/aai/dbgen/DupeTool.java')
| -rw-r--r-- | aai-resources/src/main/java/org/onap/aai/dbgen/DupeTool.java | 1877 |
1 files changed, 0 insertions, 1877 deletions
diff --git a/aai-resources/src/main/java/org/onap/aai/dbgen/DupeTool.java b/aai-resources/src/main/java/org/onap/aai/dbgen/DupeTool.java deleted file mode 100644 index 1e323b0..0000000 --- a/aai-resources/src/main/java/org/onap/aai/dbgen/DupeTool.java +++ /dev/null @@ -1,1877 +0,0 @@ -/** - * ============LICENSE_START======================================================= - * org.onap.aai - * ================================================================================ - * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END========================================================= - */ -package org.onap.aai.dbgen; -import java.io.FileInputStream; -import java.io.InputStream; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; - -import java.util.Properties; -import java.util.UUID; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.structure.Direction; -import org.apache.tinkerpop.gremlin.structure.Edge; -import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.apache.tinkerpop.gremlin.structure.VertexProperty; -import org.onap.aai.db.props.AAIProperties; -import org.onap.aai.dbmap.AAIGraphConfig; -import org.onap.aai.dbmap.AAIGraph; -import org.onap.aai.exceptions.AAIException; -import org.onap.aai.introspection.Introspector; -import org.onap.aai.introspection.Loader; -import org.onap.aai.introspection.LoaderFactory; -import org.onap.aai.introspection.ModelType; -import org.onap.aai.logging.ErrorLogHelper; -import org.onap.aai.logging.LogFormatTools; -import org.onap.aai.logging.LoggingContext; -import org.onap.aai.logging.LoggingContext.StatusCode; -import org.onap.aai.util.AAIConfig; -import org.onap.aai.util.AAIConstants; -import org.slf4j.MDC; - -import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; -import org.janusgraph.core.JanusGraphFactory; -import org.janusgraph.core.JanusGraph; - - - -public class DupeTool { - - private static final String FROMAPPID = "AAI-DB"; - private static final String TRANSID = UUID.randomUUID().toString(); - private static final String COULD_NOT_FIND_EDGE_ID_ = " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = "; - private static final String AAI_NODE_TYPE = "aai-node-type"; - private static final String KEEP_VID = "KeepVid"; - private static final String INMEMORY = "inmemory"; - - private static String graphType = "realdb"; - - public static boolean SHOULD_EXIT_VM = true; - - public static int EXIT_VM_STATUS_CODE = -1; - - public static void exit(int statusCode){ - if(SHOULD_EXIT_VM){ - System.exit(1); - } - EXIT_VM_STATUS_CODE = statusCode; - } - - - /** - * The main method. - * - * @param args the arguments - */ - public static void main(String[] args) { - System.setProperty("aai.service.name", DupeTool.class.getSimpleName()); - // Set the logging file properties to be used by EELFManager - Properties props = System.getProperties(); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "dupeTool-logback.xml"); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); - EELFLogger logger = EELFManager.getInstance().getLogger(DupeTool.class.getSimpleName()); - MDC.put("logFilenameAppender", DupeTool.class.getSimpleName()); - - LoggingContext.init(); - LoggingContext.partnerName(FROMAPPID); - LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.component("dupeTool"); - LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.targetServiceName("main"); - LoggingContext.requestId(TRANSID); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - - String defVersion = "v9"; - try { - defVersion = AAIConfig.get(AAIConstants.AAI_DEFAULT_API_VERSION_PROP); - } - catch ( AAIException ae ){ - String emsg = "Error trying to get default API Version property \n"; - System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logger.error(emsg); - exit(0); - } - - Loader loader= null; - try { - loader = LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST); - - } - catch (Exception ex){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); - logger.error("ERROR - Could not do the moxyMod.init() " + LogFormatTools.getStackTop(ex)); - exit(1); - } - JanusGraph graph1 = null; - JanusGraph graph2 = null; - Graph gt1 = null; - Graph gt2 = null; - - boolean specialTenantRule = false; - - try { - AAIConfig.init(); - int maxRecordsToFix = AAIConstants.AAI_DUPETOOL_DEFAULT_MAX_FIX; - int sleepMinutes = AAIConstants.AAI_DUPETOOL_DEFAULT_SLEEP_MINUTES; - int timeWindowMinutes = 0; // A value of 0 means that we will not have a time-window -- we will look - // at all nodes of the passed-in nodeType. - long windowStartTime = 0; // Translation of the window into a starting timestamp - - try { - String maxFixStr = AAIConfig.get("aai.dupeTool.default.max.fix"); - if( maxFixStr != null && !maxFixStr.isEmpty() ){ - maxRecordsToFix = Integer.parseInt(maxFixStr); - } - String sleepStr = AAIConfig.get("aai.dupeTool.default.sleep.minutes"); - if( sleepStr != null && !sleepStr.isEmpty() ){ - sleepMinutes = Integer.parseInt(sleepStr); - } - } - catch ( Exception e ){ - // Don't worry, we'll just use the defaults that we got from AAIConstants - logger.warn("WARNING - could not pick up aai.dupeTool values from aaiconfig.properties file. Will use defaults. "); - } - - String nodeTypeVal = ""; - String userIdVal = ""; - String filterParams = ""; - Boolean skipHostCheck = false; - Boolean autoFix = false; - String argStr4Msg = ""; - Introspector obj = null; - - if (args != null && args.length > 0) { - // They passed some arguments in that will affect processing - for (int i = 0; i < args.length; i++) { - String thisArg = args[i]; - argStr4Msg = argStr4Msg + " " + thisArg; - - if ("-nodeType".equals(thisArg)) { - i++; - if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error(" No value passed with -nodeType option. "); - exit(0); - } - nodeTypeVal = args[i]; - argStr4Msg = argStr4Msg + " " + nodeTypeVal; - } - else if ("-sleepMinutes".equals(thisArg)) { - i++; - if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error("No value passed with -sleepMinutes option."); - exit(0); - } - String nextArg = args[i]; - try { - sleepMinutes = Integer.parseInt(nextArg); - } catch (Exception e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error("Bad value passed with -sleepMinutes option: [" - + nextArg + "]"); - exit(0); - } - argStr4Msg = argStr4Msg + " " + sleepMinutes; - } - else if ("-maxFix".equals(thisArg)) { - i++; - if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error("No value passed with -maxFix option."); - exit(0); - } - String nextArg = args[i]; - try { - maxRecordsToFix = Integer.parseInt(nextArg); - } catch (Exception e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error("Bad value passed with -maxFix option: [" - + nextArg + "]"); - exit(0); - } - argStr4Msg = argStr4Msg + " " + maxRecordsToFix; - } - else if ("-timeWindowMinutes".equals(thisArg)) { - i++; - if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error("No value passed with -timeWindowMinutes option."); - exit(0); - } - String nextArg = args[i]; - try { - timeWindowMinutes = Integer.parseInt(nextArg); - } catch (Exception e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error("Bad value passed with -timeWindowMinutes option: [" - + nextArg + "]"); - exit(0); - } - argStr4Msg = argStr4Msg + " " + timeWindowMinutes; - } - else if ("-skipHostCheck".equals(thisArg)) { - skipHostCheck = true; - } - else if ("-specialTenantRule".equals(thisArg)) { - specialTenantRule = true; - } - else if ("-autoFix".equals(thisArg)) { - autoFix = true; - } - else if ("-userId".equals(thisArg)) { - i++; - if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error(" No value passed with -userId option. "); - exit(0); - } - userIdVal = args[i]; - argStr4Msg = argStr4Msg + " " + userIdVal; - } - else if ("-params4Collect".equals(thisArg)) { - i++; - if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error(" No value passed with -params4Collect option. "); - exit(0); - } - filterParams = args[i]; - argStr4Msg = argStr4Msg + " " + filterParams; - } - else { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error(" Unrecognized argument passed to DupeTool: [" - + thisArg + "]. "); - logger.error(" Valid values are: -action -userId -vertexId -edgeId -overRideProtection "); - exit(0); - } - } - } - - userIdVal = userIdVal.trim(); - if( (userIdVal.length() < 6) || "AAIADMIN".equalsIgnoreCase(userIdVal) ){ - String emsg = "userId parameter is required. [" + userIdVal + "] passed to DupeTool(). userId must be not empty and not aaiadmin \n"; - System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error(emsg); - exit(0); - } - - nodeTypeVal = nodeTypeVal.trim(); - if( nodeTypeVal.isEmpty() ){ - String emsg = " nodeType is a required parameter for DupeTool().\n"; - System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logger.error(emsg); - exit(0); - } - obj = loader.introspectorFromName(nodeTypeVal); - - if (skipHostCheck) { - logger.info(" We will skip the HostCheck as requested. "); - } - - if( timeWindowMinutes > 0 ){ - // Translate the window value (ie. 30 minutes) into a unix timestamp like - // we use in the db - so we can select data created after that time. - windowStartTime = figureWindowStartTime( timeWindowMinutes ); - } - - String msg = ""; - msg = "DupeTool called with these params: [" + argStr4Msg + "]"; - System.out.println(msg); - logger.info(msg); - - // Determine what the key fields are for this nodeType (and we want them ordered) - ArrayList <String> keyPropNamesArr = new ArrayList<String>(obj.getKeys()); - - // Determine what kinds of nodes (if any) this nodeType is dependent on for uniqueness - ArrayList<String> depNodeTypeList = new ArrayList<String>(); - Collection<String> depNTColl = obj.getDependentOn(); - Iterator<String> ntItr = depNTColl.iterator(); - while( ntItr.hasNext() ){ - depNodeTypeList.add(ntItr.next()); - } - - // Based on the nodeType, window and filterData, figure out the vertices that we will be checking - System.out.println(" ---- NOTE --- about to open graph (takes a little while)--------\n"); - graph1 = setupGraph(logger); - gt1 = getGraphTransaction( graph1, logger ); - ArrayList<Vertex> verts2Check = new ArrayList<Vertex>(); - - verts2Check = figureOutNodes2Check( TRANSID, FROMAPPID, gt1, - nodeTypeVal, windowStartTime, filterParams, logger ); - - if( verts2Check == null || verts2Check.isEmpty() ){ - msg = " No vertices found to check. Used nodeType = [" + nodeTypeVal - + "], windowMinutes = " + timeWindowMinutes - + ", filterData = [" + filterParams + "]."; - logger.info( msg ); - System.out.println( msg ); - exit(0); - } - else { - msg = " Found " + verts2Check.size() + " nodes of type " + nodeTypeVal - + " to check using passed filterParams and windowStartTime. "; - logger.info( msg ); - System.out.println( msg ); - } - - ArrayList<String> firstPassDupeSets = new ArrayList <String>(); - ArrayList <String> secondPassDupeSets = new ArrayList <String>(); - Boolean isDependentOnParent = false; - if( !obj.getDependentOn().isEmpty() ){ - isDependentOnParent = true; - } - - if( isDependentOnParent ){ - firstPassDupeSets = getDupeSets4DependentNodes( TRANSID, FROMAPPID, gt1, - defVersion, nodeTypeVal, verts2Check, keyPropNamesArr, loader, - specialTenantRule, logger ); - } - else { - firstPassDupeSets = getDupeSets4NonDepNodes( TRANSID, FROMAPPID, gt1, - defVersion, nodeTypeVal, verts2Check, keyPropNamesArr, - specialTenantRule, loader, logger ); - } - - msg = " Found " + firstPassDupeSets.size() + " sets of duplicates for this request. "; - logger.info( msg ); - System.out.println( msg ); - if( !firstPassDupeSets.isEmpty() ){ - msg = " Here is what they look like: "; - logger.info( msg ); - System.out.println( msg ); - for( int x = 0; x < firstPassDupeSets.size(); x++ ){ - msg = " Set " + x + ": [" + firstPassDupeSets.get(x) +"] "; - logger.info( msg ); - System.out.println( msg ); - showNodeDetailsForADupeSet(gt1, firstPassDupeSets.get(x), logger); - } - } - - boolean didSomeDeletesFlag = false; - ArrayList <String> dupeSetsToFix = new ArrayList <String> (); - if( autoFix && firstPassDupeSets.isEmpty() ){ - msg = "AutoFix option is on, but no dupes were found on the first pass. Nothing to fix."; - logger.info( msg ); - System.out.println( msg ); - } - else if( autoFix ){ - // We will try to fix any dupes that we can - but only after sleeping for a - // time and re-checking the list of duplicates using a seperate transaction. - try { - msg = "\n\n----------- About to sleep for " + sleepMinutes + " minutes." - + " -----------\n\n"; - logger.info( msg ); - System.out.println( msg ); - int sleepMsec = sleepMinutes * 60 * 1000; - Thread.sleep(sleepMsec); - } catch (InterruptedException ie) { - msg = "\n >>> Sleep Thread has been Interrupted <<< "; - Thread.currentThread().interrupt(); - logger.info( msg ); - System.out.println( msg ); - exit(0); - } - - graph2 = setupGraph(logger); - gt2 = getGraphTransaction( graph2, logger ); - if( isDependentOnParent ){ - secondPassDupeSets = getDupeSets4DependentNodes( TRANSID, FROMAPPID, gt2, - defVersion, nodeTypeVal, verts2Check, keyPropNamesArr, loader, - specialTenantRule, logger ); - } - else { - secondPassDupeSets = getDupeSets4NonDepNodes( TRANSID, FROMAPPID, gt2, - defVersion, nodeTypeVal, verts2Check, keyPropNamesArr, - specialTenantRule, loader, logger ); - } - - dupeSetsToFix = figureWhichDupesStillNeedFixing( firstPassDupeSets, secondPassDupeSets, logger ); - msg = "\nAfter running a second pass, there were " + dupeSetsToFix.size() - + " sets of duplicates that we think can be deleted. "; - logger.info( msg ); - System.out.println( msg ); - if( !dupeSetsToFix.isEmpty()){ - msg = " Here is what the sets look like: "; - logger.info( msg ); - System.out.println( msg ); - for( int x = 0; x < dupeSetsToFix.size(); x++ ){ - msg = " Set " + x + ": [" + dupeSetsToFix.get(x) +"] "; - logger.info( msg ); - System.out.println( msg ); - showNodeDetailsForADupeSet(gt2, dupeSetsToFix.get(x), logger); - } - } - - if( !dupeSetsToFix.isEmpty() ){ - if( dupeSetsToFix.size() > maxRecordsToFix ){ - String infMsg = " >> WARNING >> Dupe list size (" - + dupeSetsToFix.size() - + ") is too big. The maxFix we are using is: " - + maxRecordsToFix - + ". No nodes will be deleted. (use the" - + " -maxFix option to override this limit.)"; - System.out.println(infMsg); - logger.info(infMsg); - } - else { - // Call the routine that fixes known dupes - didSomeDeletesFlag = deleteNonKeepers( gt2, dupeSetsToFix, logger ); - } - } - if( didSomeDeletesFlag ){ - gt2.tx().commit(); - } - } - - } catch (AAIException e) { - logger.error("Caught AAIException while running the dupeTool: " + LogFormatTools.getStackTop(e)); - ErrorLogHelper.logException(e); - } catch (Exception ex) { - logger.error("Caught exception while running the dupeTool: "+ LogFormatTools.getStackTop(ex)); - ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun the dupeTool. "); - } finally { - if (gt1 != null && gt1.tx().isOpen()) { - // We don't change any data with gt1 - so just roll it back so it knows we're done. - try { - gt1.tx().rollback(); - } - catch (Exception ex) { - // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed - logger.warn("WARNING from final gt1.rollback() " + LogFormatTools.getStackTop(ex)); - } - } - - if (gt2 != null && gt2.tx().isOpen()) { - // Any changes that worked correctly should have already done - // their commits. - try { - gt2.tx().rollback(); - } catch (Exception ex) { - // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed - logger.warn("WARNING from final gt2.rollback() " + LogFormatTools.getStackTop(ex)); - } - } - - try { - if( graph1 != null && graph1.isOpen() ){ - closeGraph(graph1, logger); - } - } catch (Exception ex) { - // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{ - logger.warn("WARNING from final graph1.shutdown() " + LogFormatTools.getStackTop(ex)); - } - - try { - if( graph2 != null && graph2.isOpen() ){ - closeGraph(graph2, logger); - } - } catch (Exception ex) { - // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{ - logger.warn("WARNING from final graph2.shutdown() " + LogFormatTools.getStackTop(ex)); - } - } - - exit(0); - - }// end of main() - - - /** - * Collect Duplicate Sets for nodes that are NOT dependent on parent nodes. - * - * @param transId the trans id - * @param fromAppId the from app id - * @param g the g - * @param version the version - * @param nType the n type - * @param passedVertList the passed vert list - * @return the array list - */ - private static ArrayList<String> getDupeSets4NonDepNodes( String transId, - String fromAppId, Graph g, String version, String nType, - ArrayList<Vertex> passedVertList, - ArrayList <String> keyPropNamesArr, - Boolean specialTenantRule, Loader loader, EELFLogger logger ) { - - ArrayList<String> returnList = new ArrayList<String>(); - - // We've been passed a set of nodes that we want to check. - // They are all NON-DEPENDENT nodes meaning that they should be - // unique in the DB based on their KEY DATA alone. So, if - // we group them by their key data - if any key has more than one - // vertex mapped to it, those vertices are dupes. - // - // When we find duplicates, we return then as a String (there can be - // more than one duplicate for one set of key data): - // Each element in the returned arrayList might look like this: - // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we - // couldn't figure out which one to keep) - // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we - // thought the third one was the one that should survive) - - HashMap <String, ArrayList<String>> keyVals2VidHash = new HashMap <String, ArrayList<String>>(); - HashMap <String,Vertex> vtxHash = new HashMap <String,Vertex>(); - Iterator<Vertex> pItr = passedVertList.iterator(); - while (pItr.hasNext()) { - try { - Vertex tvx = pItr.next(); - String thisVid = tvx.id().toString(); - vtxHash.put(thisVid, tvx); - - // if there are more than one vertexId mapping to the same keyProps -- they are dupes - String hKey = getNodeKeyValString( tvx, keyPropNamesArr, logger ); - if( keyVals2VidHash.containsKey(hKey) ){ - // We've already seen this key - ArrayList <String> tmpVL = keyVals2VidHash.get(hKey); - tmpVL.add(thisVid); - keyVals2VidHash.put(hKey, tmpVL); - } - else { - // First time for this key - ArrayList <String> tmpVL = new ArrayList <String>(); - tmpVL.add(thisVid); - keyVals2VidHash.put(hKey, tmpVL); - } - } - catch (Exception e) { - logger.warn(" >>> Threw an error in getDupeSets4NonDepNodes - just absorb this error and move on. " + LogFormatTools.getStackTop(e)); - } - } - - for( Map.Entry<String, ArrayList<String>> entry : keyVals2VidHash.entrySet() ){ - ArrayList <String> vidList = entry.getValue(); - try { - if( !vidList.isEmpty() && vidList.size() > 1 ){ - // There are more than one vertex id's using the same key info - String dupesStr = ""; - ArrayList <Vertex> vertList = new ArrayList <Vertex> (); - for (int i = 0; i < vidList.size(); i++) { - String tmpVid = vidList.get(i); - dupesStr = dupesStr + tmpVid + "|"; - vertList.add(vtxHash.get(tmpVid)); - } - - if (dupesStr != "") { - Vertex prefV = getPreferredDupe(transId, fromAppId, - g, vertList, version, specialTenantRule, loader, logger); - if (prefV == null) { - // We could not determine which duplicate to keep - dupesStr = dupesStr + "KeepVid=UNDETERMINED"; - returnList.add(dupesStr); - } else { - dupesStr = dupesStr + "KeepVid=" + prefV.id(); - returnList.add(dupesStr); - } - } - } - } - catch (Exception e) { - logger.warn(" >>> Threw an error in getDupeSets4NonDepNodes - just absorb this error and move on. " + LogFormatTools.getStackTop(e)); - } - - } - return returnList; - - }// End of getDupeSets4NonDepNodes() - - - /** - * Collect Duplicate Sets for nodes that are dependent on parent nodes. - * - * @param transId the trans id - * @param fromAppId the from app id - * @param g the g - * @param version the version - * @param nType the n type - * @param passedVertList the passed vert list - * @param keyPropNamesArr Array (ordered) of keyProperty names - * @param specialTenantRule flag - * @param logger the logger - * @return the array list - */ - private static ArrayList<String> getDupeSets4DependentNodes( String transId, - String fromAppId, Graph g, String version, String nType, - ArrayList<Vertex> passedVertList, - ArrayList <String> keyPropNamesArr, Loader loader, - Boolean specialTenantRule, EELFLogger logger ) { - - // This is for nodeTypes that DEPEND ON A PARENT NODE FOR UNIQUNESS - - ArrayList<String> returnList = new ArrayList<String>(); - ArrayList<String> alreadyFoundDupeVidArr = new ArrayList<String>(); - - // We've been passed a set of nodes that we want to check. These are - // all nodes that ARE DEPENDENT on a PARENT Node for uniqueness. - // The first thing to do is to identify the key properties for the node-type - // and pull from the db just using those properties. - // Then, we'll check those nodes with their parent nodes to see if there - // are any duplicates. - // - // When we find duplicates, we return then as a String (there can be - // more than one duplicate for one set of key data): - // Each element in the returned arrayList might look like this: - // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we - // couldn't figure out which one to keep) - // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we - // thought the third one was the one that should survive) - HashMap<String, Object> checkVertHash = new HashMap <String,Object> (); - try { - Iterator<Vertex> pItr = passedVertList.iterator(); - while (pItr.hasNext()) { - Vertex tvx = pItr.next(); - String passedId = tvx.id().toString(); - if( !alreadyFoundDupeVidArr.contains(passedId) ){ - // We haven't seen this one before - so we should check it. - HashMap <String,Object> keyPropValsHash = getNodeKeyVals( tvx, keyPropNamesArr, logger ); - ArrayList <Vertex> tmpVertList = getNodeJustUsingKeyParams( transId, fromAppId, g, - nType, keyPropValsHash, version, logger ); - - if( tmpVertList.size() <= 1 ){ - // Even without a parent node, this thing is unique so don't worry about it. - } - else { - for( int i = 0; i < tmpVertList.size(); i++ ){ - Vertex tmpVtx = (tmpVertList.get(i)); - String tmpVid = tmpVtx.id().toString(); - alreadyFoundDupeVidArr.add(tmpVid); - - String hKey = getNodeKeyValString( tmpVtx, keyPropNamesArr, logger ); - if( checkVertHash.containsKey(hKey) ){ - // add it to an existing list - ArrayList <Vertex> tmpVL = (ArrayList <Vertex>)checkVertHash.get(hKey); - tmpVL.add(tmpVtx); - checkVertHash.put(hKey, tmpVL); - } - else { - // First time for this key - ArrayList <Vertex> tmpVL = new ArrayList <Vertex>(); - tmpVL.add(tmpVtx); - checkVertHash.put(hKey, tmpVL); - } - } - } - } - } - - // More than one node have the same key fields since they may - // depend on a parent node for uniqueness. Since we're finding - // more than one, we want to check to see if any of the - // vertices that have this set of keys are also pointing at the - // same 'parent' node. - // Note: for a given set of key data, it is possible that there - // could be more than one set of duplicates. - for (Entry<String, Object> lentry : checkVertHash.entrySet()) { - ArrayList <Vertex> thisIdSetList = (ArrayList <Vertex>)lentry.getValue(); - if (thisIdSetList == null || thisIdSetList.size() < 2) { - // Nothing to check for this set. - continue; - } - - HashMap<String, ArrayList<Vertex>> vertsGroupedByParentHash = groupVertsByDepNodes( - transId, fromAppId, g, version, nType, - thisIdSetList, loader); - for (Map.Entry<String, ArrayList<Vertex>> entry : vertsGroupedByParentHash - .entrySet()) { - ArrayList<Vertex> thisParentsVertList = entry - .getValue(); - if (thisParentsVertList.size() > 1) { - // More than one vertex found with the same key info - // hanging off the same parent/dependent node - String dupesStr = ""; - for (int i = 0; i < thisParentsVertList.size(); i++) { - dupesStr = dupesStr + thisParentsVertList.get(i).id() + "|"; - } - if (dupesStr != "") { - Vertex prefV = getPreferredDupe(transId, - fromAppId, g, thisParentsVertList, - version, specialTenantRule, loader, logger); - - if (prefV == null) { - // We could not determine which duplicate to keep - dupesStr = dupesStr + "KeepVid=UNDETERMINED"; - returnList.add(dupesStr); - } - else { - dupesStr = dupesStr + "KeepVid=" - + prefV.id().toString(); - returnList.add(dupesStr); - } - } - } - } - } - - } catch (Exception e) { - logger.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. " + LogFormatTools.getStackTop(e)); - } - - return returnList; - - }// End of getDupeSets4DependentNodes() - - - private static Graph getGraphTransaction(JanusGraph graph, EELFLogger logger){ - - Graph gt = null; - try { - if( graph != null ){ - gt = graph.newTransaction(); - } else { - String emsg = "could not get graph object in DupeTool. \n"; - System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); - logger.error(emsg); - exit(0); - } - } - catch (Exception e) { - String msg = e.toString(); - System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); - logger.error(msg); - exit(0); - } - - return gt; - - }// End of getGraphTransaction() - - - - public static void showNodeInfo(EELFLogger logger, Vertex tVert, Boolean displayAllVidsFlag ){ - - try { - Iterator<VertexProperty<Object>> pI = tVert.properties(); - String infStr = ">>> Found Vertex with VertexId = " + tVert.id() + ", properties: "; - System.out.println( infStr ); - logger.info(infStr); - while( pI.hasNext() ){ - VertexProperty<Object> tp = pI.next(); - infStr = " [" + tp.key() + "|" + tp.value() + "] "; - System.out.println( infStr ); - logger.info(infStr); - } - - ArrayList <String> retArr = collectEdgeInfoForNode( logger, tVert, displayAllVidsFlag ); - for( String infoStr : retArr ){ - System.out.println( infoStr ); - logger.info(infoStr); - } - } - catch (Exception e){ - String warnMsg = " -- Error -- trying to display edge info. [" + e.getMessage() + "]"; - System.out.println( warnMsg ); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); - logger.warn(warnMsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - } - - }// End of showNodeInfo() - - public static ArrayList <String> collectEdgeInfoForNode( EELFLogger logger, Vertex tVert, boolean displayAllVidsFlag ){ - ArrayList <String> retArr = new ArrayList <String> (); - Direction dir = Direction.OUT; - for ( int i = 0; i <= 1; i++ ){ - if( i == 1 ){ - // Second time through we'll look at the IN edges. - dir = Direction.IN; - } - Iterator <Edge> eI = tVert.edges(dir); - if( ! eI.hasNext() ){ - retArr.add("No " + dir + " edges were found for this vertex. "); - } - while( eI.hasNext() ){ - Edge ed = eI.next(); - String lab = ed.label(); - Vertex vtx = null; - if( dir == Direction.OUT ){ - // get the vtx on the "other" side - vtx = ed.inVertex(); - } - else { - // get the vtx on the "other" side - vtx = ed.outVertex(); - } - if( vtx == null ){ - retArr.add(COULD_NOT_FIND_EDGE_ID_ + ed.id() + " <<< "); - continue; - } - - String nType = vtx.<String>property(AAI_NODE_TYPE).orElse(null); - if( displayAllVidsFlag ){ - // This should rarely be needed - String vid = vtx.id().toString(); - retArr.add("Found an " + dir + " edge (" + lab + ") between this vertex and a [" + nType + "] node with VtxId = " + vid ); - } - else { - // This is the normal case - retArr.add("Found an " + dir + " edge (" + lab + ") between this vertex and a [" + nType + "] node. "); - } - } - } - return retArr; - - }// end of collectEdgeInfoForNode() - - - private static long figureWindowStartTime( int timeWindowMinutes ){ - // Given a window size, calculate what the start-timestamp would be. - - if( timeWindowMinutes <= 0 ){ - // This just means that there is no window... - return 0; - } - long unixTimeNow = System.currentTimeMillis(); - long windowInMillis = timeWindowMinutes * 60l * 1000l; - - return unixTimeNow - windowInMillis; - } // End of figureWindowStartTime() - - - - /** - * Gets the node(s) just using key params. - * - * @param transId the trans id - * @param fromAppId the from app id - * @param graph the graph - * @param nodeType the node type - * @param keyPropsHash the key props hash - * @param apiVersion the api version - * @return the node just using key params - * @throws AAIException the AAI exception - */ - public static ArrayList <Vertex> getNodeJustUsingKeyParams( String transId, String fromAppId, Graph graph, String nodeType, - HashMap<String,Object> keyPropsHash, String apiVersion, EELFLogger logger ) throws AAIException{ - - ArrayList <Vertex> retVertList = new ArrayList <Vertex> (); - - // We assume that all NodeTypes have at least one key-property defined. - // Note - instead of key-properties (the primary key properties), a user could pass - // alternate-key values if they are defined for the nodeType. - ArrayList<String> kName = new ArrayList<String>(); - ArrayList<Object> kVal = new ArrayList<Object>(); - if( keyPropsHash == null || keyPropsHash.isEmpty() ) { - throw new AAIException("AAI_6120", " NO key properties passed for this getNodeJustUsingKeyParams() request. NodeType = [" + nodeType + "]. "); - } - - int i = -1; - for( Map.Entry<String, Object> entry : keyPropsHash.entrySet() ){ - i++; - kName.add(i, entry.getKey()); - kVal.add(i, entry.getValue()); - } - int topPropIndex = i; - Vertex tiV = null; - String propsAndValuesForMsg = ""; - Iterator<Vertex> verts = null; - GraphTraversalSource g = graph.traversal(); - try { - if( topPropIndex == 0 ){ - propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ") "; - verts= g.V().has(kName.get(0),kVal.get(0)).has(AAI_NODE_TYPE,nodeType); - } - else if( topPropIndex == 1 ){ - propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", " - + kName.get(1) + " = " + kVal.get(1) + ") "; - verts = g.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(AAI_NODE_TYPE,nodeType); - } - else if( topPropIndex == 2 ){ - propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", " - + kName.get(1) + " = " + kVal.get(1) + ", " - + kName.get(2) + " = " + kVal.get(2) + ") "; - verts= g.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has(AAI_NODE_TYPE,nodeType); - } - else if( topPropIndex == 3 ){ - propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", " - + kName.get(1) + " = " + kVal.get(1) + ", " - + kName.get(2) + " = " + kVal.get(2) + ", " - + kName.get(3) + " = " + kVal.get(3) + ") "; - verts= g.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has(kName.get(3),kVal.get(3)).has(AAI_NODE_TYPE,nodeType); - } - else { - throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n"); - } - } - catch( Exception ex ){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logger.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "] " + LogFormatTools.getStackTop(ex)); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - } - - if( verts != null ){ - while( verts.hasNext() ){ - tiV = verts.next(); - retVertList.add(tiV); - } - } - - if( retVertList.isEmpty() ){ - logger.debug("DEBUG No node found for nodeType = [" + nodeType + - "], propsAndVal = " + propsAndValuesForMsg ); - } - - return retVertList; - - }// End of getNodeJustUsingKeyParams() - - - - /** - * Gets the node(s) just using key params. - * - * @param transId the trans id - * @param fromAppId the from app id - * @param graph the graph - * @param nodeType the node type - * @param windowStartTime the window start time - * @return the nodes - * @throws AAIException the AAI exception - */ - public static ArrayList <Vertex> figureOutNodes2Check( String transId, String fromAppId, - Graph graph, String nodeType, long windowStartTime, - String propsString, EELFLogger logger ) { - - ArrayList <Vertex> retVertList = new ArrayList <Vertex> (); - String msg = ""; - GraphTraversal<Vertex,Vertex> tgQ = graph.traversal().V().has(AAI_NODE_TYPE,nodeType); - String qStringForMsg = "graph.traversal().V().has(\"aai-node-type\"," + nodeType + ")"; - - if( propsString != null && !propsString.trim().isEmpty() ){ - propsString = propsString.trim(); - int firstPipeLoc = propsString.indexOf("|"); - if( firstPipeLoc <= 0 ){ - msg = "Bad props4Collect passed: [" + propsString + "]. \n Expecting a format like, 'propName1|propVal1,propName2|propVal2'"; - System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logger.error(msg); - exit(0); - } - - // Note - if they're only passing on parameter, there won't be any commas - String [] paramArr = propsString.split(","); - for( int i = 0; i < paramArr.length; i++ ){ - int pipeLoc = paramArr[i].indexOf("|"); - if( pipeLoc <= 0 ){ - msg = "Bad propsString passed: [" + propsString + "]. \n Expecting a format like, 'propName1|propVal1,propName2|propVal2'"; - System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logger.error(msg); - exit(0); - } - else { - String propName = paramArr[i].substring(0,pipeLoc); - String propVal = paramArr[i].substring(pipeLoc + 1); - tgQ = tgQ.has(propName,propVal); - qStringForMsg = qStringForMsg + ".has(" + propName + "," + propVal + ")"; - } - } - } - - if(tgQ != null) { - Iterator<Vertex> vertItor = tgQ; - while (vertItor.hasNext()) { - Vertex tiV = vertItor.next(); - if (windowStartTime <= 0) { - // We're not applying a time-window - retVertList.add(tiV); - continue; - } - - Object objTimeStamp = tiV.property("aai-created-ts").orElse(null); - if (objTimeStamp == null) { - // No timestamp - so just take it - retVertList.add(tiV); - continue; - } - - long thisNodeCreateTime = (long) objTimeStamp; - if (thisNodeCreateTime > windowStartTime) { - // It is in our window, so we can take it - retVertList.add(tiV); - } - } - } else { - msg = "Bad JanusGraphQuery object. "; - System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); - logger.error(msg); - exit(0); - } - - if( retVertList.isEmpty() ){ - logger.debug("DEBUG No node found for: [" + qStringForMsg + ", with aai-created-ts > " + windowStartTime ); - } - - return retVertList; - - }// End of figureOutNodes2Check() - - - /** - * Gets the preferred dupe. - * - * @param transId the trans id - * @param fromAppId the from app id - * @param g the g - * @param dupeVertexList the dupe vertex list - * @param ver the ver - * @param logger the logger - * @return Vertex - * @throws AAIException the AAI exception - */ - public static Vertex getPreferredDupe( String transId, - String fromAppId, Graph g, - ArrayList<Vertex> dupeVertexList, String ver, - Boolean specialTenantRule, Loader loader, EELFLogger logger ) - throws AAIException { - - // This method assumes that it is being passed a List of vertex objects - // which violate our uniqueness constraints. - - Vertex nullVtx = null; - - if (dupeVertexList == null) { - return nullVtx; - } - int listSize = dupeVertexList.size(); - if (listSize == 0) { - return nullVtx; - } - if (listSize == 1) { - return ( dupeVertexList.get(0)); - } - - Vertex vtxPreferred = null; - Vertex currentFaveVtx = dupeVertexList.get(0); - for (int i = 1; i < listSize; i++) { - Vertex vtxB = dupeVertexList.get(i); - vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g, - currentFaveVtx, vtxB, ver, specialTenantRule, loader, logger); - if (vtxPreferred == null) { - // We couldn't choose one - return nullVtx; - } else { - currentFaveVtx = vtxPreferred; - } - } - - return (currentFaveVtx); - - } // end of getPreferredDupe() - - - /** - * Pick one of two dupes. - * - * @param transId the trans id - * @param fromAppId the from app id - * @param g the g - * @param vtxA the vtx A - * @param vtxB the vtx B - * @param ver the ver - * @param specialTenantRule flag - * @param logger the logger - * @return Vertex - * @throws AAIException the AAI exception - */ - public static Vertex pickOneOfTwoDupes(String transId, - String fromAppId, Graph g, Vertex vtxA, - Vertex vtxB, String ver, Boolean specialTenantRule, Loader loader, EELFLogger logger) throws AAIException { - - Vertex nullVtx = null; - Vertex preferredVtx = null; - - Long vidA = new Long(vtxA.id().toString()); - Long vidB = new Long(vtxB.id().toString()); - - String vtxANodeType = ""; - String vtxBNodeType = ""; - Object obj = vtxA.<Object>property(AAI_NODE_TYPE).orElse(null); - if (obj != null) { - vtxANodeType = obj.toString(); - } - obj = vtxB.<Object>property(AAI_NODE_TYPE).orElse(null); - if (obj != null) { - vtxBNodeType = obj.toString(); - } - - if (vtxANodeType.isEmpty() || (!vtxANodeType.equals(vtxBNodeType))) { - // Either they're not really dupes or there's some bad data - so - // don't pick one - return nullVtx; - } - - // Check that node A and B both have the same key values (or else they - // are not dupes) - // (We'll check dep-node later) - Collection<String> keyProps = loader.introspectorFromName(vtxANodeType).getKeys(); - Iterator<String> keyPropI = keyProps.iterator(); - while (keyPropI.hasNext()) { - String propName = keyPropI.next(); - String vtxAKeyPropVal = ""; - obj = vtxA.<Object>property(propName).orElse(null); - if (obj != null) { - vtxAKeyPropVal = obj.toString(); - } - String vtxBKeyPropVal = ""; - obj = vtxB.<Object>property(propName).orElse(null); - if (obj != null) { - vtxBKeyPropVal = obj.toString(); - } - - if (vtxAKeyPropVal.equals("") - || (!vtxAKeyPropVal.equals(vtxBKeyPropVal))) { - // Either they're not really dupes or they are missing some key - // data - so don't pick one - return nullVtx; - } - } - - // Collect the vid's and aai-node-types of the vertices that each vertex - // (A and B) is connected to. - ArrayList<String> vtxIdsConn2A = new ArrayList<String>(); - ArrayList<String> vtxIdsConn2B = new ArrayList<String>(); - HashMap<String, String> nodeTypesConn2A = new HashMap<String, String>(); - HashMap<String, String> nodeTypesConn2B = new HashMap<String, String>(); - - ArrayList <String> retArr = new ArrayList <String> (); - Iterator <Edge> eAI = vtxA.edges(Direction.BOTH); - while( eAI.hasNext() ){ - Edge ed = eAI.next(); - Vertex tmpVtx; - if (vtxA.equals(ed.inVertex())) { - tmpVtx = ed.outVertex(); - } else { - tmpVtx = ed.inVertex(); - } - if( tmpVtx == null ){ - retArr.add(COULD_NOT_FIND_EDGE_ID_ + ed.id() + " <<< "); - } - else { - String conVid = tmpVtx.id().toString(); - String nt = ""; - obj = tmpVtx.<Object>property(AAI_NODE_TYPE).orElse(null); - if (obj != null) { - nt = obj.toString(); - } - nodeTypesConn2A.put(nt, conVid); - vtxIdsConn2A.add(conVid); - } - } - - Iterator <Edge> eBI = vtxB.edges(Direction.BOTH); - while( eBI.hasNext() ){ - Edge ed = eBI.next(); - Vertex tmpVtx; - - if (vtxB.equals(ed.inVertex())) { - tmpVtx = ed.outVertex(); - } else { - tmpVtx = ed.inVertex(); - } - if( tmpVtx == null ){ - retArr.add(COULD_NOT_FIND_EDGE_ID_ + ed.id() + " <<< "); - } - else { - String conVid = tmpVtx.id().toString(); - String nt = ""; - obj = tmpVtx.<Object>property(AAI_NODE_TYPE).orElse(null); - if (obj != null) { - nt = obj.toString(); - } - nodeTypesConn2B.put(nt, conVid); - vtxIdsConn2B.add(conVid); - } - } - - // 1 - If this kind of node needs a dependent node for uniqueness, then - // verify that they both nodes point to the same dependent - // node (otherwise they're not really duplicates) - // Note - there are sometimes more than one dependent node type since - // one nodeType can be used in different ways. But for a - // particular node, it will only have one dependent node that - // it's connected to. - Collection<String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn(); - if (depNodeTypes.isEmpty()) { - // This kind of node is not dependent on any other. That is ok. - } else { - String depNodeVtxId4A = ""; - String depNodeVtxId4B = ""; - Iterator<String> iter = depNodeTypes.iterator(); - while (iter.hasNext()) { - String depNodeType = iter.next(); - if (nodeTypesConn2A.containsKey(depNodeType)) { - // This is the dependent node type that vertex A is using - depNodeVtxId4A = nodeTypesConn2A.get(depNodeType); - } - if (nodeTypesConn2B.containsKey(depNodeType)) { - // This is the dependent node type that vertex B is using - depNodeVtxId4B = nodeTypesConn2B.get(depNodeType); - } - } - if (depNodeVtxId4A.isEmpty() - || (!depNodeVtxId4A.equals(depNodeVtxId4B))) { - // Either they're not really dupes or there's some bad data - so - // don't pick either one - return nullVtx; - } - } - - if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) { - // 2 - If they both have edges to all the same vertices, then return - // the one with the lower vertexId. - - // OR (2b)-- if this is the SPECIAL case -- of - // "tenant|vserver vs. tenant|service-subscription" - // then we pick/prefer the one that's connected to - // the service-subscription. AAI-8172 - boolean allTheSame = true; - Iterator<String> iter = vtxIdsConn2A.iterator(); - while (iter.hasNext()) { - String vtxIdConn2A = iter.next(); - if (!vtxIdsConn2B.contains(vtxIdConn2A)) { - allTheSame = false; - break; - } - } - - if (allTheSame) { - if (vidA < vidB) { - preferredVtx = vtxA; - } else { - preferredVtx = vtxB; - } - } - else if ( specialTenantRule && vtxIdsConn2A.size() == 2 && "tenant".equals(vtxANodeType)){ - // They asked us to apply a special rule if it applies - // We're dealing with two tenant nodes which each just have - // two connections. One must be the parent (cloud-region) - // which we check in step 1 above. If one connects to - // a vserver and the other connects to a service-subscription, - // our special rule is to keep the one connected - // to the - if( nodeTypesConn2A.containsKey("vserver") && nodeTypesConn2B.containsKey("service-subscription") ){ - String infMsg = " WARNING >>> we are using the special tenant rule to choose to " + - " delete tenant vtxId = " + vidA + ", and keep tenant vtxId = " + vidB ; - System.out.println(infMsg); - logger.info( infMsg ); - preferredVtx = vtxB; - } - else if( nodeTypesConn2B.containsKey("vserver") && nodeTypesConn2A.containsKey("service-subscription") ){ - String infMsg = " WARNING >>> we are using the special tenant rule to choose to " + - " delete tenant vtxId = " + vidB + ", and keep tenant vtxId = " + vidA ; - System.out.println(infMsg); - logger.info( infMsg ); - preferredVtx = vtxA; - } - } - } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) { - // 3 - VertexA is connected to more things than vtxB. - // We'll pick VtxA if its edges are a superset of vtxB's edges. - boolean missingOne = false; - Iterator<String> iter = vtxIdsConn2B.iterator(); - while (iter.hasNext()) { - String vtxIdConn2B = iter.next(); - if (!vtxIdsConn2A.contains(vtxIdConn2B)) { - missingOne = true; - break; - } - } - if (!missingOne) { - preferredVtx = vtxA; - } - } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) { - // 4 - VertexB is connected to more things than vtxA. - // We'll pick VtxB if its edges are a superset of vtxA's edges. - boolean missingOne = false; - Iterator<String> iter = vtxIdsConn2A.iterator(); - while (iter.hasNext()) { - String vtxIdConn2A = iter.next(); - if (!vtxIdsConn2B.contains(vtxIdConn2A)) { - missingOne = true; - break; - } - } - if (!missingOne) { - preferredVtx = vtxB; - } - } else { - preferredVtx = nullVtx; - } - - return (preferredVtx); - - } // end of pickOneOfTwoDupes() - - - /** - * Group verts by dep nodes. - * - * @param transId the trans id - * @param fromAppId the from app id - * @param g the g - * @param version the version - * @param nType the n type - * @param passedVertList the passed vert list - * @param loader loader - * @return the hash map - * @throws AAIException the AAI exception - */ - private static HashMap<String, ArrayList<Vertex>> groupVertsByDepNodes( - String transId, String fromAppId, Graph g, String version, - String nType, ArrayList<Vertex> passedVertList, Loader loader) - throws AAIException { - - // Given a list of JanusGraph Vertices, group them together by dependent - // nodes. Ie. if given a list of ip address nodes (assumed to all - // have the same key info) they might sit under several different - // parent vertices. - // Under Normal conditions, there would only be one per parent -- but - // we're trying to find duplicates - so we allow for the case - // where more than one is under the same parent node. - - HashMap<String, ArrayList<Vertex>> retHash = new HashMap<String, ArrayList<Vertex>>(); - - // Find out what types of nodes the passed in nodes can depend on - ArrayList<String> depNodeTypeL = new ArrayList<String>(); - Collection<String> depNTColl = loader.introspectorFromName(nType).getDependentOn(); - Iterator<String> ntItr = depNTColl.iterator(); - while (ntItr.hasNext()) { - depNodeTypeL.add(ntItr.next()); - } - // For each vertex they passed us, we want find the vertex it - // is dependent on so we can keep track of who-all is connected - // to that parent. - if (passedVertList != null) { - Iterator<Vertex> iter = passedVertList.iterator(); - while (iter.hasNext()) { - Vertex thisVert = iter.next(); - Iterator <String> depNtItr = depNTColl.iterator(); - while (depNtItr.hasNext()) { - GraphTraversal<Vertex, Vertex> modPipe = null; - // NOTE -- if we change the direction of parent/child edges, we will need - // the "in" below to become "out" - modPipe = g.traversal().V(thisVert).in().has(AAI_NODE_TYPE, depNtItr.next() ); - if( modPipe == null || !modPipe.hasNext() ){ - //System.out.println("DEBUG - didn't find any [" + targetStep + "] connected to this guy (which is ok)"); - continue; - } - - while( modPipe.hasNext() ){ - Vertex depVert = modPipe.next(); - String parentVid = depVert.id().toString(); - if (retHash.containsKey(parentVid)) { - // add this vert to the list for this parent key - retHash.get(parentVid).add(thisVert); - } else { - // This is the first one we found on this parent - ArrayList<Vertex> vList = new ArrayList<Vertex>(); - vList.add(thisVert); - retHash.put(parentVid, vList); - } - } - } - } - } - - return retHash; - - }// end of groupVertsByDepNodes() - - - /** - * Delete non keepers if appropriate. - * - * @param g the g - * @param dupeInfoList the dupe info string - * @param logger the EELFLogger - * @return the boolean - */ - private static Boolean deleteNonKeepers(Graph g, - ArrayList<String> dupeInfoList, EELFLogger logger ) { - - // This assumes that each dupeInfoString is in the format of - // pipe-delimited vid's followed by either "keepVid=xyz" or "keepVid=UNDETERMINED" - // ie. "3456|9880|keepVid=3456" - - boolean didADelFlag = false; - for( int n = 0; n < dupeInfoList.size(); n++ ){ - String dupeInfoString = dupeInfoList.get(n); - boolean tmpFlag = deleteNonKeeperForOneSet( g, dupeInfoString, logger ); - didADelFlag = tmpFlag || didADelFlag; - } - - return didADelFlag; - - }// end of deleteNonKeepers() - - - /** - * Delete non keepers if appropriate. - * - * @param g the g - * @param dupeInfoString the dupe string - * @param logger the EELFLogger - * @return the boolean - */ - private static Boolean deleteNonKeeperForOneSet(Graph g, - String dupeInfoString, EELFLogger logger ) { - - Boolean deletedSomething = false; - // This assumes that each dupeInfoString is in the format of - // pipe-delimited vid's followed by either "keepVid=xyz" or "keepVid=UNDETERMINED" - // ie. "3456|9880|keepVid=3456" - - - String[] dupeArr = dupeInfoString.split("\\|"); - ArrayList<String> idArr = new ArrayList<String>(); - int lastIndex = dupeArr.length - 1; - for (int i = 0; i <= lastIndex; i++) { - if (i < lastIndex) { - // This is not the last entry, it is one of the dupes, - String vidString = dupeArr[i]; - idArr.add(vidString); - continue; - } - - // This is the last entry which should tell us if we have a - // preferred keeper - String prefString = dupeArr[i]; - if ("KeepVid=UNDETERMINED".equals(prefString)) { - // They sent us a bad string -- nothing should be deleted if - // no dupe could be tagged as preferred. - return false; - } - // If we know which to keep, then the prefString should look - // like, "KeepVid=12345" - String[] prefArr = prefString.split("="); - if (prefArr.length != 2 || !KEEP_VID.equals(prefArr[0])) { - String emsg = "Bad format. Expecting KeepVid=999999"; - System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logger.error(emsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - return false; - } - - String keepVidStr = prefArr[1]; - if (idArr.contains(keepVidStr)) { - idArr.remove(keepVidStr); - // So now, the idArr should just contain the vid's - // that we want to remove. - for (int x = 0; x < idArr.size(); x++) { - boolean okFlag = true; - String thisVid = idArr.get(x); - try { - long longVertId = Long.parseLong(thisVid); - Vertex vtx = g.traversal().V(longVertId).next(); - String msg = "--->>> We will delete node with VID = " + thisVid + " <<<---"; - System.out.println(msg); - logger.info(msg); - vtx.remove(); - } - catch (Exception e) { - okFlag = false; - String emsg = "ERROR trying to delete VID = " + thisVid + ", [" + e + "]"; - System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logger.error(emsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - } - if (okFlag) { - String infMsg = " DELETED VID = " + thisVid; - logger.info(infMsg); - System.out.println(infMsg); - deletedSomething = true; - } - } - } else { - String emsg = "ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = [" - + dupeInfoString + "]"; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logger.error(emsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - System.out.println(emsg); - return false; - } - - }// for each vertex in a group - - return deletedSomething; - - }// end of deleteNonKeeperForOneSet() - - - /** - * Get values of the key properties for a node. - * - * @param tvx the vertex to pull the properties from - * @param keyPropNamesArr ArrayList (ordered) of key prop names - * @param logger the EELFLogger - * @return a hashMap of the propertyNames/values - */ - private static HashMap <String,Object> getNodeKeyVals( Vertex tvx, - ArrayList <String> keyPropNamesArr, EELFLogger logger ) { - - HashMap <String,Object> retHash = new HashMap <String,Object>(); - Iterator <String> propItr = keyPropNamesArr.iterator(); - while( propItr.hasNext() ){ - String propName = propItr.next(); - if( tvx != null ){ - Object propValObj = tvx.property(propName).orElse(null); - retHash.put(propName, propValObj); - } - } - return retHash; - - }// End of getNodeKeyVals() - - - /** - * Get values of the key properties for a node as a single string - * - * @param tvx the vertex to pull the properties from - * @param keyPropNamesArr collection of key prop names - * @param logger the EELFLogger - * @return a String of concatenated values - */ - private static String getNodeKeyValString( Vertex tvx, - ArrayList <String> keyPropNamesArr, EELFLogger logger ) { - - // -- NOTE -- for what we're using this for, we would need to - // guarantee that the properties are always in the same order - - String retString = ""; - Iterator <String> propItr = keyPropNamesArr.iterator(); - while( propItr.hasNext() ){ - String propName = propItr.next(); - if( tvx != null ){ - Object propValObj = tvx.property(propName).orElse(null); - retString = " " + retString + propValObj.toString(); - } - } - return retString; - - }// End of getNodeKeyValString() - - - /** - * Find duplicate sets from two dupe runs. - * - * @param firstPassDupeSets from the first pass - * @param secondPassDupeSets from the second pass - * @param logger logger - * @return commonDupeSets that are common to both passes and have a determined keeper - */ - private static ArrayList <String> figureWhichDupesStillNeedFixing( ArrayList <String>firstPassDupeSets, - ArrayList <String> secondPassDupeSets, EELFLogger logger ){ - - ArrayList <String> common2BothSet = new ArrayList <String> (); - - // We just want to look for entries from the first set which have identical (almost) - // entries in the secondary set. I say "almost" because the order of the - // vid's to delete may be in a different order, but we only want to use it if - // they have all the same values. Note also - we're just looking for - // the sets where we have a candidate to delete. - - // The duplicate-set Strings are in this format: - // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we - // couldn't figure out which one to keep) - // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we - // thought the third one was the one that should survive) - - if( firstPassDupeSets == null || firstPassDupeSets.isEmpty() - || secondPassDupeSets == null || secondPassDupeSets.isEmpty() ){ - // If either set is empty, then our return list has to be empty too - return common2BothSet; - } - - boolean needToParse = false; - for( int x = 0; x < secondPassDupeSets.size(); x++ ){ - String secPassDupeSetStr = secondPassDupeSets.get(x); - if( secPassDupeSetStr.endsWith("UNDETERMINED") ){ - // This is a set of dupes where we could not pick one - // to delete - so don't include it on our list for - // fixing. - } - else if( firstPassDupeSets.contains(secPassDupeSetStr) ){ - // We have lucked out and do not even need to parse this since - // it was in the other array with any dupes listed in the same order - // This is actually the most common scenario since there is - // usually only one dupe, so order doesn't matter. - common2BothSet.add(secPassDupeSetStr); - } - else { - // We'll need to do some parsing to check this one - needToParse = true; - } - } - - if( needToParse ){ - // Make a hash from the first and second Pass data - // where the key is the vid to KEEP and the value is an - // array of (String) vids that would get deleted. - HashMap <String,ArrayList<String>> firstPassHash = makeKeeperHashOfDupeStrings( firstPassDupeSets, common2BothSet, logger ); - - HashMap <String,ArrayList<String>> secPassHash = makeKeeperHashOfDupeStrings( secondPassDupeSets, common2BothSet, logger ); - - // Loop through the secondPass data and keep the ones - // that check out against the firstPass set. - for( Map.Entry<String, ArrayList<String>> entry : secPassHash.entrySet() ){ - boolean skipThisOne = false; - String secKey = entry.getKey(); - ArrayList <String> secList = entry.getValue(); - if( !firstPassHash.containsKey(secKey) ){ - // The second pass found this delete candidate, but not the first pass - skipThisOne = true; - } - else { - // They both think they should keep this VID, check the associated deletes for it. - ArrayList <String> firstList = firstPassHash.get(secKey); - for( int z = 0; z < secList.size(); z++ ){ - if( !firstList.contains(secList.get(z)) ){ - // The first pass did not think this needed to be deleted - skipThisOne = true; - } - } - } - if( skipThisOne ) { - continue; - } - - // Put the string back together and pass it back - // Not beautiful, but no time to make it nice right now... - // Put it back in the format: "3456|9880|keepVid=3456" - String thisDelSetStr = ""; - for( int z = 0; z < secList.size(); z++ ){ - if( z == 0 ){ - thisDelSetStr = secList.get(z); - } - else { - thisDelSetStr = thisDelSetStr + "|" + secList.get(z); - } - } - thisDelSetStr = thisDelSetStr + "|keepVid=" + secKey; - common2BothSet.add(thisDelSetStr); - } - } - return common2BothSet; - - }// figureWhichDupesStillNeedFixing - - - private static HashMap <String, ArrayList <String>> makeKeeperHashOfDupeStrings( ArrayList <String> dupeSets, - ArrayList <String> excludeSets, EELFLogger logger ){ - - HashMap <String,ArrayList<String>> keeperHash = new HashMap <String, ArrayList<String>>(); - - for( int x = 0; x < dupeSets.size(); x++ ){ - String tmpSetStr = dupeSets.get(x); - if( excludeSets.contains(tmpSetStr) ){ - // This isn't one of the ones we needed to parse. - continue; - } - - String[] dupeArr = tmpSetStr.split("\\|"); - ArrayList<String> delIdArr = new ArrayList<String>(); - int lastIndex = dupeArr.length - 1; - for (int i = 0; i <= lastIndex; i++) { - if (i < lastIndex) { - // This is not the last entry, it is one of the dupes - delIdArr.add(dupeArr[i]); - continue; - } - - // This is the last entry which should tell us if we - // have a preferred keeper and how many dupes we had - String prefString = dupeArr[i]; - if( i == 1 ){ - // There was only one dupe, so if we were gonna find - // it, we would have found it above with no parsing. - } - else if ("KeepVid=UNDETERMINED".equals(prefString)) { - // This one had no determined keeper, so we don't - // want it. - } - else { - // If we know which to keep, then the prefString - // should look like, "KeepVid=12345" - String[] prefArr = prefString.split("="); - if( prefArr.length != 2 || !KEEP_VID.equals(prefArr[0]) ) { - String infMsg = "Bad format in figureWhichDupesStillNeedFixing(). Expecting " + - " KeepVid=999999 but string looks like: [" + tmpSetStr + "]"; - System.out.println(infMsg); - logger.info(infMsg); - } - else { - keeperHash.put(prefArr[0], delIdArr); - } - } - - } - } - - return keeperHash; - - }// End makeHashOfDupeStrings() - - - /** - * Get values of the key properties for a node. - * - * @param g the g - * @param dupeInfoString - * @param logger the EELFLogger - * @return void - */ - static private void showNodeDetailsForADupeSet(Graph g, String dupeInfoString, EELFLogger logger) { - - // dang... parsing this string once again... - - String[] dupeArr = dupeInfoString.split("\\|"); - int lastIndex = dupeArr.length - 1; - for (int i = 0; i <= lastIndex; i++) { - if (i < lastIndex) { - // This is not the last entry, it is one of the dupes, - String vidString = dupeArr[i]; - long longVertId = Long.parseLong(vidString); - Vertex vtx = g.traversal().V(longVertId).next(); - showNodeInfo(logger, vtx, false); - } else { - // This is the last entry which should tell us if we have a - // preferred keeper - String prefString = dupeArr[i]; - if (prefString.equals("KeepVid=UNDETERMINED")) { - String msg = " Our algorithm cannot choose from among these, so they will all be kept. -------\n"; - System.out.println(msg); - logger.info(msg); - } else { - // If we know which to keep, then the prefString should look - // like, "KeepVid=12345" - String[] prefArr = prefString.split("="); - if (prefArr.length != 2 || (!prefArr[0].equals(KEEP_VID))) { - String emsg = "Bad format. Expecting KeepVid=999999"; - System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logger.error(emsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - } else { - String keepVidStr = prefArr[1]; - String msg = " vid = " + keepVidStr + " is the one that we would KEEP. ------\n"; - System.out.println(msg); - logger.info(msg); - } - } - } - } - - }// End of showNodeDetailsForADupeSet() - - private static int graphIndex = 1; - - public static JanusGraph setupGraph(EELFLogger logger){ - - JanusGraph janusGraph = null; - - - try (InputStream inputStream = new FileInputStream(AAIConstants.REALTIME_DB_CONFIG);){ - - Properties properties = new Properties(); - properties.load(inputStream); - - if(INMEMORY.equals(properties.get("storage.backend"))){ - janusGraph = AAIGraph.getInstance().getGraph(); - graphType = INMEMORY; - } else { - janusGraph = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(DupeTool.class.getSimpleName()).withGraphType("realtime" + graphIndex).buildConfiguration()); - graphIndex++; - } - } catch (Exception e) { - logger.error("Unable to open the graph", LogFormatTools.getStackTop(e)); - } - - return janusGraph; - } - - public static void closeGraph(JanusGraph graph, EELFLogger logger){ - - try { - if(INMEMORY.equals(graphType)) { - return; - } - if( graph != null && graph.isOpen() ){ - graph.tx().close(); - graph.close(); - } - } catch (Exception ex) { - // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{ - logger.warn("WARNING from final graph.shutdown()", ex); - } - } -} - - |