From cfa1951bcb9636da23d51fcdf89a5171b37096ba Mon Sep 17 00:00:00 2001 From: "Muller, Andrew (am8383)" Date: Fri, 11 Aug 2017 14:56:15 -0400 Subject: [AAI-154 Amsterdam] Check in titan refactor Change-Id: I277417cfa608228f107ac94e0253a83667602040 Signed-off-by: Muller, Andrew (am8383) --- .../java/org/openecomp/aai/dbgen/DataGrooming.java | 64 ++++++++++++---------- .../org/openecomp/aai/dbgen/SchemaGenerator.java | 5 +- .../java/org/openecomp/aai/dbmap/AAIGraph.java | 8 +-- .../aai/serialization/engines/TitanDBEngine.java | 2 - .../engines/TransactionalGraphEngine.java | 14 ++--- .../openecomp/aai/util/UniquePropertyCheck.java | 31 ++++++----- 6 files changed, 66 insertions(+), 58 deletions(-) (limited to 'aai-core/src/main/java/org') diff --git a/aai-core/src/main/java/org/openecomp/aai/dbgen/DataGrooming.java b/aai-core/src/main/java/org/openecomp/aai/dbgen/DataGrooming.java index 2acfe904..88af8719 100644 --- a/aai-core/src/main/java/org/openecomp/aai/dbgen/DataGrooming.java +++ b/aai-core/src/main/java/org/openecomp/aai/dbgen/DataGrooming.java @@ -43,6 +43,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Property; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; @@ -64,10 +65,8 @@ import org.openecomp.aai.util.FormatDate; import com.att.eelf.configuration.Configuration; import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; -import com.thinkaurelius.titan.core.TitanEdge; import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.TitanTransaction; public class DataGrooming { @@ -319,8 +318,8 @@ public class DataGrooming { boolean executeFinalCommit = false; Set deleteCandidateList = new LinkedHashSet<>(); Set processedVertices = new LinkedHashSet<>(); - TitanTransaction g = null; - TitanTransaction g2 = null; + Graph g = null; + Graph g2 = null; try { AAIConfig.init(); String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP @@ -612,7 +611,7 @@ public class DataGrooming { if ( (thisNtDeleteCount > 0) && singleCommits ) { // NOTE - the singleCommits option is not used in normal processing - g.commit(); + g.tx().commit(); g = AAIGraph.getInstance().getGraph().newTransaction(); } @@ -650,8 +649,7 @@ public class DataGrooming { } ArrayList vertList = new ArrayList<>(); - Iterable vIt3 = g.query().vertices(); - Iterator vItor3 = vIt3.iterator(); + Iterator vItor3 = g.traversal().V(); // Gotta hold these in a List - or else HBase times out as you cycle // through these while (vItor3.hasNext()) { @@ -731,7 +729,7 @@ public class DataGrooming { } if( ! ghost2CheckOff ){ - Vertex connectedVert = g2.getVertex(vIdLong); + Vertex connectedVert = g2.traversal().V(vIdLong).next(); if( connectedVert == null ) { LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong); cantGetUsingVid = true; @@ -740,7 +738,7 @@ public class DataGrooming { // it is still a ghost since even though we can get data about it using the FIRST graph // object. try { - ghost2 = g.getVertex(vIdLong); + ghost2 = g.traversal().V(vIdLong).next(); } catch( Exception ex){ LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex); @@ -773,7 +771,7 @@ public class DataGrooming { } if (singleCommits) { // NOTE - the singleCommits option is not used in normal processing - g.commit(); + g.tx().commit(); g = AAIGraph.getInstance().getGraph().newTransaction(); } deleteCount++; @@ -793,7 +791,7 @@ public class DataGrooming { e.remove(); if (singleCommits) { // NOTE - the singleCommits option is not used in normal processing - g.commit(); + g.tx().commit(); g = AAIGraph.getInstance().getGraph().newTransaction(); } deleteCount++; @@ -844,13 +842,13 @@ public class DataGrooming { } if( ! ghost2CheckOff ){ - Vertex connectedVert = g2.getVertex(vIdLong); + Vertex connectedVert = g2.traversal().V(vIdLong).next(); if( connectedVert == null ) { cantGetUsingVid = true; LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong); // If we can get this ghost with the other graph-object, then get it -- it's still a ghost try { - ghost2 = g.getVertex(vIdLong); + ghost2 = g.traversal().V(vIdLong).next(); } catch( Exception ex){ LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex); @@ -882,7 +880,7 @@ public class DataGrooming { } if (singleCommits) { // NOTE - the singleCommits option is not used in normal processing - g.commit(); + g.tx().commit(); g = AAIGraph.getInstance().getGraph().newTransaction(); } deleteCount++; @@ -902,7 +900,7 @@ public class DataGrooming { e.remove(); if (singleCommits) { // NOTE - the singleCommits option is not used in normal processing - g.commit(); + g.tx().commit(); g = AAIGraph.getInstance().getGraph().newTransaction(); } deleteCount++; @@ -1210,25 +1208,25 @@ public class DataGrooming { } } - if (g != null && !g.isClosed()) { + if (g != null && g.tx().isOpen()) { // Any changes that worked correctly should have already done // their commits. try { if (executeFinalCommit) { - g.commit(); + g.tx().commit(); } - g.rollback(); + g.tx().rollback(); } catch (Exception ex) { // Don't throw anything because Titan sometimes is just saying that the graph is already closed LOGGER.warn("WARNING from final graphTransaction.rollback()", ex); } } - if (g2 != null && !g2.isClosed()) { + if (g2 != null && g2.tx().isOpen()) { // Any changes that worked correctly should have already done // their commits. try { - g2.rollback(); + g2.tx().rollback(); } catch (Exception ex) { // Don't throw anything because Titan sometimes is just saying that the graph is already closed LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex); @@ -1665,7 +1663,7 @@ public class DataGrooming { * @return the array list */ private static List checkAndProcessDupes(String transId, - String fromAppId, TitanTransaction g, GraphTraversalSource source, String version, String nType, + String fromAppId, Graph g, GraphTraversalSource source, String version, String nType, List passedVertList, Boolean dupeFixOn, Set deleteCandidateList, Boolean singleCommits, ArrayList alreadyFoundDupeGroups, Loader loader ) { @@ -1909,7 +1907,7 @@ public class DataGrooming { * @param singleCommits the single commits * @return the boolean */ - private static Boolean deleteNonKeepersIfAppropriate(TitanTransaction g, + private static Boolean deleteNonKeepersIfAppropriate(Graph g, String dupeInfoString, String vidToKeep, Set deleteCandidateList, Boolean singleCommits) { @@ -1963,11 +1961,11 @@ public class DataGrooming { long longVertId = Long .parseLong(thisVid); Vertex vtx = g - .getVertex(longVertId); + .traversal().V(longVertId).next(); vtx.remove(); if (singleCommits) { // NOTE - the singleCommits option is not used in normal processing - g.commit(); + g.tx().commit(); g = AAIGraph.getInstance().getGraph().newTransaction(); } } catch (Exception e) { @@ -2095,9 +2093,14 @@ public class DataGrooming { retArr.add("No IN edges were found for this vertex. "); } while( eI.hasNext() ){ - TitanEdge ed = (TitanEdge) eI.next(); + Edge ed = eI.next(); String lab = ed.label(); - Vertex vtx = ed.otherVertex(tVert); + Vertex vtx; + if (tVert.equals(ed.inVertex())) { + vtx = ed.outVertex(); + } else { + vtx = ed.inVertex(); + } if( vtx == null ){ retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< "); } @@ -2114,9 +2117,14 @@ public class DataGrooming { retArr.add("No OUT edges were found for this vertex. "); } while( eI.hasNext() ){ - TitanEdge ed = (TitanEdge) eI.next(); + Edge ed = eI.next(); String lab = ed.label(); - Vertex vtx = ed.otherVertex(tVert); + Vertex vtx; + if (tVert.equals(ed.inVertex())) { + vtx = ed.outVertex(); + } else { + vtx = ed.inVertex(); + } if( vtx == null ){ retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< "); } diff --git a/aai-core/src/main/java/org/openecomp/aai/dbgen/SchemaGenerator.java b/aai-core/src/main/java/org/openecomp/aai/dbgen/SchemaGenerator.java index c5186b01..9f85d029 100644 --- a/aai-core/src/main/java/org/openecomp/aai/dbgen/SchemaGenerator.java +++ b/aai-core/src/main/java/org/openecomp/aai/dbgen/SchemaGenerator.java @@ -28,9 +28,7 @@ import java.util.Optional; import java.util.Set; import org.apache.tinkerpop.gremlin.structure.Vertex; - import org.openecomp.aai.db.props.AAIProperties; -import org.openecomp.aai.exceptions.AAIException; import org.openecomp.aai.introspection.Introspector; import org.openecomp.aai.introspection.Loader; import org.openecomp.aai.introspection.LoaderFactory; @@ -39,6 +37,7 @@ import org.openecomp.aai.schema.enums.PropertyMetadata; import org.openecomp.aai.serialization.db.EdgeRule; import org.openecomp.aai.serialization.db.EdgeRules; import org.openecomp.aai.util.AAIConfig; + import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import com.google.common.collect.Multimap; @@ -185,7 +184,7 @@ public class SchemaGenerator{ String imsg = "-- About to call graphMgmt commit"; System.out.println(imsg); LOGGER.info(imsg); - + graphMgmt.commit(); }// End of loadSchemaIntoTitan() diff --git a/aai-core/src/main/java/org/openecomp/aai/dbmap/AAIGraph.java b/aai-core/src/main/java/org/openecomp/aai/dbmap/AAIGraph.java index 72beb852..b3cc8994 100644 --- a/aai-core/src/main/java/org/openecomp/aai/dbmap/AAIGraph.java +++ b/aai-core/src/main/java/org/openecomp/aai/dbmap/AAIGraph.java @@ -29,16 +29,16 @@ import java.util.Map; import java.util.Properties; import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.io.IoCore; - import org.openecomp.aai.dbgen.SchemaGenerator; import org.openecomp.aai.exceptions.AAIException; import org.openecomp.aai.util.AAIConstants; + import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.TitanTransaction; import com.thinkaurelius.titan.core.schema.TitanManagement; /** @@ -110,9 +110,9 @@ public class AAIGraph { try { String location = System.getProperty("snapshot.location"); logAndPrint(logger, "Loading snapshot to inmemory graph."); - TitanTransaction transaction = graph.newTransaction(); + Graph transaction = graph.newTransaction(); transaction.io(IoCore.graphson()).readGraph(location); - transaction.commit(); + transaction.tx().commit(); logAndPrint(logger, "Snapshot loaded to inmemory graph."); } catch (IOException e) { graph.close(); diff --git a/aai-core/src/main/java/org/openecomp/aai/serialization/engines/TitanDBEngine.java b/aai-core/src/main/java/org/openecomp/aai/serialization/engines/TitanDBEngine.java index 55054f1b..8611ba6b 100644 --- a/aai-core/src/main/java/org/openecomp/aai/serialization/engines/TitanDBEngine.java +++ b/aai-core/src/main/java/org/openecomp/aai/serialization/engines/TitanDBEngine.java @@ -26,11 +26,9 @@ import java.util.List; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; - import org.openecomp.aai.dbmap.DBConnectionType; import org.openecomp.aai.introspection.Loader; import org.openecomp.aai.serialization.db.TitanGraphSingleton; -import com.thinkaurelius.titan.core.TitanVertex; public class TitanDBEngine extends TransactionalGraphEngine { diff --git a/aai-core/src/main/java/org/openecomp/aai/serialization/engines/TransactionalGraphEngine.java b/aai-core/src/main/java/org/openecomp/aai/serialization/engines/TransactionalGraphEngine.java index 3f259aed..9192b589 100644 --- a/aai-core/src/main/java/org/openecomp/aai/serialization/engines/TransactionalGraphEngine.java +++ b/aai-core/src/main/java/org/openecomp/aai/serialization/engines/TransactionalGraphEngine.java @@ -25,8 +25,8 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ReadOnlyStrategy; +import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; - import org.openecomp.aai.dbmap.DBConnectionType; import org.openecomp.aai.introspection.Loader; import org.openecomp.aai.query.builder.GremlinTraversal; @@ -36,8 +36,8 @@ import org.openecomp.aai.query.builder.TraversalQuery; import org.openecomp.aai.serialization.db.GraphSingleton; import org.openecomp.aai.serialization.engines.query.GraphTraversalQueryEngine; import org.openecomp.aai.serialization.engines.query.QueryEngine; + import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.TitanTransaction; import com.thinkaurelius.titan.core.schema.TitanManagement; public abstract class TransactionalGraphEngine { @@ -48,7 +48,7 @@ public abstract class TransactionalGraphEngine { protected QueryStyle style = null; protected final DBConnectionType connectionType; protected final Loader loader; - protected TitanTransaction currentTx = null; + protected Graph currentTx = null; protected GraphTraversalSource currentTraversal = null; protected GraphTraversalSource readOnlyTraversal = null; private final Admin admin; @@ -184,7 +184,7 @@ public abstract class TransactionalGraphEngine { return queryBuilder; } - public TitanTransaction startTransaction() { + public Graph startTransaction() { if (this.tx() == null) { this.currentTx = this.getGraph().newTransaction(); this.currentTraversal = this.tx().traversal(); @@ -195,7 +195,7 @@ public abstract class TransactionalGraphEngine { public void rollback() { if (this.tx() != null) { - this.tx().rollback(); + this.tx().tx().rollback(); this.currentTx = null; this.currentTraversal = null; this.readOnlyTraversal = null; @@ -203,14 +203,14 @@ public abstract class TransactionalGraphEngine { } public void commit() { if (this.tx() != null) { - this.tx().commit(); + this.tx().tx().commit(); this.currentTx = null; this.currentTraversal = null; this.readOnlyTraversal = null; } } - public TitanTransaction tx() { + public Graph tx() { return this.currentTx; } diff --git a/aai-core/src/main/java/org/openecomp/aai/util/UniquePropertyCheck.java b/aai-core/src/main/java/org/openecomp/aai/util/UniquePropertyCheck.java index b5c5cbc5..a2b77506 100644 --- a/aai-core/src/main/java/org/openecomp/aai/util/UniquePropertyCheck.java +++ b/aai-core/src/main/java/org/openecomp/aai/util/UniquePropertyCheck.java @@ -27,18 +27,17 @@ import java.util.UUID; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.openecomp.aai.exceptions.AAIException; import org.slf4j.MDC; -import org.openecomp.aai.exceptions.AAIException; import com.att.eelf.configuration.Configuration; import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; -import com.thinkaurelius.titan.core.TitanEdge; import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.TitanTransaction; -import com.thinkaurelius.titan.core.TitanVertex; @@ -70,7 +69,7 @@ public class UniquePropertyCheck { System.exit(1); } String propertyName = args[0]; - TitanTransaction graph = null; + Graph graph = null; try { AAIConfig.init(); @@ -115,7 +114,7 @@ public class UniquePropertyCheck { * @param logger the logger * @return the boolean */ - public static Boolean runTheCheckForUniqueness( String transId, String fromAppId, TitanTransaction graph, + public static Boolean runTheCheckForUniqueness( String transId, String fromAppId, Graph graph, String propertyName, EELFLogger logger ){ // Note - property can be found in more than one nodetype @@ -128,11 +127,10 @@ public class UniquePropertyCheck { int propCount = 0; int dupeCount = 0; - Iterable vertItr = graph.query().has(propertyName).vertices(); - Iterator vertItor = vertItr.iterator(); + Iterator vertItor = graph.traversal().V().has(propertyName); while( vertItor.hasNext() ){ propCount++; - TitanVertex v = (TitanVertex)vertItor.next(); + Vertex v = vertItor.next(); String thisVid = v.id().toString(); Object val = (v.property(propertyName)).orElse(null); if( valuesAndVidHash.containsKey(val) ){ @@ -174,7 +172,7 @@ public class UniquePropertyCheck { for( int i = 0; i < vidArr.length; i++ ){ String vidString = vidArr[i]; Long idLong = Long.valueOf(vidString); - TitanVertex tvx = (TitanVertex)graph.getVertex(idLong); + Vertex tvx = graph.traversal().V(idLong).next(); showPropertiesAndEdges( TRANSID, FROMAPPID, tvx, logger ); } } @@ -185,7 +183,7 @@ public class UniquePropertyCheck { } finally { if( graph != null ){ - graph.rollback(); + graph.tx().rollback(); } } @@ -202,7 +200,7 @@ public class UniquePropertyCheck { * @param tVert the t vert * @param logger the logger */ - private static void showPropertiesAndEdges( String transId, String fromAppId, TitanVertex tVert, + private static void showPropertiesAndEdges( String transId, String fromAppId, Vertex tVert, EELFLogger logger ){ if( tVert == null ){ @@ -232,9 +230,14 @@ public class UniquePropertyCheck { logAndPrint(logger, "No edges were found for this vertex. "); } while( eI.hasNext() ){ - TitanEdge ed = (TitanEdge) eI.next(); + Edge ed = eI.next(); String lab = ed.label(); - TitanVertex vtx = (TitanVertex) ed.otherVertex(tVert); + Vertex vtx; + if (tVert.equals(ed.inVertex())) { + vtx = ed.outVertex(); + } else { + vtx = ed.inVertex(); + } if( vtx == null ){ logAndPrint(logger, " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< "); } -- cgit 1.2.3-korg