From 6406d19ed660371107aec3317856a492f6e22d0f Mon Sep 17 00:00:00 2001 From: Fiete Ostkamp Date: Wed, 12 Jun 2024 14:00:53 +0200 Subject: Update aai-common to 1.14.0 in graphadmin - update aai-common to 1.14.0 which contains Janusgraph update to 0.3.3 - introduce -v parameter to the datasnapshot scripts to specify graphson version Issue-ID: AAI-3871 Change-Id: I4efac6b8ef6cf6f5b264677c5a85e38cd226a194 Signed-off-by: Fiete Ostkamp --- pom.xml | 44 ++-- src/main/java/org/onap/aai/GraphAdminApp.java | 6 - .../org/onap/aai/datasnapshot/DataSnapshot.java | 33 ++- .../aai/datasnapshot/DataSnapshot4HistInit.java | 248 +++++++++++---------- .../java/org/onap/aai/dbgen/GraphSONPartialIO.java | 15 +- .../org/onap/aai/dbgen/schemamod/SchemaMod.java | 20 +- .../onap/aai/dbgen/schemamod/SchemaMod4Hist.java | 14 +- .../dbgen/schemamod/SchemaModInternal4Hist.java | 37 +-- .../etc/appprops/janusgraph-realtime.properties | 4 +- src/main/resources/logback.xml | 2 +- .../configuration/janusgraph.internal.properties | 6 - .../migration/v12/MigrateHUBEvcInventoryTest.java | 2 +- .../v12/MigrateINVPhysicalInventoryMethodTest.java | 2 +- .../v12/MigrateINVPhysicalInventoryTest.java | 2 +- .../migration/v12/MigrateInvEvcInventoryTest.java | 2 +- .../migration/v12/MigratePATHEvcInventoryTest.java | 2 +- .../migration/v12/MigrateSAREvcInventoryTest.java | 2 +- .../v13/MigrateForwarderEvcCircuitIdTest.java | 2 +- .../MigrateInstanceGroupModelInvariantIdTest.java | 2 +- .../MigrateInstanceGroupModelVersionIdTest.java | 2 +- .../v13/MigrateVnfcModelInvariantIdTest.java | 2 +- .../v13/MigrateVnfcModelVersionIdTest.java | 2 +- .../MigrateSameSourcedRCTROPServerDataTest.java | 2 +- .../migration/v14/MigrateSdnaIvlanDataTest.java | 2 +- ...serverDedupWithDifferentSourcesOfTruthTest.java | 2 +- .../migration/v15/MigrateRadcomChangesTest.java | 2 +- src/test/resources/logback.xml | 12 +- 27 files changed, 232 insertions(+), 239 deletions(-) delete mode 100644 src/main/resources/org/janusgraph/graphdb/configuration/janusgraph.internal.properties diff --git a/pom.xml b/pom.xml index 5a3e1db..8d17076 100755 --- a/pom.xml +++ b/pom.xml @@ -26,11 +26,11 @@ org.onap.aai.aai-common aai-parent - 1.13.5 + 1.14.0 org.onap.aai.graphadmin aai-graphadmin - 1.13.7-SNAPSHOT + 1.14.0-SNAPSHOT @@ -55,7 +55,7 @@ localhost:5000 1.0.0 1.12.4 - 1.13.6 + 1.14.0 ${project.build.directory}/${project.artifactId}-${project.version}-build/ onap @@ -86,7 +86,7 @@ ${project.basedir}/snapshots/int1-data.graphson 0.46 - + onap N/A @@ -108,6 +108,7 @@ 1.8.1 5.16.7 4.9.3 + 3.3.3 @@ -691,18 +692,6 @@ spring-boot-starter-test test - - - org.junit.vintage - junit-vintage-engine - test - - - org.hamcrest - hamcrest-core - - - @@ -949,22 +938,19 @@ - org.apache.maven.plugins - maven-surefire-plugin - - - alphabetical - - + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0 + + + -Xmx4096m + alphabetical + + + org.springframework.boot spring-boot-maven-plugin - - 1.5.21.RELEASE ${start-class} ZIP diff --git a/src/main/java/org/onap/aai/GraphAdminApp.java b/src/main/java/org/onap/aai/GraphAdminApp.java index 1339604..ebb4634 100644 --- a/src/main/java/org/onap/aai/GraphAdminApp.java +++ b/src/main/java/org/onap/aai/GraphAdminApp.java @@ -19,20 +19,15 @@ */ package org.onap.aai; -import com.att.eelf.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.onap.aai.aailog.logs.AaiDebugLog; import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbmap.AAIGraph; -import java.util.Properties; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.ErrorLogHelper; -import org.onap.aai.logging.LogFormatTools; import org.onap.aai.nodes.NodeIngestor; import org.onap.aai.util.AAIConfig; -import org.onap.aai.util.AAIConstants; import org.onap.aai.util.ExceptionTranslator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; @@ -48,7 +43,6 @@ import org.springframework.scheduling.annotation.EnableScheduling; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.UUID; @SpringBootApplication // Scan the specific packages that has the beans/components diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java index cdb858e..5015213 100644 --- a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java +++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java @@ -43,6 +43,8 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.io.IoCore; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.JanusGraphFactory; import org.janusgraph.core.util.JanusGraphCleanup; @@ -145,7 +147,6 @@ public class DataSnapshot { } long maxNodesPerFile4Create = cArgs.maxNodesPerFile; - // cArgs.snapshotType = "graphson"; Long vertAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS; itemName = "aai.datasnapshot.vertex.add.delay.ms"; try { @@ -353,9 +354,16 @@ public class DataSnapshot { new File(targetDir).mkdirs(); LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while) "); + String graphsonVersionArg = cArgs.graphsonVersion; + if (graphsonVersionArg != GraphSONVersion.V1_0.getVersion() && graphsonVersionArg != GraphSONVersion.V2_0.getVersion() && graphsonVersionArg != GraphSONVersion.V3_0.getVersion()) { + LOGGER.warn("The graphsonVersion argument was not in the correct format, defaulting to V1. Argument value should be either one of [{}|{}|{}]", GraphSONVersion.V1_0,GraphSONVersion.V2_0,GraphSONVersion.V3_0); + graphsonVersionArg = GraphSONVersion.V1_0.getVersion(); + } + GraphSONVersion graphsonVersion = GraphSONVersion.valueOf("V" + graphsonVersionArg.replace(".", "_")); + if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT")) && threadCount4Create == 1 ){ - graph = writeSnapshotFile(command, targetDir, snapshotType); + graph = writeSnapshotFile(command, targetDir, snapshotType, graphsonVersion); } else if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT")) @@ -765,13 +773,13 @@ public class DataSnapshot { } // Now add inputStreams.elements() to the Vector, - // inputStreams.elements() will return Enumerations - InputStream sis = new SequenceInputStream(inputStreamsV.elements()); - LOGGER.debug("Begin loading data from " + fCount + " files -----"); - if("gryo".equalsIgnoreCase(snapshotType)){ + // inputStreams.elements() will return Enumerations + InputStream sis = new SequenceInputStream(inputStreamsV.elements()); + LOGGER.debug("Begin loading data from " + fCount + " files -----"); + if("gryo".equalsIgnoreCase(snapshotType)){ graph.io(IoCore.gryo()).reader().create().readGraph(sis, graph); } else { - graph.io(IoCore.graphson()).reader().create().readGraph(sis, graph); + graph.io(GraphSONIo.build(graphsonVersion)).reader().create().readGraph(sis, graph); } LOGGER.debug("Completed the inputGraph command, now try to commit()... "); graph.tx().commit(); @@ -823,7 +831,7 @@ public class DataSnapshot { } - private JanusGraph writeSnapshotFile(String command, String targetDir, String format) throws IOException { + private JanusGraph writeSnapshotFile(String command, String targetDir, String format, GraphSONVersion graphsonVersion) throws IOException { JanusGraph graph; // ------------------------------------------------------------------------------- // They want to take a snapshot on a single thread and have it go in a single file @@ -841,12 +849,14 @@ public class DataSnapshot { String newSnapshotOutFname = null; long timeA = System.nanoTime(); - if(format == "gryo") { + if("gryo".equalsIgnoreCase(format)) { + LOGGER.debug("Exporting snapshot in gryo format"); newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "snapshot_" + dteStr + ".gryo"; graph.io(IoCore.gryo()).writeGraph(newSnapshotOutFname); } else { + LOGGER.debug("Exporting snapshot in graphson format"); newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr; - graph.io(IoCore.graphson()).writeGraph(newSnapshotOutFname); + graph.io(GraphSONIo.build(graphsonVersion)).writeGraph(newSnapshotOutFname); } LOGGER.debug("Snapshot written to " + newSnapshotOutFname); long timeB = System.nanoTime(); @@ -1048,6 +1058,9 @@ public class DataSnapshot { @Parameter(names = "-f", description = "previous snapshot file to reload") public String oldFileName = ""; + @Parameter(names = "-v", description = "Graphson version of the snapshot file. Defaults to 1") + public String graphsonVersion = GraphSONVersion.V1_0.getVersion(); + @Parameter(names = "-snapshotType", description = "snapshot type of gryo or graphson") public String snapshotType = "graphson"; diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java index 35448e4..43c31e1 100644 --- a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java +++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java @@ -44,6 +44,8 @@ import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.tinkerpop.gremlin.structure.io.IoCore; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.JanusGraphFactory; import org.janusgraph.core.util.JanusGraphCleanup; @@ -83,9 +85,9 @@ public class DataSnapshot4HistInit { private static final Set SNAPSHOT_RELOAD_COMMANDS = new HashSet<>(); private static final String MIGRATION_PROCESS_NAME = "migration"; - + private static boolean historyEnabled; - + private LoaderFactory loaderFactory; private SchemaVersions schemaVersions; @@ -93,14 +95,14 @@ public class DataSnapshot4HistInit { SNAPSHOT_RELOAD_COMMANDS.add("RELOAD_DATA"); SNAPSHOT_RELOAD_COMMANDS.add("RELOAD_DATA_MULTI"); } - + private CommandLineArgs cArgs; - + public DataSnapshot4HistInit(LoaderFactory loaderFactory, SchemaVersions schemaVersions){ this.loaderFactory = loaderFactory; this.schemaVersions = schemaVersions; } - + /** * The main method. * @@ -108,14 +110,14 @@ public class DataSnapshot4HistInit { * the arguments */ public static void main(String[] args) { - + // Set the logging file properties to be used by EELFManager System.setProperty("aai.service.name", DataSnapshot4HistInit.class.getSimpleName()); Properties props = System.getProperties(); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); - + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); initializer.initialize(ctx); @@ -130,20 +132,20 @@ public class DataSnapshot4HistInit { ErrorLogHelper.logException(ae); AAISystemExitUtil.systemExitCloseAAIGraph(1); } - + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); if( !historyEnabled ) { String emsg = "Error - DataSnapshot4HistInit may only be used when history.enabled=true. "; System.out.println(emsg); AAIException ae = new AAIException("AAI_6128", emsg); - ErrorLogHelper.logException(ae); + ErrorLogHelper.logException(ae); AAISystemExitUtil.systemExitCloseAAIGraph(1); } - + LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions"); DataSnapshot4HistInit dataSnapshotHI = new DataSnapshot4HistInit(loaderFactory, schemaVersions); - + boolean success = dataSnapshotHI.executeCommand(args); if(success){ AAISystemExitUtil.systemExitCloseAAIGraph(0); @@ -155,14 +157,14 @@ public class DataSnapshot4HistInit { public boolean executeCommand(String[] args) { - + Boolean dbClearFlag = false; JanusGraph graph = null; - String command = "UNKNOWN"; + String command = "UNKNOWN"; String oldSnapshotFileName = ""; boolean success = true; - - cArgs = new CommandLineArgs(); + + cArgs = new CommandLineArgs(); String itemName = "aai.datasnapshot.threads.for.create"; try { String val = AAIConfig.get(itemName); @@ -184,7 +186,7 @@ public class DataSnapshot4HistInit { LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); } long maxNodesPerFile4Create = cArgs.maxNodesPerFile; - + cArgs.snapshotType = "graphson"; Long vertAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS; itemName = "aai.datasnapshot.vertex.add.delay.ms"; @@ -196,7 +198,7 @@ public class DataSnapshot4HistInit { }catch ( Exception e ){ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); } - + Long edgeAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_EDGE_ADD_DELAY_MS; itemName = "aai.datasnapshot.edge.add.delay.ms"; try { @@ -207,7 +209,7 @@ public class DataSnapshot4HistInit { }catch ( Exception e ){ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); } - + Long failureDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_FAILURE_DELAY_MS; itemName = "aai.datasnapshot.failure.delay.ms"; try { @@ -218,7 +220,7 @@ public class DataSnapshot4HistInit { }catch ( Exception e ){ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); } - + Long retryDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_RETRY_DELAY_MS; itemName = "aai.datasnapshot.retry.delay.ms"; try { @@ -229,7 +231,7 @@ public class DataSnapshot4HistInit { }catch ( Exception e ){ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); } - + int maxErrorsPerThread = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_ERRORS_PER_THREAD; itemName = "aai.datasnapshot.max.errors.per.thread"; try { @@ -240,7 +242,7 @@ public class DataSnapshot4HistInit { }catch ( Exception e ){ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); } - + Long vertToEdgeProcDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_TO_EDGE_PROC_DELAY_MS; itemName = "aai.datasnapshot.vertex.to.edge.proc.delay.ms"; try { @@ -251,7 +253,7 @@ public class DataSnapshot4HistInit { }catch ( Exception e ){ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); } - + itemName = "aai.datasnapshot.stagger.thread.delay.ms"; try { String val = AAIConfig.get(itemName); @@ -260,8 +262,8 @@ public class DataSnapshot4HistInit { } }catch ( Exception e ){ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); - } - + } + long debugAddDelayTime = 1; // Default to 1 millisecond Boolean debug4Create = false; // By default we do not use debugging for snapshot creation JCommander jCommander; @@ -270,15 +272,15 @@ public class DataSnapshot4HistInit { jCommander.setProgramName(DataSnapshot4HistInit.class.getSimpleName()); } catch (ParameterException e1) { AAIException ae = new AAIException("AAI_6128", e1 , "Error - invalid value passed to list of args - "+args); - ErrorLogHelper.logException(ae); + ErrorLogHelper.logException(ae); AAISystemExitUtil.systemExitCloseAAIGraph(1); } - - + + if (args.length >= 1) { command = cArgs.command; } - + String source = cArgs.caller; String snapshotType = "graphson"; if( SNAPSHOT_RELOAD_COMMANDS.contains(cArgs.command)){ @@ -306,7 +308,7 @@ public class DataSnapshot4HistInit { AAISystemExitUtil.systemExitCloseAAIGraph(1); } LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount4Create ); - + if( maxNodesPerFile4Create < 1000 || maxNodesPerFile4Create > 1000000 ){ ErrorLogHelper.logError("AAI_6128", "Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]"); LOGGER.debug("Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]"); @@ -314,7 +316,7 @@ public class DataSnapshot4HistInit { AAISystemExitUtil.systemExitCloseAAIGraph(1); } LOGGER.debug(" Will do Threaded Snapshot with maxNodesPerFile = " + maxNodesPerFile4Create ); - + // If doing a "threaded" snapshot, they need to specify how many threads to use // They can also use debug mode if they pass the word "DEBUG" to do the nodes one at a time to see where it breaks. if( cArgs.debugFlag.equals("DEBUG") ){ @@ -322,7 +324,7 @@ public class DataSnapshot4HistInit { } LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount4Create + ", and DEBUG-flag set to: " + debug4Create ); - + if (debug4Create) { // If doing a "threaded" snapshot, they need to specify how many threads to use (param 1) // They can also use debug mode if they pass the word "DEBUG" to do the nodes one (param 2) @@ -384,9 +386,9 @@ public class DataSnapshot4HistInit { } } long scriptStartTime = System.currentTimeMillis(); - - threadCount4Create = cArgs.threadCount; - + + threadCount4Create = cArgs.threadCount; + //Print Defaults LOGGER.debug("DataSnapshot4HistInit command is [" + cArgs.command + "]"); LOGGER.debug("File name to reload snapshot [" + cArgs.oldFileName + "]"); @@ -401,7 +403,7 @@ public class DataSnapshot4HistInit { LOGGER.debug("VertToEdgeProcDelay is [" + cArgs.vertToEdgeProcDelay + "]"); LOGGER.debug("StaggerThreadDelay is [" + cArgs.staggerThreadDelay + "]"); LOGGER.debug("Caller process is ["+ cArgs.caller + "]"); - + //Print non-default values if (!AAIConfig.isEmpty(cArgs.fileName)){ LOGGER.debug("Snapshot file name (if not default) to use is [" + cArgs.fileName + "]"); @@ -412,7 +414,7 @@ public class DataSnapshot4HistInit { if (!AAIConfig.isEmpty(cArgs.oldFileDir)){ LOGGER.debug("Directory path (if not default) to load the old snapshot file from is [" + cArgs.oldFileDir + "]"); } - + ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { AAIConfig.init(); @@ -423,7 +425,14 @@ public class DataSnapshot4HistInit { new File(targetDir).mkdirs(); LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while) "); - + + String graphsonVersionArg = cArgs.graphsonVersion; + if (graphsonVersionArg != GraphSONVersion.V1_0.getVersion() && graphsonVersionArg != GraphSONVersion.V2_0.getVersion() && graphsonVersionArg != GraphSONVersion.V3_0.getVersion()) { + LOGGER.warn("The graphsonVersion argument was not in the correct format, defaulting to V1. Argument value should be either one of [{}|{}|{}]", GraphSONVersion.V1_0,GraphSONVersion.V2_0,GraphSONVersion.V3_0); + graphsonVersionArg = GraphSONVersion.V1_0.getVersion(); + } + GraphSONVersion graphsonVersion = GraphSONVersion.valueOf("V" + graphsonVersionArg.replace(".", "_")); + if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT")) && threadCount4Create == 1 ){ // ------------------------------------------------------------------------------- @@ -439,7 +448,7 @@ public class DataSnapshot4HistInit { String newSnapshotOutFname = null; long timeA = System.nanoTime(); newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr; - graph.io(IoCore.graphson()).writeGraph(newSnapshotOutFname); + graph.io(GraphSONIo.build(graphsonVersion)).writeGraph(newSnapshotOutFname); LOGGER.debug("Snapshot written to " + newSnapshotOutFname); long timeB = System.nanoTime(); long diffTime = timeB - timeA; @@ -447,12 +456,12 @@ public class DataSnapshot4HistInit { long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); LOGGER.debug(" -- Single-Thread dataSnapshot took: " + minCount + " minutes, " + secCount + " seconds " ); - - } - else if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT")) + + } + else if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT")) && threadCount4Create > 1 ){ // ------------------------------------------------------------ - // They want the creation of the snapshot to be spread out via + // They want the creation of the snapshot to be spread out via // threads and go to multiple files // ------------------------------------------------------------ LOGGER.debug(" Command = " + command ); @@ -479,12 +488,12 @@ public class DataSnapshot4HistInit { long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); LOGGER.debug(" -- To count all vertices in DB it took: " + minCount + " minutes, " + secCount + " seconds " ); - - int fileCount4Create = figureOutFileCount( totalVertCount, threadCount4Create, + + int fileCount4Create = figureOutFileCount( totalVertCount, threadCount4Create, maxNodesPerFile4Create ); - int threadPassesNeeded = (int) Math.ceil((double)fileCount4Create / (double)threadCount4Create); - long nodesPerFile = (long) Math.ceil((double)totalVertCount / (double)fileCount4Create); - + int threadPassesNeeded = (int) Math.ceil((double)fileCount4Create / (double)threadCount4Create); + long nodesPerFile = (long) Math.ceil((double)totalVertCount / (double)fileCount4Create); + LOGGER.debug(" We will run this many simultaneous threads: " + threadCount4Create ); LOGGER.debug(" Required number of passes: " + threadPassesNeeded ); LOGGER.debug(" Max Nodes per file: " + maxNodesPerFile4Create ); @@ -498,8 +507,8 @@ public class DataSnapshot4HistInit { String tk = "" + t; vertIdListHash.put( tk, vIdList); } - - int currentTNum = 0; + + int currentTNum = 0; String currentTKey = "0"; long thisThrIndex = 0; Iterator vtxItr = graph.vertices(); // Getting ALL vertices! @@ -515,11 +524,11 @@ public class DataSnapshot4HistInit { long vid = (long)(vtxItr.next()).id(); (vertIdListHash.get(currentTKey)).add(vid); } - + // close this graph instance thing here since we have all the ids graph.tx().rollback(); graph.tx().close(); - + long timeB = System.nanoTime(); diffTime = timeB - timeA2; minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); @@ -530,9 +539,9 @@ public class DataSnapshot4HistInit { // Need to print out each set of vertices using it's own thread // NOTE - we may have more files to generate than number of threads - which // just means that ALL the files won't necessarily be generated in parallel. - + int fileNo = 0; - for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){ + for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){ ArrayList threadArr = new ArrayList (); // For each Pass, kick off all the threads and wait until they finish long timeP1 = System.nanoTime(); @@ -540,14 +549,14 @@ public class DataSnapshot4HistInit { String fileNoStr = "" + fileNo; String subFName = newSnapshotOutFname + ".P" + fileNoStr; LOGGER.debug(" DEBUG >>> kick off pass # " + passNo + ", thread # " + thNum); - Thread thr = new Thread(new PrintVertexDetails(graph, subFName, + Thread thr = new Thread(new PrintVertexDetails(graph, subFName, vertIdListHash.get(fileNoStr), - debug4Create, debugAddDelayTime, + debug4Create, debugAddDelayTime, snapshotType, LOGGER) ); thr.start(); threadArr.add(thr); fileNo++; - } + } // Make sure all the threads finish before considering this Pass finished. for( int thNum = 0; thNum < threadCount4Create; thNum++ ){ if( null != threadArr.get(thNum) ){ @@ -561,7 +570,7 @@ public class DataSnapshot4HistInit { LOGGER.debug(" Pass number " + passNo + " (out of " + threadPassesNeeded + ") took " + minCount + " minutes, " + secCount + " seconds "); } - + long timeC = System.nanoTime(); diffTime = timeC - timeB; minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); @@ -569,7 +578,7 @@ public class DataSnapshot4HistInit { LOGGER.debug(" -- To write all the data out to snapshot files, it took: " + minCount + " minutes, " + secCount + " seconds " ); - + } else if( command.equals("MULTITHREAD_RELOAD") ){ // --------------------------------------------------------------------- // They want the RELOAD of the snapshot to be spread out via threads @@ -577,11 +586,11 @@ public class DataSnapshot4HistInit { // snapshot is written to. Ie. if you have a single-file snapshot, // then this will be single-threaded. // If the number of files is greater than the 'threadCount' parameter, - // then we will use more than one pass to keep the number of simultaneous + // then we will use more than one pass to keep the number of simultaneous // threads below the threadCount param. // LOGGER.debug(" Command = " + command ); - + if (cArgs.oldFileDir != null && !cArgs.oldFileDir.isEmpty()){ targetDir = cArgs.oldFileDir; } @@ -589,12 +598,12 @@ public class DataSnapshot4HistInit { int fCount = snapFilesArr.size(); int threadPassesNeeded = (int) Math.ceil((double)fCount / (double)threadCount4Create); int filesPerPass = (int) Math.ceil((double)fCount / (double)threadPassesNeeded); - + JanusGraph graph1 = AAIGraph.getInstance().getGraph(); long timeStart = System.nanoTime(); GraphAdminDBUtils.logConfigs(graph1.configuration()); HashMap old2NewVertIdMap = new HashMap (); - HashMap > nodeKeyNames = new HashMap > (); + HashMap > nodeKeyNames = new HashMap > (); try { LOGGER.debug("call getNodeKeyNames ()" ); nodeKeyNames = getNodeKeyNames(); @@ -606,11 +615,11 @@ public class DataSnapshot4HistInit { ExecutorService executor = Executors.newFixedThreadPool(fCount); int threadFailCount = 0; - + LOGGER.debug(" -- vertAddDelayMs = " + vertAddDelayMs + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs + ", maxErrorsPerThread = " + maxErrorsPerThread ); - + // -------------------------------------- // Step 1 -- Load empty vertices // -------------------------------------- @@ -633,18 +642,18 @@ public class DataSnapshot4HistInit { listFutV.add(future); LOGGER.debug(" -- Starting PartialDbLoad VERT_ONLY file # "+ fileNo + "( passNo = " + passNo + ", passIndex = " + thisPassCount + ")"); - + thisPassCount++; fileNo++; } - + int threadCount4Reload = 0; for(Future> fut : listFutV){ threadCount4Reload++; try { old2NewVertIdMap.putAll(fut.get()); - LOGGER.debug(" -- back from PartialVertexLoader. returned pass # " - + passNo + ", thread # " + LOGGER.debug(" -- back from PartialVertexLoader. returned pass # " + + passNo + ", thread # " + threadCount4Reload + ", current size of old2NewVertMap is: " + old2NewVertIdMap.size() ); } @@ -660,7 +669,7 @@ public class DataSnapshot4HistInit { } } } // end of passes for loading empty vertices - + executor.shutdown(); if( threadFailCount > 0 ) { String emsg = " FAILURE >> " + threadFailCount + " Vertex-loader thread(s) failed to complete successfully. "; @@ -678,7 +687,7 @@ public class DataSnapshot4HistInit { // Give the DB a little time to chew on all those new vertices Thread.sleep(vertToEdgeProcDelay); - + // ------------------------------------------------------------- // Step 2 -- Load Edges and properties onto the empty vertices // ------------------------------------------------------------- @@ -691,7 +700,7 @@ public class DataSnapshot4HistInit { int thisPassCount = 0; while( (thisPassCount < filesPerPass) && (fileNo < fCount) ){ - File f = snapFilesArr.get(fileNo); + File f = snapFilesArr.get(fileNo); String fname = f.getName(); String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname; Thread.sleep(cArgs.staggerThreadDelay); // Stagger the threads a bit @@ -700,15 +709,15 @@ public class DataSnapshot4HistInit { edgeAddDelayMs, failureDelayMs, retryDelayMs, old2NewVertIdMap, maxErrorsPerThread, LOGGER, scriptStartTime, nodeKeyNames); - + Future > future = (Future>) executor.submit(eLoader); // add future to the list, we can wait for it below listFutEdg.add(future); LOGGER.debug(" -- Starting PartialPropAndEdgeLoader4HistInit file # " - + fileNo + " (pass # " + passNo + ", passIndex " + + fileNo + " (pass # " + passNo + ", passIndex " + thisPassCount + ")" ); - + thisPassCount++; fileNo++; } @@ -717,7 +726,7 @@ public class DataSnapshot4HistInit { for( Future> fut : listFutEdg ){ threadCount4Reload++; try{ - fut.get(); + fut.get(); LOGGER.debug(" -- back from PartialPropAndEdgeLoader4HistInit. pass # " + passNo + ", thread # " + threadCount4Reload ); } @@ -732,8 +741,8 @@ public class DataSnapshot4HistInit { ErrorLogHelper.logException(ae); } } - - } // end of passes for reloading edges and properties + + } // end of passes for reloading edges and properties executor.shutdown(); @@ -758,7 +767,7 @@ public class DataSnapshot4HistInit { long totalSecCount = TimeUnit.NANOSECONDS.toSeconds(totalDiffTime) - (60 * totalMinCount); LOGGER.debug(" -- TOTAL multi-threaded reload time: " + totalMinCount + " minutes, " + totalSecCount + " seconds " ); - + } else if (command.equals("CLEAR_ENTIRE_DATABASE")) { // ------------------------------------------------------------------ // They are calling this to clear the db before re-loading it @@ -786,10 +795,10 @@ public class DataSnapshot4HistInit { String rtConfig = AAIConstants.REALTIME_DB_CONFIG; String serviceName = System.getProperty("aai.service.name", DataSnapshot4HistInit.class.getSimpleName()); LOGGER.debug("Getting new configs for clearig"); - + PropertiesConfiguration propertiesConfiguration = new AAIGraphConfig.Builder(rtConfig).forService(serviceName).withGraphType(REALTIME_DB).buildConfiguration(); LOGGER.debug("Open New Janus Graph"); - + JanusGraph janusGraph = JanusGraphFactory.open(propertiesConfiguration); verifyGraph(janusGraph); GraphAdminDBUtils.logConfigs(janusGraph.configuration()); @@ -800,7 +809,7 @@ public class DataSnapshot4HistInit { LOGGER.debug(" reloading data or the data will be put in without indexes. "); dbClearFlag = true; LOGGER.debug("All done clearing DB"); - + } else if (command.equals("RELOAD_DATA")) { // --------------------------------------------------------------------------- // They want to restore the database from either a single file, or a group @@ -816,7 +825,7 @@ public class DataSnapshot4HistInit { LOGGER.debug(emsg); AAISystemExitUtil.systemExitCloseAAIGraph(1); } - + long timeA = System.nanoTime(); ArrayList snapFilesArr = new ArrayList (); @@ -839,15 +848,15 @@ public class DataSnapshot4HistInit { } } } - + if( snapFilesArr.isEmpty() ){ String emsg = "oldSnapshotFile " + onePieceSnapshotFname + "(with or without .P0) could not be found."; LOGGER.debug(emsg); AAISystemExitUtil.systemExitCloseAAIGraph(1); } - + int fCount = snapFilesArr.size(); - Vector inputStreamsV = new Vector<>(); + Vector inputStreamsV = new Vector<>(); for( int i = 0; i < fCount; i++ ){ File f = snapFilesArr.get(i); String fname = f.getName(); @@ -869,10 +878,10 @@ public class DataSnapshot4HistInit { // inputStreams.elements() will return Enumerations InputStream sis = new SequenceInputStream(inputStreamsV.elements()); LOGGER.debug("Begin loading data from " + fCount + " files -----"); - if("gryo".equalsIgnoreCase(snapshotType)){ + if("gryo".equalsIgnoreCase(snapshotType)){ graph.io(IoCore.gryo()).reader().create().readGraph(sis, graph); } else { - graph.io(IoCore.graphson()).reader().create().readGraph(sis, graph); + graph.io(GraphSONIo.build(graphsonVersion)).reader().create().readGraph(sis, graph); } LOGGER.debug("Completed the inputGraph command, now try to commit()... "); graph.tx().commit(); @@ -880,14 +889,14 @@ public class DataSnapshot4HistInit { long vCount = graph.traversal().V().count().next(); LOGGER.debug("A little after repopulating from an old snapshot, we see: " + vCount + " vertices in the db."); - + long timeB = System.nanoTime(); long diffTime = timeB - timeA; long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); LOGGER.debug(" -- To Reload this snapshot, it took: " + minCount + " minutes, " + secCount + " seconds " ); - + LOGGER.debug("A little after repopulating from an old snapshot, we see: " + vCount + " vertices in the db."); } else { @@ -923,11 +932,11 @@ public class DataSnapshot4HistInit { return success; } - + public static HashMap > getNodeKeyNames() { - + HashMap > keyNameHash = new HashMap > (); - Loader loader = LoaderUtil.getLatestVersion(); + Loader loader = LoaderUtil.getLatestVersion(); Set> entrySet = loader.getAllObjects().entrySet(); // Get a collection of the names of the key properties for each nodeType for (Entry entry : entrySet) { @@ -935,12 +944,12 @@ public class DataSnapshot4HistInit { Set keyPropsSet = entry.getValue().getKeys(); ArrayList keyProps = new ArrayList (); keyProps.addAll(keyPropsSet); - keyNameHash.put(nType, keyProps); + keyNameHash.put(nType, keyProps); } return keyNameHash; } - - + + private static ArrayList getFilesToProcess(String targetDir, String oldSnapshotFileName, boolean doingClearDb) throws Exception { @@ -1019,26 +1028,26 @@ public class DataSnapshot4HistInit { } - - public static int figureOutFileCount( long totalVertCount, int threadCount4Create, + + public static int figureOutFileCount( long totalVertCount, int threadCount4Create, long maxNodesPerFile ) { - + // NOTE - we would always like to use all of our threads. That is, if // we could process all the data with 16 threads, but our threadCount4Create is - // only 15, we will do two passes and use all 15 threads each pass which will + // only 15, we will do two passes and use all 15 threads each pass which will // create a total of 30 files. Each file will be a bit smaller so the overall // time for the two passes should be faster. if( totalVertCount <= 0 || threadCount4Create <= 0 || maxNodesPerFile <= 0) { return 1; } - - long maxNodesPerPass = threadCount4Create * maxNodesPerFile; - int numberOfPasses = (int) Math.ceil( (double)totalVertCount / (double)maxNodesPerPass); + + long maxNodesPerPass = threadCount4Create * maxNodesPerFile; + int numberOfPasses = (int) Math.ceil( (double)totalVertCount / (double)maxNodesPerPass); int fileCt = threadCount4Create * numberOfPasses; - - return fileCt; + + return fileCt; } - + class CommandLineArgs { @@ -1054,9 +1063,12 @@ public class DataSnapshot4HistInit { @Parameter(names = "-snapshotType", description = "snapshot type of gryo or graphson") public String snapshotType = "graphson"; + @Parameter(names = "-v", description = "Graphson version of the snapshot file. Defaults to 1") + public String graphsonVersion = GraphSONVersion.V1_0.getVersion(); + @Parameter(names = "-threadCount", description = "thread count for create") public int threadCount = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_THREADS_FOR_CREATE; - + @Parameter(names = "-maxNodesPerFile", description = "Max nodes per file") public long maxNodesPerFile = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_NODES_PER_FILE_FOR_CREATE; @@ -1065,13 +1077,13 @@ public class DataSnapshot4HistInit { @Parameter(names = "-debugAddDelayTime", description = "delay in ms between each Add for debug mode") public long debugAddDelayTime = 1L; - + @Parameter(names = "-vertAddDelayMs", description = "delay in ms while adding each vertex") public long vertAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS.longValue(); - + @Parameter(names = "-edgeAddDelayMs", description = "delay in ms while adding each edge") public long edgeAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_EDGE_ADD_DELAY_MS.longValue(); - + @Parameter(names = "-failureDelayMs", description = "delay in ms when failure to load vertex or edge in snapshot") public long failureDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_FAILURE_DELAY_MS.longValue(); @@ -1080,25 +1092,25 @@ public class DataSnapshot4HistInit { @Parameter(names = "-maxErrorsPerThread", description = "max errors allowed per thread") public int maxErrorsPerThread = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_ERRORS_PER_THREAD; - + @Parameter(names = "-vertToEdgeProcDelay", description = "vertex to edge processing delay in ms") public long vertToEdgeProcDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_TO_EDGE_PROC_DELAY_MS.longValue(); - + @Parameter(names = "-staggerThreadDelay", description = "thread delay stagger time in ms") public long staggerThreadDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_STAGGER_THREAD_DELAY_MS; - + @Parameter(names = "-fileName", description = "file name for generating snapshot ") public String fileName = ""; - + @Parameter(names = "-snapshotDir", description = "file path for generating snapshot ") public String snapshotDir = ""; - + @Parameter(names = "-oldFileDir", description = "directory containing the old snapshot file for reloading") public String oldFileDir = ""; - + @Parameter(names = "-caller", description = "process invoking the dataSnapshot") public String caller = ""; - + } - -} \ No newline at end of file + +} diff --git a/src/main/java/org/onap/aai/dbgen/GraphSONPartialIO.java b/src/main/java/org/onap/aai/dbgen/GraphSONPartialIO.java index 915db69..c557491 100644 --- a/src/main/java/org/onap/aai/dbgen/GraphSONPartialIO.java +++ b/src/main/java/org/onap/aai/dbgen/GraphSONPartialIO.java @@ -127,16 +127,6 @@ public final class GraphSONPartialIO implements Io registry(final IoRegistry registry) { - this.registry = registry; - return this; - } - @Override public Io.Builder onMapper(final Consumer onMapper) { this.onMapper = onMapper; @@ -154,5 +144,10 @@ public final class GraphSONPartialIO implements Io boolean requiresVersion(V version) { + return version.equals(GraphSONVersion.V1_0); + } } } diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java index 7fbc449..928494e 100644 --- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java +++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java @@ -47,11 +47,11 @@ public class SchemaMod { private final LoaderFactory loaderFactory; private final SchemaVersions schemaVersions; - + private static boolean historyEnabled; private Logger logger = LoggerFactory.getLogger(SchemaMod.class.getSimpleName()); - + public SchemaMod(LoaderFactory loaderFactory, SchemaVersions schemaVersions){ this.loaderFactory = loaderFactory; @@ -61,7 +61,7 @@ public class SchemaMod { public void execute(String[] args) { logger = LoggerFactory.getLogger(SchemaMod.class.getSimpleName()); - + // NOTE -- We're just working with properties that are used for NODES // for now. String propName = ""; @@ -73,7 +73,7 @@ public class SchemaMod { long commitBlockSize = 120000; String usageString = "Usage: SchemaMod propertyName targetDataType targetIndexInfo preserveDataFlag consistencyLockFlag [blockSize] \n"; - + if (args.length == 5) { propName = args[0]; targetDataType = args[1]; @@ -93,7 +93,7 @@ public class SchemaMod { String emsg = "Incorrect number of Parameters passed. \n" + usageString; logAndPrint(logger, emsg); System.exit(1); - } + } if (propName.equals("")) { String emsg = "Bad parameter - propertyName cannot be empty. \n" + usageString; logAndPrint(logger, emsg); @@ -112,7 +112,7 @@ public class SchemaMod { logAndPrint(logger, emsg); System.exit(1); } - + try { if( !commitBlockSizeStr.equals("")) { // They're over-riding the commitBlockSize @@ -153,8 +153,6 @@ public class SchemaMod { engine = new JanusGraphDBEngine(queryStyle, loader); SchemaModInternalBatch internal = new SchemaModInternalBatch(engine, logger, propName, targetDataType, targetIndexInfo, Boolean.parseBoolean(preserveDataFlag), Boolean.parseBoolean(consistencyLockFlag), commitBlockSize); internal.execute(); - engine.startTransaction(); - engine.tx().close(); logAndPrint(logger, "------ Completed the SchemaMod -------- "); } catch (Exception e) { String emsg = "Not able to complete the requested SchemaMod \n"; @@ -197,14 +195,14 @@ public class SchemaMod { ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } - + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); if( historyEnabled ) { String emsg = "Regular SchemaMod may not be used when history.enabled=true. "; System.out.println(emsg); throw new AAIException("AAI-4005",emsg); } - + LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions"); SchemaMod schemaMod = new SchemaMod(loaderFactory, schemaVersions); @@ -213,4 +211,4 @@ public class SchemaMod { System.exit(0); } -} \ No newline at end of file +} diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java index 4c393eb..c6b591d 100644 --- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java +++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java @@ -47,7 +47,7 @@ public class SchemaMod4Hist { private final LoaderFactory loaderFactory; private final SchemaVersions schemaVersions; - + private static boolean historyEnabled; public SchemaMod4Hist(LoaderFactory loaderFactory, SchemaVersions schemaVersions){ @@ -64,7 +64,7 @@ public class SchemaMod4Hist { Logger logger = LoggerFactory.getLogger(SchemaMod4Hist.class.getSimpleName()); MDC.put("logFilenameAppender", SchemaMod4Hist.class.getSimpleName()); - + // NOTE -- We're just working with properties that are used for NODES // for now. @@ -111,7 +111,7 @@ public class SchemaMod4Hist { logAndPrint(logger, emsg + "[" + ae.getMessage() + "]"); System.exit(1); } - + // Give a big warning if the DbMaps.PropertyDataTypeMap value does not // agree with what we're doing String warningMsg = ""; @@ -143,8 +143,6 @@ public class SchemaMod4Hist { // preserve-data-flag last parameter since this is for HISTORY. SchemaModInternal4Hist internal = new SchemaModInternal4Hist(engine, logger, propName, targetDataType, targetIndexInfo, true); internal.execute(); - engine.startTransaction(); - engine.tx().close(); logAndPrint(logger, "------ Completed the SchemaMod -------- "); } catch (Exception e) { String emsg = "Not able to complete the requested SchemaMod4Hist \n"; @@ -181,14 +179,14 @@ public class SchemaMod4Hist { ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } - + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); if( !historyEnabled ) { String emsg = "SchemaMod4Hist may only be used when history.enabled=true. "; System.out.println(emsg); throw new AAIException("AAI-4005",emsg); } - + LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions"); SchemaMod4Hist schemaMod4H = new SchemaMod4Hist(loaderFactory, schemaVersions); @@ -197,4 +195,4 @@ public class SchemaMod4Hist { System.exit(0); } -} \ No newline at end of file +} diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java index f656b9f..aef53cd 100644 --- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java +++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java @@ -20,17 +20,17 @@ package org.onap.aai.dbgen.schemamod; import java.util.Iterator; -import java.util.UUID; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.serialization.engines.TransactionalGraphEngine; import org.onap.aai.util.FormatDate; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.janusgraph.core.Cardinality; import org.janusgraph.core.PropertyKey; import org.janusgraph.core.schema.JanusGraphManagement; +import org.janusgraph.graphdb.database.management.ManagementSystem; public class SchemaModInternal4Hist { private final TransactionalGraphEngine engine; @@ -40,7 +40,7 @@ public class SchemaModInternal4Hist { private final boolean preserveData; private final Cardinality cardinality; private final Logger logger; - + public SchemaModInternal4Hist(TransactionalGraphEngine engine, Logger logger, String propName, String type, String indexType, boolean preserveData) { this.engine = engine; this.propName = propName; @@ -50,7 +50,7 @@ public class SchemaModInternal4Hist { this.cardinality = Cardinality.LIST; // Always use this for History this.logger = logger; } - + private Class determineClass(String type) { final Class result; if (type.equals("String")) { @@ -74,7 +74,7 @@ public class SchemaModInternal4Hist { logAndPrint(logger, emsg); throw new RuntimeException(emsg); } - + return result; } @@ -95,32 +95,35 @@ public class SchemaModInternal4Hist { logAndPrint(logger, emsg); System.exit(1); } - + // Rename this property to a backup name (old name with "retired_" // appended plus a dateStr) FormatDate fd = new FormatDate("MMddHHmm", "GMT"); String dteStr= fd.getDateTime(); - + String retiredName = propName + "-" + dteStr + "-RETIRED"; graphMgt.changeName(origPropKey, retiredName); - + // Create a new property using the original property name and the // targetDataType PropertyKey freshPropKey = graphMgt.makePropertyKey(propName).dataType(type) .cardinality(cardinality).make(); - + // Create an index if needed (regular index will be used instead of unique for history) - if (indexType.equals("index") || indexType.equals("uniqueIndex")) { - String freshIndexName = propName + dteStr; + boolean needsIndex = indexType.equals("index") || indexType.equals("uniqueIndex"); + String freshIndexName = propName + dteStr; + if (needsIndex) { graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).buildCompositeIndex(); } - + logAndPrint(logger, "Committing schema changes with graphMgt.commit()"); graphMgt.commit(); engine.commit(); + if (needsIndex) { + ManagementSystem.awaitGraphIndexStatus(AAIGraph.getInstance().getGraph(), freshIndexName).call(); + } Graph grTmp2 = engine.startTransaction(); - - + // For each node that has this property, update the new from the old // and then remove the // old property from that node @@ -146,7 +149,7 @@ public class SchemaModInternal4Hist { logAndPrint(logger, "INFO -- just did the remove of the " + retiredName + " from this vertex. (vid=" + tmpVid + ")"); } - + success = true; } catch (Exception ex) { logAndPrint(logger, "Threw a regular Exception: "); @@ -166,7 +169,7 @@ public class SchemaModInternal4Hist { } } } - + /** * Log and print. * @@ -177,5 +180,5 @@ public class SchemaModInternal4Hist { System.out.println(msg); logger.debug(msg); } - + } diff --git a/src/main/resources/etc/appprops/janusgraph-realtime.properties b/src/main/resources/etc/appprops/janusgraph-realtime.properties index 3a7df75..22ce38e 100644 --- a/src/main/resources/etc/appprops/janusgraph-realtime.properties +++ b/src/main/resources/etc/appprops/janusgraph-realtime.properties @@ -2,7 +2,7 @@ # ============LICENSE_START======================================================= # org.onap.aai # ================================================================================ -# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# Copyright � 2017 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ storage.hostname=localhost #schema.default=none storage.lock.wait-time=300 storage.hbase.table=aaigraph-dev1.dev -storage.hbase.ext.zookeeper.znode.parent=/hbase-unsecure +# storage.hbase.ext.zookeeper.znode.parent=/hbase-unsecure # Setting db-cache to false ensure the fastest propagation of changes across servers cache.db-cache = false diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 68cb299..bf6882a 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -1135,7 +1135,7 @@ - + diff --git a/src/main/resources/org/janusgraph/graphdb/configuration/janusgraph.internal.properties b/src/main/resources/org/janusgraph/graphdb/configuration/janusgraph.internal.properties deleted file mode 100644 index dcbbf83..0000000 --- a/src/main/resources/org/janusgraph/graphdb/configuration/janusgraph.internal.properties +++ /dev/null @@ -1,6 +0,0 @@ -# TODO: Please remove this file once all we move away from 0.2.3 -# The reason we need this is because we built our own 0.2.3-SNAPSHOT -# Janusgraph checks this file to see which it is compatible with - -janusgraph.compatible-versions=0.1.0,0.1.1,0.2.0,0.2.1,0.2.2,0.2.3-SNAPSHOT -janusgraph.version=0.2.3 diff --git a/src/test/java/org/onap/aai/migration/v12/MigrateHUBEvcInventoryTest.java b/src/test/java/org/onap/aai/migration/v12/MigrateHUBEvcInventoryTest.java index 33afbfd..32190b5 100644 --- a/src/test/java/org/onap/aai/migration/v12/MigrateHUBEvcInventoryTest.java +++ b/src/test/java/org/onap/aai/migration/v12/MigrateHUBEvcInventoryTest.java @@ -296,7 +296,7 @@ public class MigrateHUBEvcInventoryTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventoryMethodTest.java b/src/test/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventoryMethodTest.java index 2b2a61d..005517a 100644 --- a/src/test/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventoryMethodTest.java +++ b/src/test/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventoryMethodTest.java @@ -77,7 +77,7 @@ public class MigrateINVPhysicalInventoryMethodTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventoryTest.java b/src/test/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventoryTest.java index aae92d4..e946a10 100644 --- a/src/test/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventoryTest.java +++ b/src/test/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventoryTest.java @@ -86,7 +86,7 @@ public class MigrateINVPhysicalInventoryTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v12/MigrateInvEvcInventoryTest.java b/src/test/java/org/onap/aai/migration/v12/MigrateInvEvcInventoryTest.java index f895c34..65e1905 100644 --- a/src/test/java/org/onap/aai/migration/v12/MigrateInvEvcInventoryTest.java +++ b/src/test/java/org/onap/aai/migration/v12/MigrateInvEvcInventoryTest.java @@ -83,7 +83,7 @@ public class MigrateInvEvcInventoryTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v12/MigratePATHEvcInventoryTest.java b/src/test/java/org/onap/aai/migration/v12/MigratePATHEvcInventoryTest.java index b387f38..e302002 100644 --- a/src/test/java/org/onap/aai/migration/v12/MigratePATHEvcInventoryTest.java +++ b/src/test/java/org/onap/aai/migration/v12/MigratePATHEvcInventoryTest.java @@ -224,7 +224,7 @@ public class MigratePATHEvcInventoryTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v12/MigrateSAREvcInventoryTest.java b/src/test/java/org/onap/aai/migration/v12/MigrateSAREvcInventoryTest.java index 9af7bb6..2951540 100644 --- a/src/test/java/org/onap/aai/migration/v12/MigrateSAREvcInventoryTest.java +++ b/src/test/java/org/onap/aai/migration/v12/MigrateSAREvcInventoryTest.java @@ -141,7 +141,7 @@ public class MigrateSAREvcInventoryTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v13/MigrateForwarderEvcCircuitIdTest.java b/src/test/java/org/onap/aai/migration/v13/MigrateForwarderEvcCircuitIdTest.java index add3ace..154b253 100644 --- a/src/test/java/org/onap/aai/migration/v13/MigrateForwarderEvcCircuitIdTest.java +++ b/src/test/java/org/onap/aai/migration/v13/MigrateForwarderEvcCircuitIdTest.java @@ -144,7 +144,7 @@ public class MigrateForwarderEvcCircuitIdTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantIdTest.java b/src/test/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantIdTest.java index bf12089..8adc9e5 100644 --- a/src/test/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantIdTest.java +++ b/src/test/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantIdTest.java @@ -74,7 +74,7 @@ public class MigrateInstanceGroupModelInvariantIdTest extends AAISetup{ TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionIdTest.java b/src/test/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionIdTest.java index d4333c5..4350f75 100644 --- a/src/test/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionIdTest.java +++ b/src/test/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionIdTest.java @@ -77,7 +77,7 @@ public class MigrateInstanceGroupModelVersionIdTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantIdTest.java b/src/test/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantIdTest.java index a570a2c..04c8ab6 100644 --- a/src/test/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantIdTest.java +++ b/src/test/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantIdTest.java @@ -76,7 +76,7 @@ public class MigrateVnfcModelInvariantIdTest extends AAISetup{ TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionIdTest.java b/src/test/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionIdTest.java index c11ff11..c79b06f 100644 --- a/src/test/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionIdTest.java +++ b/src/test/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionIdTest.java @@ -78,7 +78,7 @@ public class MigrateVnfcModelVersionIdTest extends AAISetup{ TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v14/MigrateSameSourcedRCTROPServerDataTest.java b/src/test/java/org/onap/aai/migration/v14/MigrateSameSourcedRCTROPServerDataTest.java index e09b36b..1a177f7 100644 --- a/src/test/java/org/onap/aai/migration/v14/MigrateSameSourcedRCTROPServerDataTest.java +++ b/src/test/java/org/onap/aai/migration/v14/MigrateSameSourcedRCTROPServerDataTest.java @@ -852,7 +852,7 @@ public class MigrateSameSourcedRCTROPServerDataTest extends AAISetup{ TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v14/MigrateSdnaIvlanDataTest.java b/src/test/java/org/onap/aai/migration/v14/MigrateSdnaIvlanDataTest.java index 83291b8..7f381f8 100644 --- a/src/test/java/org/onap/aai/migration/v14/MigrateSdnaIvlanDataTest.java +++ b/src/test/java/org/onap/aai/migration/v14/MigrateSdnaIvlanDataTest.java @@ -173,7 +173,7 @@ public class MigrateSdnaIvlanDataTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v14/PserverDedupWithDifferentSourcesOfTruthTest.java b/src/test/java/org/onap/aai/migration/v14/PserverDedupWithDifferentSourcesOfTruthTest.java index f52258c..e8bebe5 100644 --- a/src/test/java/org/onap/aai/migration/v14/PserverDedupWithDifferentSourcesOfTruthTest.java +++ b/src/test/java/org/onap/aai/migration/v14/PserverDedupWithDifferentSourcesOfTruthTest.java @@ -372,7 +372,7 @@ public class PserverDedupWithDifferentSourcesOfTruthTest extends AAISetup{ TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/java/org/onap/aai/migration/v15/MigrateRadcomChangesTest.java b/src/test/java/org/onap/aai/migration/v15/MigrateRadcomChangesTest.java index 5ba5644..a7ac9d8 100644 --- a/src/test/java/org/onap/aai/migration/v15/MigrateRadcomChangesTest.java +++ b/src/test/java/org/onap/aai/migration/v15/MigrateRadcomChangesTest.java @@ -266,7 +266,7 @@ public class MigrateRadcomChangesTest extends AAISetup { TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin()); GraphTraversalSource traversal = g; - GraphTraversalSource readOnly = tx.traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + GraphTraversalSource readOnly = graph.traversal().withStrategies(ReadOnlyStrategy.instance()); when (spy.tx()).thenReturn(tx); when(spy.asAdmin()).thenReturn(adminSpy); when(adminSpy.getTraversalSource()).thenReturn(traversal); diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml index 045bddd..f8aa27a 100644 --- a/src/test/resources/logback.xml +++ b/src/test/resources/logback.xml @@ -880,8 +880,7 @@ - - + @@ -1022,7 +1021,7 @@ - + @@ -1032,10 +1031,10 @@ - + - + @@ -1147,5 +1146,6 @@ + - \ No newline at end of file + -- cgit 1.2.3-korg