diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rwxr-xr-x | pom.xml | 15 | ||||
| -rw-r--r-- | src/main/java/org/onap/aai/rest/util/EchoResponse.java | 4 | ||||
| -rw-r--r-- | src/main/java/org/onap/aai/schema/GenTester.java | 200 | ||||
| -rw-r--r-- | src/main/resources/etc/appprops/janusgraph-cached.properties | 12 |
5 files changed, 140 insertions, 92 deletions
@@ -1,6 +1,7 @@ /target/ /oxm/ .idea/ +.vscode/ .settings/ debug-logs/ .project @@ -26,7 +26,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4</version> + <version>1.13.5</version> </parent> <groupId>org.onap.aai.graphadmin</groupId> <artifactId>aai-graphadmin</artifactId> @@ -55,7 +55,7 @@ <docker.push.registry>localhost:5000</docker.push.registry> <aai.docker.version>1.0.0</aai.docker.version> <aai.schema.service.version>1.12.3</aai.schema.service.version> - <aai.common.version>1.13.4</aai.common.version> + <aai.common.version>1.13.5</aai.common.version> <aai.build.directory>${project.build.directory}/${project.artifactId}-${project.version}-build/ </aai.build.directory> <aai.docker.namespace>onap</aai.docker.namespace> @@ -666,6 +666,17 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-core</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/src/main/java/org/onap/aai/rest/util/EchoResponse.java b/src/main/java/org/onap/aai/rest/util/EchoResponse.java index 91841df..b033473 100644 --- a/src/main/java/org/onap/aai/rest/util/EchoResponse.java +++ b/src/main/java/org/onap/aai/rest/util/EchoResponse.java @@ -93,8 +93,8 @@ public class EchoResponse extends RESTAPI { exceptionList.put(new AAIException("AAI_0002", "OK"), templateVars); response = Response.status(Status.OK) - .entity(ErrorLogHelper.getRESTAPIInfoResponse( - new ArrayList<>(headers.getAcceptableMediaTypes()), exceptionList)) + .entity(ErrorLogHelper.getRESTAPIInfoResponse(new ArrayList<>(headers.getAcceptableMediaTypes()) + , exceptionList)) .build(); } catch (Exception e) { diff --git a/src/main/java/org/onap/aai/schema/GenTester.java b/src/main/java/org/onap/aai/schema/GenTester.java index defe8bb..b7f4a8f 100644 --- a/src/main/java/org/onap/aai/schema/GenTester.java +++ b/src/main/java/org/onap/aai/schema/GenTester.java @@ -22,8 +22,14 @@ package org.onap.aai.schema; import com.att.eelf.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.janusgraph.core.EdgeLabel; import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.JanusGraphVertex; import org.janusgraph.core.schema.JanusGraphManagement; +import org.janusgraph.core.schema.RelationTypeIndex; +import org.janusgraph.core.schema.SchemaAction; +import org.janusgraph.graphdb.database.StandardJanusGraph; +import org.janusgraph.graphdb.database.management.ManagementSystem; import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbgen.SchemaGenerator; import org.onap.aai.dbmap.AAIGraph; @@ -32,22 +38,23 @@ import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.util.*; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import java.util.List; import java.util.Properties; +import java.util.Set; import java.util.UUID; - public class GenTester { private static Logger LOGGER; private static boolean historyEnabled; - + /** * The main method. * * @param args the arguments */ - public static void main(String[] args) throws AAIException{ - + public static void main(String[] args) throws AAIException { + JanusGraph graph = null; System.setProperty("aai.service.name", GenTester.class.getSimpleName()); // Set the logging file properties to be used by EELFManager @@ -56,120 +63,141 @@ public class GenTester { props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); LOGGER = LoggerFactory.getLogger(GenTester.class); boolean addDefaultCR = true; - + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); initializer.initialize(ctx); try { ctx.scan( "org.onap.aai.config", - "org.onap.aai.setup" - ); + "org.onap.aai.setup"); ctx.refresh(); } catch (Exception e) { AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); - LOGGER.error("Problems running the tool "+aai.getMessage()); + LOGGER.error("Problems running the tool " + aai.getMessage()); ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } - historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); - if( historyEnabled ) { - String amsg = "GenTester may only be used when history.enabled=false. "; - System.out.println(amsg); - LOGGER.debug(amsg); - return; + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled", "false")); + if (historyEnabled) { + String amsg = "GenTester may only be used when history.enabled=false. "; + System.out.println(amsg); + LOGGER.debug(amsg); + return; } try { - LOGGER.debug("GenTester uses either cql jar or Cassandra jar"); + LOGGER.debug("GenTester uses either cql jar or Cassandra jar"); AAIConfig.init(); - if (args != null && args.length > 0 ){ - if( "genDbRulesOnly".equals(args[0]) ){ - ErrorLogHelper.logError("AAI_3100", - " This option is no longer supported. What was in DbRules is now derived from the OXM files. "); - return; - } - else if ( "GEN_DB_WITH_NO_SCHEMA".equals(args[0]) ){ - // Note this is done to create an empty DB with no Schema so that + if (args != null && args.length > 0) { + if ("genDbRulesOnly".equals(args[0])) { + ErrorLogHelper.logError("AAI_3100", + " This option is no longer supported. What was in DbRules is now derived from the OXM files. "); + return; + } else if ("GEN_DB_WITH_NO_SCHEMA".equals(args[0])) { + // Note this is done to create an empty DB with no Schema so that // an HBase copyTable can be used to set up a copy of the db. String imsg = " ---- NOTE --- about to load a graph without doing any schema processing (takes a little while) -------- "; - System.out.println(imsg); - LOGGER.debug(imsg); + System.out.println(imsg); + LOGGER.debug(imsg); graph = AAIGraph.getInstance().getGraph(); - - if( graph == null ){ - ErrorLogHelper.logError("AAI_5102", "Error creating JanusGraph graph."); - return; - } - else { - String amsg = "Successfully loaded a JanusGraph graph without doing any schema work. "; - System.out.println(amsg); - LOGGER.debug(amsg); - return; - } - } else if ("GEN_DB_WITH_NO_DEFAULT_CR".equals(args[0])) { - addDefaultCR = false; - } - else { - ErrorLogHelper.logError("AAI_3000", "Unrecognized argument passed to GenTester.java: [" + args[0] + "]. "); - - String emsg = "Unrecognized argument passed to GenTester.java: [" + args[0] + "]. "; - System.out.println(emsg); - LOGGER.error(emsg); - - emsg = "Either pass no argument for normal processing, or use 'GEN_DB_WITH_NO_SCHEMA'."; - System.out.println(emsg); - LOGGER.error(emsg); - - return; - } - } - - //AAIConfig.init(); + + if (graph == null) { + ErrorLogHelper.logError("AAI_5102", "Error creating JanusGraph graph."); + return; + } else { + LOGGER.debug("Successfully loaded a JanusGraph graph without doing any schema work."); + return; + } + } else if ("GEN_DB_WITH_NO_DEFAULT_CR".equals(args[0])) { + addDefaultCR = false; + } else { + ErrorLogHelper.logError("AAI_3000", "Unrecognized argument passed to GenTester.java: [" + args[0] + "]. "); + + String emsg = "Unrecognized argument passed to GenTester.java: [" + args[0] + "]. "; + System.out.println(emsg); + LOGGER.error(emsg); + + emsg = "Either pass no argument for normal processing, or use 'GEN_DB_WITH_NO_SCHEMA'."; + System.out.println(emsg); + LOGGER.error(emsg); + + return; + } + } + + // AAIConfig.init(); ErrorLogHelper.loadProperties(); - String imsg = " ---- NOTE --- about to open graph (takes a little while)--------;"; - System.out.println(imsg); - LOGGER.debug(imsg); + + LOGGER.debug("about to open graph (takes a little while)"); graph = AAIGraph.getInstance().getGraph(); - - if( graph == null ){ + + if (graph == null) { ErrorLogHelper.logError("AAI_5102", "Error creating JanusGraph graph. "); return; } GraphAdminDBUtils.logConfigs(graph.configuration()); - // Load the propertyKeys, indexes and edge-Labels into the DB - JanusGraphManagement graphMgt = graph.openManagement(); + LOGGER.debug("-- Loading new schema elements into JanusGraph --"); - imsg = "-- Loading new schema elements into JanusGraph --"; - System.out.println(imsg); - LOGGER.debug(imsg); - SchemaGenerator.loadSchemaIntoJanusGraph(graphMgt, null); + boolean dbNotEmpty = (graph.traversal().V().limit(1).hasNext()); + LOGGER.info("DB is not empty. Newly created indexes will also be reindexed."); + List<String> vertexesToReindex = SchemaGenerator.loadSchemaIntoJanusGraph(graph, null, dbNotEmpty); + LOGGER.debug("-- committing transaction "); + graph.tx().commit(); - if( graph != null ){ - imsg = "-- graph commit"; - System.out.println(imsg); - LOGGER.debug(imsg); - graph.tx().commit(); + if (!vertexesToReindex.isEmpty()) { - imsg = "-- graph shutdown "; - System.out.println(imsg); - LOGGER.debug(imsg); - graph.close(); - } - - } catch(Exception ex) { - ErrorLogHelper.logError("AAI_4000", ex.getMessage()); - System.exit(1); - } - + killTransactionsAndInstances(graph); + LOGGER.info("Number of edge indexes to reindex: " + vertexesToReindex.size()); + SchemaGenerator.reindexEdgeIndexes(graph, vertexesToReindex); + } else { + LOGGER.info("Nothing to reindex."); + } + graph.close(); + LOGGER.info("Closed the graph"); - - LOGGER.debug("-- all done, if program does not exit, please kill."); - System.exit(0); - } + } catch (Exception ex) { + ErrorLogHelper.logError("AAI_4000", ex.getMessage()); + System.exit(1); + } + LOGGER.debug("-- all done, if program does not exit, please kill."); + System.exit(0); + } + /** + * Radical approach to avoiding index update failures. + * Indexes can get stuck in INSTALLED state, when there are stale transactions + * or JanusGraph instances. + * This is because a state change needs to be acknowledged by all instances + * before transitioning. + * + * @param graph + * @return + */ + private static void killTransactionsAndInstances(JanusGraph graph) { + graph.tx().rollback(); + final StandardJanusGraph janusGraph = (StandardJanusGraph) graph; + janusGraph.getOpenTransactions().stream().forEach(transaction -> { + LOGGER.debug("Closing open transaction [{}] before schema generation", transaction.toString()); + transaction.rollback(); + }); + + final JanusGraphManagement graphMgtForClosing = graph.openManagement(); + + Set<String> instances = graphMgtForClosing.getOpenInstances(); + LOGGER.info("Number of open instances: {}", instances.size()); + LOGGER.info("Currently open instances: [{}]", instances); + instances.stream() + .filter(instance -> !instance.contains("graphadmin")) // Potentially comment this out, should there be issues with the schema creation job + .filter(instance -> !instance.contains("(current)")) + .forEach(instance -> { + LOGGER.debug("Closing open JanusGraph instance [{}] before reindexing procedure", instance); + graphMgtForClosing.forceCloseInstance(instance); + }); + graphMgtForClosing.commit(); + } }
\ No newline at end of file diff --git a/src/main/resources/etc/appprops/janusgraph-cached.properties b/src/main/resources/etc/appprops/janusgraph-cached.properties index 40bf765..f04152d 100644 --- a/src/main/resources/etc/appprops/janusgraph-cached.properties +++ b/src/main/resources/etc/appprops/janusgraph-cached.properties @@ -2,7 +2,7 @@ # ============LICENSE_START======================================================= # org.onap.aai # ================================================================================ -# Copyright © 2017-18 AT&T Intellectual Property. All rights reserved. +# Copyright � 2017-18 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,8 +20,16 @@ query.fast-property=true query.smart-limit=false # the following parameters are not reloaded automatically and require a manual bounce -storage.backend=inmemory +storage.backend=cql storage.hostname=localhost +storage.cql.keyspace=aaigraph +storage.username=cassandra +storage.password=cassandra + +storage.cql.read-consistency-level=LOCAL_QUORUM +storage.cql.write-consistency-level=LOCAL_QUORUM +storage.cql.replication-factor=3 +storage.cql.only-use-local-consistency-for-system-operations=true #schema.default=none storage.lock.wait-time=300 |