aboutsummaryrefslogtreecommitdiffstats
path: root/src/main/java/org/onap/aai/dbgen
diff options
context:
space:
mode:
Diffstat (limited to 'src/main/java/org/onap/aai/dbgen')
-rw-r--r--src/main/java/org/onap/aai/dbgen/DupeTool.java205
-rw-r--r--src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java906
-rw-r--r--src/main/java/org/onap/aai/dbgen/GraphSONPartialIO.java158
-rw-r--r--src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java354
-rw-r--r--src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java45
5 files changed, 1610 insertions, 58 deletions
diff --git a/src/main/java/org/onap/aai/dbgen/DupeTool.java b/src/main/java/org/onap/aai/dbgen/DupeTool.java
index 7b7ef99..fd5ae00 100644
--- a/src/main/java/org/onap/aai/dbgen/DupeTool.java
+++ b/src/main/java/org/onap/aai/dbgen/DupeTool.java
@@ -19,22 +19,19 @@
*/
package org.onap.aai.dbgen;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.util.*;
-import java.util.Map.Entry;
-
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.onap.aai.db.props.AAIProperties;
-import org.onap.aai.dbmap.AAIGraphConfig;
+import org.apache.tinkerpop.gremlin.structure.*;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.onap.aai.config.PropertyPasswordConfiguration;
import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.edges.enums.AAIDirection;
import org.onap.aai.edges.enums.EdgeProperty;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.Introspector;
@@ -45,19 +42,19 @@ import org.onap.aai.logging.ErrorLogHelper;
import org.onap.aai.logging.LogFormatTools;
import org.onap.aai.logging.LoggingContext;
import org.onap.aai.logging.LoggingContext.StatusCode;
-import org.onap.aai.edges.enums.AAIDirection;
import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.ExceptionTranslator;
+import org.onap.aai.util.GraphAdminConstants;
import org.slf4j.MDC;
-
-import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-import org.janusgraph.core.JanusGraphFactory;
-import org.janusgraph.core.JanusGraph;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.*;
+import java.util.Map.Entry;
+
public class DupeTool {
private static final EELFLogger logger = EELFManager.getInstance().getLogger(DupeTool.class.getSimpleName());
@@ -76,6 +73,7 @@ public class DupeTool {
}
private LoaderFactory loaderFactory;
+ private int dupeGroupCount = 0;
public DupeTool(LoaderFactory loaderFactory, SchemaVersions schemaVersions){
this(loaderFactory, schemaVersions, true);
@@ -89,7 +87,7 @@ public class DupeTool {
public void execute(String[] args){
- String defVersion = "v12";
+ String defVersion = "v15";
try {
defVersion = AAIConfig.get(AAIConstants.AAI_DEFAULT_API_VERSION_PROP);
} catch (AAIException ae) {
@@ -101,7 +99,7 @@ public class DupeTool {
exit(0);
}
-
+ dupeGroupCount = 0;
Loader loader = null;
try {
loader = loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion());
@@ -120,8 +118,8 @@ public class DupeTool {
try {
AAIConfig.init();
- int maxRecordsToFix = AAIConstants.AAI_DUPETOOL_DEFAULT_MAX_FIX;
- int sleepMinutes = AAIConstants.AAI_DUPETOOL_DEFAULT_SLEEP_MINUTES;
+ int maxRecordsToFix = GraphAdminConstants.AAI_DUPETOOL_DEFAULT_MAX_FIX;
+ int sleepMinutes = GraphAdminConstants.AAI_DUPETOOL_DEFAULT_SLEEP_MINUTES;
int timeWindowMinutes = 0; // A value of 0 means that we will not have a time-window -- we will look
// at all nodes of the passed-in nodeType.
long windowStartTime = 0; // Translation of the window into a starting timestamp
@@ -137,7 +135,7 @@ public class DupeTool {
}
} catch (Exception e) {
// Don't worry, we'll just use the defaults that we got from AAIConstants
- logger.warn("WARNING - could not pick up aai.dupeTool values from aaiconfig.properties file. Will use defaults. ");
+ logger.warn("WARNING - could not pick up aai.dupeTool values from aaiconfig.properties file. Will use defaults. " + e.getMessage());
}
String nodeTypeVal = "";
@@ -364,7 +362,7 @@ public class DupeTool {
showNodeDetailsForADupeSet(gt1, firstPassDupeSets.get(x), logger);
}
}
-
+ dupeGroupCount = firstPassDupeSets.size();
boolean didSomeDeletesFlag = false;
ArrayList<String> dupeSetsToFix = new ArrayList<String>();
if (autoFix && firstPassDupeSets.size() == 0) {
@@ -405,6 +403,7 @@ public class DupeTool {
+ " sets of duplicates that we think can be deleted. ";
logger.info(msg);
System.out.println(msg);
+
if (dupeSetsToFix.size() > 0) {
msg = " Here is what the sets look like: ";
logger.info(msg);
@@ -492,7 +491,7 @@ public class DupeTool {
*
* @param args the arguments
*/
- public static void main(String[] args) {
+ public static void main(String[] args) throws AAIException {
System.setProperty("aai.service.name", DupeTool.class.getSimpleName());
// Set the logging file properties to be used by EELFManager
@@ -511,11 +510,23 @@ public class DupeTool {
LoggingContext.statusCode(StatusCode.COMPLETE);
LoggingContext.responseCode(LoggingContext.SUCCESS);
- AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
- "org.onap.aai.config",
- "org.onap.aai.setup"
- );
-
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
+ PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
+ initializer.initialize(ctx);
+ try {
+ ctx.scan(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+ ctx.refresh();
+ } catch (Exception e) {
+ AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
+ logger.error("Problems running DupeTool "+aai.getMessage());
+ LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
+ throw aai;
+ }
LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
DupeTool dupeTool = new DupeTool(loaderFactory, schemaVersions);
@@ -1080,10 +1091,12 @@ public class DupeTool {
Boolean specialTenantRule, Loader loader, EELFLogger logger)
throws AAIException {
- // This method assumes that it is being passed a List of vertex objects
- // which violate our uniqueness constraints.
-
+ // This method assumes that it is being passed a List of
+ // vertex objects which violate our uniqueness constraints.
+ // Note - returning a null vertex means we could not
+ // safely pick one to keep (Ie. safely know which to delete.)
Vertex nullVtx = null;
+ GraphTraversalSource gts = g.traversal();
if (dupeVertexList == null) {
return nullVtx;
@@ -1095,12 +1108,35 @@ public class DupeTool {
if (listSize == 1) {
return (dupeVertexList.get(0));
}
+
+ // If they don't all have the same aai-uri, then we will not
+ // choose between them - we'll need someone to manually
+ // check to pick which one makes sense to keep.
+ Object uriOb = dupeVertexList.get(0).<Object>property("aai-uri").orElse(null);
+ if( uriOb == null || uriOb.toString().equals("") ){
+ // this is a bad node - hopefully will be picked up by phantom checker
+ return nullVtx;
+ }
+ String thisUri = uriOb.toString();
+ for (int i = 1; i < listSize; i++) {
+ uriOb = dupeVertexList.get(i).<Object>property("aai-uri").orElse(null);
+ if( uriOb == null || uriOb.toString().equals("") ){
+ // this is a bad node - hopefully will be picked up by phantom checker
+ return nullVtx;
+ }
+ String nextUri = uriOb.toString();
+ if( !thisUri.equals(nextUri)){
+ // there are different URI's on these - so we can't pick
+ // a dupe to keep. Someone will need to look at it.
+ return nullVtx;
+ }
+ }
Vertex vtxPreferred = null;
Vertex currentFaveVtx = dupeVertexList.get(0);
for (int i = 1; i < listSize; i++) {
Vertex vtxB = dupeVertexList.get(i);
- vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
+ vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, gts,
currentFaveVtx, vtxB, ver, specialTenantRule, loader, logger);
if (vtxPreferred == null) {
// We couldn't choose one
@@ -1110,7 +1146,14 @@ public class DupeTool {
}
}
- return (currentFaveVtx);
+ if( currentFaveVtx != null && checkAaiUriOk(gts, currentFaveVtx, logger) ){
+ return (currentFaveVtx);
+ }
+ else {
+ // We had a preferred vertex, but its aai-uri was bad, so
+ // we will not recommend one to keep.
+ return nullVtx;
+ }
} // end of getPreferredDupe()
@@ -1120,7 +1163,7 @@ public class DupeTool {
*
* @param transId the trans id
* @param fromAppId the from app id
- * @param g the g
+ * @param g the graphTraversalSource
* @param vtxA the vtx A
* @param vtxB the vtx B
* @param ver the ver
@@ -1130,7 +1173,7 @@ public class DupeTool {
* @throws AAIException the AAI exception
*/
public Vertex pickOneOfTwoDupes(String transId,
- String fromAppId, Graph g, Vertex vtxA,
+ String fromAppId, GraphTraversalSource gts, Vertex vtxA,
Vertex vtxB, String ver, Boolean specialTenantRule, Loader loader, EELFLogger logger) throws AAIException {
Vertex nullVtx = null;
@@ -1289,11 +1332,13 @@ public class DupeTool {
}
if (allTheSame) {
- if (vidA < vidB) {
- preferredVtx = vtxA;
- } else {
- preferredVtx = vtxB;
- }
+ if ( checkAaiUriOk(gts, vtxA, logger) ) {
+ preferredVtx = vtxA;
+ }
+ else if ( checkAaiUriOk(gts, vtxB, logger) ) {
+ preferredVtx = vtxB;
+ }
+ // else we're picking neither because neither one had a working aai-uri index property
} else if (specialTenantRule) {
// They asked us to apply a special rule if it applies
if (vtxIdsConn2A.size() == 2 && vtxANodeType.equals("tenant")) {
@@ -1575,6 +1620,71 @@ public class DupeTool {
}// End of getNodeKeyVals()
+
+
+ /**
+ * makes sure aai-uri exists and can be used to get this node back
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param graph the graph
+ * @param vtx
+ * @param EELFLogger
+ * @return true if aai-uri is populated and the aai-uri-index points to this vtx
+ * @throws AAIException the AAI exception
+ */
+ private Boolean checkAaiUriOk( GraphTraversalSource graph, Vertex origVtx, EELFLogger eLogger )
+ throws AAIException{
+ String aaiUriStr = "";
+ try {
+ Object ob = origVtx.<Object>property("aai-uri").orElse(null);
+ String origVid = origVtx.id().toString();
+ if (ob == null || ob.toString().equals("")) {
+ // It is missing its aai-uri
+ eLogger.debug("DEBUG No [aai-uri] property found for vid = ["
+ + origVid + "] " );
+ return false;
+ }
+ else {
+ aaiUriStr = ob.toString();
+ Iterator <Vertex> verts = graph.V().has("aai-uri",aaiUriStr);
+ int count = 0;
+ while( verts.hasNext() ){
+ count++;
+ Vertex foundV = verts.next();
+ String foundVid = foundV.id().toString();
+ if( !origVid.equals(foundVid) ){
+ eLogger.debug("DEBUG aai-uri key property ["
+ + aaiUriStr + "] for vid = ["
+ + origVid + "] brought back different vertex with vid = ["
+ + foundVid + "]." );
+ return false;
+ }
+ }
+ if( count == 0 ){
+ eLogger.debug("DEBUG aai-uri key property ["
+ + aaiUriStr + "] for vid = ["
+ + origVid + "] could not be used to query for that vertex. ");
+ return false;
+ }
+ else if( count > 1 ){
+ eLogger.debug("DEBUG aai-uri key property ["
+ + aaiUriStr + "] for vid = ["
+ + origVid + "] brought back multiple ("
+ + count + ") vertices instead of just one. ");
+ return false;
+ }
+ }
+ }
+ catch( Exception ex ){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ eLogger.error(" ERROR trying to get node with aai-uri: [" + aaiUriStr + "]" + LogFormatTools.getStackTop(ex));
+ }
+ return true;
+
+ }// End of checkAaiUriOk()
+
/**
* Get values of the key properties for a node as a single string
@@ -1850,5 +1960,14 @@ public class DupeTool {
logger.warn("WARNING from final graph.shutdown()", ex);
}
}
+
+ public int getDupeGroupCount() {
+ return dupeGroupCount;
+ }
+
+ public void setDupeGroupCount(int dgCount) {
+ this.dupeGroupCount = dgCount;
+ }
+
}
diff --git a/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java b/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java
new file mode 100644
index 0000000..ecd95a7
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java
@@ -0,0 +1,906 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
+import org.apache.tinkerpop.gremlin.structure.Element;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.node.ObjectNode;
+import org.codehaus.jackson.type.TypeReference;
+import org.onap.aai.config.PropertyPasswordConfiguration;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.dbmap.DBConnectionType;
+import org.onap.aai.dbmap.InMemoryGraph;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.EdgeRule;
+import org.onap.aai.edges.EdgeRuleQuery;
+import org.onap.aai.edges.enums.AAIDirection;
+import org.onap.aai.edges.enums.EdgeType;
+import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException;
+import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LogFormatTools;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.parsers.uri.URIToObject;
+import org.onap.aai.serialization.db.DBSerializer;
+import org.onap.aai.serialization.engines.InMemoryDBEngine;
+import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.AAISystemExitUtil;
+import org.onap.aai.util.ExceptionTranslator;
+import org.slf4j.MDC;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import java.io.*;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/*
+ * The Class ListEndpoints.
+ */
+public class DynamicPayloadGenerator {
+
+ /*
+ * Create a Dynamic memory graph instance which should not affect the
+ * AAIGraph
+ */
+ private InMemoryGraph inMemGraph = null;
+
+ private InMemoryDBEngine dbEngine;
+ private InputStream sequenceInputStreams;
+ /*
+ * Loader, QueryStyle, ConnectionType for the Serializer
+ */
+ private Loader loader;
+ private String urlBase;
+ private BufferedWriter bw = null;
+ private boolean exitFlag = true;
+ private CommandLineArgs cArgs;
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DynamicPayloadGenerator.class);
+
+ private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL;
+ private static final DBConnectionType type = DBConnectionType.CACHED;
+ private static final ModelType introspectorFactoryType = ModelType.MOXY;
+ private final LoaderFactory loaderFactory;
+ private final EdgeIngestor edgeRules;
+ private final SchemaVersions schemaVersions;
+ private final SchemaVersion version;
+
+ public DynamicPayloadGenerator(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions){
+ this.loaderFactory = loaderFactory;
+ this.edgeRules = edgeIngestor;
+ this.schemaVersions = schemaVersions;
+ this.version = schemaVersions.getDefaultVersion();
+ }
+
+ /**
+ * The run method.
+ *
+ * @param args
+ * the arguments
+ * @param exitFlag true if running from a shell script to call system exit, false if running from scheduled task
+ * @throws AAIException
+ * @throws Exception
+ */
+
+ public static void run (LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions, String[] args, boolean isSystemExit) {
+ //
+ MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName());
+ DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator(loaderFactory, edgeIngestor, schemaVersions);
+ payloadgen.exitFlag = isSystemExit;
+ try {
+ payloadgen.init(args);
+
+ payloadgen.generatePayloads();
+ } catch (AAIException e) {
+ LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
+ } catch (IOException e) {
+ LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
+ }
+ if ( isSystemExit ) {
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ else {
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+
+ }
+ public static void main(String[] args) throws AAIException {
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
+ PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
+ initializer.initialize(ctx);
+ try {
+ ctx.scan(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+ ctx.refresh();
+ } catch (Exception e) {
+ AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
+ LOGGER.error("Problems running tool "+aai.getMessage());
+ LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
+ throw aai;
+
+ }
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ EdgeIngestor edgeIngestor = ctx.getBean(EdgeIngestor.class);
+ SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
+ run (loaderFactory, edgeIngestor, schemaVersions, args, true);
+ }
+
+
+ public void taskExit() {
+ if ( this.exitFlag ) {
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ else {
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ }
+ public void init(String[] args) throws AAIException {
+ cArgs = new CommandLineArgs();
+ JCommander jCommander = new JCommander(cArgs, args);
+ jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName());
+ LOGGER.info("Snapshot file " + cArgs.dataSnapshot);
+
+
+ // TODO- How to add dynamic.properties
+
+ LOGGER.info("output file " + cArgs.output);
+ LOGGER.info("format file " + cArgs.format);
+ LOGGER.info("schema enabled " + cArgs.schemaEnabled);
+ LOGGER.info("Multiple snapshots " + cArgs.isMultipleSnapshot);
+ LOGGER.info("Is Partial Graph " + cArgs.isPartialGraph);
+
+ if (cArgs.config.isEmpty())
+ cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties";
+
+ LOGGER.info("config file " + cArgs.config);
+ if (cArgs.nodePropertyFile.isEmpty())
+ cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json";
+ LOGGER.info("nodePropertyFile file " + cArgs.nodePropertyFile);
+
+ if (cArgs.inputFilterPropertyFile.isEmpty())
+ cArgs.inputFilterPropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/inputFilters.json";
+ LOGGER.info("inputFilterPropertyFile file " + cArgs.inputFilterPropertyFile);
+
+ if (cArgs.isPartialGraph)
+ cArgs.dataSnapshot = cArgs.dataSnapshot+".partial";
+
+ if (!cArgs.isMultipleSnapshot) {
+ validateFile(cArgs.dataSnapshot);
+ } else {
+ // for multiple snapshots dataSnapshot + ".P" is the prefix of the
+ // files
+ sequenceInputStreams = validateMultipleSnapshots(cArgs.dataSnapshot);
+ }
+
+ LOGGER.info("Datasnapshot file " + cArgs.dataSnapshot);
+ AAIConfig.init();
+
+ urlBase = AAIConfig.get("aai.server.url.base", "");
+
+ }
+
+ public void generatePayloads() throws AAIException, IOException {
+
+ List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile);
+ /*
+ * Read the inputFilters which will include for each node-type the regex that needs to be
+ * applied and the filtered-node-type
+ * For eg: complex --> apply regex on cloud-region and then traverse to complex
+ * complex --> filtered-node-type: cloud-region, filters: include regex on cloud-region
+ */
+ /*
+ * Example:
+ * { "cloud-region" :
+ * {"filtered-node-type":"cloud-region",
+ * "filters": [ { "property": "cloud-owner", "regex": "att-aic" },
+ * { "property": "cloud-region-id", "regex": "M*" },
+ * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" }
+ * ] },
+ * "complex" : {
+ * "filtered-node-type":"cloud-region",
+ * "filters": [ { "property": "cloud-owner", "regex": "att-aic" },
+ * { "property": "cloud-region-id", "regex": "M*" },
+ * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" }
+ * ] },
+ *
+ * } }
+ */
+ Map<String, Map<String, String>> inputFilters = readInputFilterPropertyFile(cArgs.inputFilterPropertyFile);
+ Map<String, String> filteredNodeTypes = findFilteredNodeTypes(cArgs.inputFilterPropertyFile);
+ // Read the input filter criteria
+ LOGGER.info("Load the Graph");
+
+ this.loadGraph();
+ LOGGER.info("Generate payload");
+ this.generatePayload(nodeFilters, inputFilters, filteredNodeTypes);
+ LOGGER.info("Close graph");
+ this.closeGraph();
+
+ }
+
+ private List<Map<String, List<String>>> readFile(String inputFile) throws IOException {
+
+ // validate that we can read the inputFile
+ validateFile(inputFile);
+
+ InputStream is = new FileInputStream(inputFile);
+ Scanner scanner = new Scanner(is);
+ String jsonFile = scanner.useDelimiter("\\Z").next();
+ scanner.close();
+
+ List<Map<String, List<String>>> allNodes = new ArrayList<>();
+ Map<String, List<String>> filterCousins = new HashMap<>();
+ Map<String, List<String>> filterParents = new HashMap<>();
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ JsonNode rootNode = mapper.readTree(jsonFile);
+
+ Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
+
+ while (nodeFields.hasNext()) {
+ Entry<String, JsonNode> entry = nodeFields.next();
+ String nodeType = entry.getKey();
+ JsonNode nodeProperty = entry.getValue();
+
+ JsonNode cousinFilter = nodeProperty.path("cousins");
+ JsonNode parentFilter = nodeProperty.path("parents");
+ List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(),
+ new TypeReference<ArrayList<String>>() {
+ });
+
+ List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(),
+ new TypeReference<ArrayList<String>>() {
+ });
+ for (String cousin : cousins) {
+ LOGGER.info("Cousins-Filtered " + cousin);
+ }
+ for (String parent : parents) {
+ LOGGER.info("Parents-Filtered " + parent);
+ }
+ filterCousins.put(nodeType, cousins);
+ filterParents.put(nodeType, parents);
+
+ }
+
+ allNodes.add(filterCousins);
+ allNodes.add(filterParents);
+ return allNodes;
+
+ }
+
+ /* Example:
+{
+ "cloud-region" : {
+ "filtered-node-type" :"cloud-region",
+ "filters": [
+ {
+ "property": "cloud-owner",
+ "regex": "att-aic"
+ },
+ {
+ "property": "cloud-region-id",
+ "regex": "M*"
+ },
+ {
+ "property": "cloud-region-version",
+ "regex": "aic2.5|aic3.0"
+ }
+ ]
+ },
+ "complex" : {
+ "filters":[
+ ]
+
+ }
+}
+*/
+ private Map<String, Map<String, String>> readInputFilterPropertyFile(String inputFile) throws IOException {
+
+ validateFile(inputFile);
+
+ InputStream is = new FileInputStream(inputFile);
+ Scanner scanner = new Scanner(is);
+ String jsonFile = scanner.useDelimiter("\\Z").next();
+ scanner.close();
+
+ Map<String, Map<String, String>> propToRegex = new HashMap<String, Map<String, String>>();
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ JsonNode rootNode = mapper.readTree(jsonFile);
+
+ Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
+
+ while (nodeFields.hasNext()) {
+ Entry<String, JsonNode> entry = nodeFields.next();
+ String nodeType = entry.getKey();
+ JsonNode nodeProperty = entry.getValue();
+
+ JsonNode filter = nodeProperty.path("filters");
+ List<JsonNode> filterMap = new ObjectMapper().readValue(filter.traverse(),
+ new TypeReference<ArrayList<JsonNode>>() {
+ });
+ HashMap<String, String> filterMaps = new HashMap<String, String>();
+ for (JsonNode n : filterMap) {
+ filterMaps.put(n.get("property").asText(), n.get("regex").asText());
+ }
+
+ propToRegex.put(nodeType, filterMaps);
+ }
+ return (propToRegex);
+ }
+
+ private Map<String, String> findFilteredNodeTypes(String inputFile) throws IOException {
+
+ validateFile(inputFile);
+
+ InputStream is = new FileInputStream(inputFile);
+ Scanner scanner = new Scanner(is);
+ String jsonFile = scanner.useDelimiter("\\Z").next();
+ scanner.close();
+
+ Map<String, String> filteredNodeTypes = new HashMap<String, String>();
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ JsonNode rootNode = mapper.readTree(jsonFile);
+
+ Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
+
+ while (nodeFields.hasNext()) {
+ Entry<String, JsonNode> entry = nodeFields.next();
+ String nodeType = entry.getKey();
+ JsonNode nodeProperty = entry.getValue();
+
+ JsonNode filter = nodeProperty.path("filtered-node-type");
+
+ filteredNodeTypes.put(nodeType, filter.asText());
+ }
+ return (filteredNodeTypes);
+ }
+
+ public void loadGraph() throws IOException {
+
+ loadGraphIntoMemory();
+ buildDbEngine();
+
+ }
+
+ private void loadGraphIntoMemory() throws IOException {
+ if (!(cArgs.isMultipleSnapshot)) {
+ inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled,
+ cArgs.isPartialGraph);
+ } else {
+ inMemGraph = new InMemoryGraph.Builder().build(sequenceInputStreams, cArgs.config, cArgs.schemaEnabled,
+ cArgs.isPartialGraph);
+ }
+ }
+
+ private void buildDbEngine() {
+ // TODO : parametrise version
+ loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+
+ dbEngine = new InMemoryDBEngine(queryStyle, type, loader, inMemGraph.getGraph());
+ dbEngine.startTransaction();
+ }
+
+ private void generatePayload(List<Map<String, List<String>>> nodeFilters,
+ Map<String, Map<String, String>> inputFilters, Map<String, String> filteredNodeTypes)
+ throws AAIException, IOException {
+
+ Map<String, List<String>> filterCousinsMap = nodeFilters.get(0);
+ Map<String, List<String>> filterParentsMap = nodeFilters.get(1);
+ Set<String> nodeTypes = filterCousinsMap.keySet();
+
+ for (String nodeType : nodeTypes) {
+ if ("DMAAP-MR".equals(cArgs.format)) {
+ bw = createFile(nodeType + ".json");
+ }
+ List<String> filterCousins = filterCousinsMap.get(nodeType);
+ List<String> filterParents = filterParentsMap.get(nodeType);
+ Map<String, String> nodeInputFilterMap = inputFilters.get(nodeType);
+ String filteredNodeType = nodeType;
+ if(filteredNodeTypes.get(nodeType) != null && !filteredNodeTypes.get(nodeType).isEmpty())
+ filteredNodeType = filteredNodeTypes.get(nodeType);
+ readVertices(nodeType, filterCousins, filterParents, nodeInputFilterMap, filteredNodeType);
+ if(bw != null)
+ bw.close();
+ LOGGER.info("All Done-" + nodeType);
+ }
+
+ }
+
+ private BufferedWriter createFile(String outfileName) throws IOException {
+ // FileLocation
+ String fileName = outfileName;
+ File outFile = new File(fileName);
+ FileWriter fw = null;
+ LOGGER.info(" Will write to " + fileName);
+ try {
+ fw = new FileWriter(outFile.getAbsoluteFile());
+ } catch (IOException i) {
+ String emsg = "Unable to write to " + fileName + " Exception = " + i.getMessage();
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ throw i;
+ }
+ return new BufferedWriter(fw);
+ }
+
+ private void createDirectory(String dirName) throws IOException {
+ // FileLocation
+ Path pathDir = null;
+ try {
+ pathDir = Paths.get(dirName);
+ } catch (InvalidPathException i) {
+ String emsg = "Directory " + dirName + " could not be found.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ try {
+ Files.createDirectories(pathDir);
+ } catch (Exception e) {
+ String emsg = "Directory " + dirName + " could not be created. Exception = " + e.getMessage();
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ }
+
+ public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents,
+ Map<String, String> nodeInputFilters, String filteredNodeType) throws AAIException, IOException {
+
+ DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
+
+ /*
+ * Start with nodeType you need to filter and then traverse to the actual nodeType
+ */
+ GraphTraversal<Vertex, Vertex> gtraversal = inMemGraph.getGraph().traversal().V().has("aai-node-type",
+ filteredNodeType);
+
+
+ // input regex
+ if (nodeInputFilters != null && (!nodeInputFilters.isEmpty())) {
+ for (Map.Entry<String, String> entry : nodeInputFilters.entrySet()) {
+ String property = entry.getKey();
+ String regex = entry.getValue();
+ Pattern pa = Pattern.compile(regex);
+
+ gtraversal = gtraversal.has(property, P.test((t, p) -> {
+ Matcher m = ((Pattern) p).matcher((CharSequence) t);
+ boolean b = m.matches();
+ return b;
+ }, pa));
+ }
+ }
+
+ /*
+ * Tenant, AZ, Complex, Zone, pserver come here
+ */
+ if (!filteredNodeType.equals(nodeType)) {
+
+ EdgeRuleQuery treeEdgeRuleQuery = new EdgeRuleQuery
+ .Builder(filteredNodeType, nodeType)
+ .edgeType(EdgeType.TREE)
+ .build();
+
+ EdgeRuleQuery cousinEdgeQuery = new EdgeRuleQuery
+ .Builder(filteredNodeType, nodeType)
+ .edgeType(EdgeType.COUSIN)
+ .build();
+
+ EdgeRule rule = null;
+ boolean hasTreeEdgeRule = true;
+
+ try {
+ rule = edgeRules.getRule(treeEdgeRuleQuery);
+ } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
+ hasTreeEdgeRule = false;
+ }
+
+ if(!hasTreeEdgeRule) {
+ try {
+ rule = edgeRules.getRule(cousinEdgeQuery);
+ } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
+ LOGGER.error("Unable to get a tree or cousin edge between {} and {}", filteredNodeType, nodeType);
+ return;
+ }
+ }
+
+ if (rule.getDirection().toString().equals(AAIDirection.OUT.toString())) {
+ gtraversal.out(rule.getLabel()).has("aai-node-type", nodeType);
+ } else {
+ gtraversal.in(rule.getLabel()).has("aai-node-type", nodeType);
+ }
+
+ }
+
+ String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP;
+ createDirectory(dirName);
+ // TODO: Formatter
+
+ if ("DMAAP-MR".equals(cArgs.format)) {
+ while (gtraversal.hasNext()) {
+ if (bw != null)
+ bw = createFile(nodeType + ".json");
+ Vertex node = gtraversal.next();
+ Introspector nodeObj = serializer.getLatestVersionView(node);
+ createPayloadForDmaap(node, nodeObj);
+ }
+ } else {
+ if ("PAYLOAD".equals(cArgs.format)) {
+ int counter = 0;
+ while (gtraversal.hasNext()) {
+ Vertex node = gtraversal.next();
+ try {
+ counter++;
+ String filename = dirName + counter + "-" + nodeType + ".json";
+ bw = createFile(filename);
+ Introspector obj = loader.introspectorFromName(nodeType);
+ Set<Vertex> seen = new HashSet<>();
+ int depth = AAIProperties.MAXIMUM_DEPTH;
+ boolean nodeOnly = false;
+
+ Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly);
+ TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree);
+ serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins,
+ filterParents);
+ createPayloadForPut(obj);
+ if(bw != null)
+ bw.close();
+
+ URI uri = serializer.getURIForVertex(node);
+ String filenameWithUri = dirName + counter + "-" + nodeType + ".txt";
+ bw = createFile(filenameWithUri);
+ bw.write(uri.toString());
+ bw.newLine();
+ bw.close();
+ } catch (Exception e) {
+ String emsg = "Caught exception while processing [" + counter + "-" + nodeType + "] continuing";
+ System.out.println(emsg);
+ LOGGER.error(emsg);
+
+ }
+ }
+ }
+ }
+
+ }
+
+ public void createPayloadForPut(Introspector nodeObj) throws IOException {
+
+ String entityJson = nodeObj.marshal(false);
+ ObjectMapper mapper = new ObjectMapper();
+
+ ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson);
+ rootNode.remove("resource-version");
+
+ bw.newLine();
+ bw.write(rootNode.toString());
+ bw.newLine();
+ }
+
+ public void createPayloadForDmaap(Vertex node, Introspector nodeObj)
+ throws AAIException, UnsupportedEncodingException {
+
+ DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
+
+ URI uri = serializer.getURIForVertex(node);
+
+ String sourceOfTruth = "";
+ HashMap<String, Introspector> relatedVertices = new HashMap<>();
+ List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node);
+
+ for (Vertex vertex : vertexChain) {
+ try {
+
+ Introspector vertexObj = serializer.getVertexProperties(vertex);
+
+ relatedVertices.put(vertexObj.getObjectId(), vertexObj);
+ } catch (AAIUnknownObjectException e) {
+ LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
+ }
+
+ }
+
+ String transactionId = "TXID";
+ createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices);
+
+ }
+
+ public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj,
+ Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException {
+
+ String action = "CREATE";
+ final Introspector notificationEvent = loader.introspectorFromName("notification-event");
+
+ try {
+ Introspector eventHeader = loader.introspectorFromName("notification-event-header");
+ URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects);
+
+ String entityLink = urlBase + version + uri;
+
+ notificationEvent.setValue("cambria-partition", "AAI");
+
+ eventHeader.setValue("entity-link", entityLink);
+ eventHeader.setValue("action", action);
+ eventHeader.setValue("entity-type", obj.getDbName());
+ eventHeader.setValue("top-entity-type", parser.getTopEntityName());
+ eventHeader.setValue("source-name", sourceOfTruth);
+ eventHeader.setValue("version", version.toString());
+ eventHeader.setValue("id", transactionId);
+ eventHeader.setValue("event-type", "AAI-BASELINE");
+ if (eventHeader.getValue("domain") == null) {
+ eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK"));
+ }
+
+ if (eventHeader.getValue("sequence-number") == null) {
+ eventHeader.setValue("sequence-number",
+ AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK"));
+ }
+
+ if (eventHeader.getValue("severity") == null) {
+ eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK"));
+ }
+
+ if (eventHeader.getValue("id") == null) {
+ eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString());
+
+ }
+
+ if (eventHeader.getValue("timestamp") == null) {
+ eventHeader.setValue("timestamp", genDate());
+ }
+
+ List<Object> parentList = parser.getParentList();
+ parentList.clear();
+
+ if (!parser.getTopEntity().equals(parser.getEntity())) {
+ Introspector child;
+ String json = obj.marshal(false);
+ child = parser.getLoader().unmarshal(parser.getEntity().getName(), json);
+ parentList.add(child.getUnderlyingObject());
+ }
+
+ final Introspector eventObject;
+
+ String json = "";
+ if (parser.getTopEntity().equals(parser.getEntity())) {
+ json = obj.marshal(false);
+ eventObject = loader.unmarshal(obj.getName(), json);
+ } else {
+ json = parser.getTopEntity().marshal(false);
+
+ eventObject = loader.unmarshal(parser.getTopEntity().getName(), json);
+ }
+ notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject());
+ notificationEvent.setValue("entity", eventObject.getUnderlyingObject());
+
+ String entityJson = notificationEvent.marshal(false);
+
+ bw.newLine();
+ bw.write(entityJson);
+
+ } catch (AAIUnknownObjectException e) {
+ LOGGER.error("Fatal error - notification-event-header object not found!");
+ } catch (Exception e) {
+ LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e));
+ }
+ }
+
+ private void closeGraph() {
+ inMemGraph.getGraph().tx().rollback();
+ inMemGraph.getGraph().close();
+ }
+
+ public static String genDate() {
+ Date date = new Date();
+ DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS");
+ return formatter.format(date);
+ }
+
+ public static String genDate2() {
+ Date date = new Date();
+ DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
+ return formatter.format(date);
+ }
+
+ private void validateFile(String filename) {
+ File f = new File(filename);
+ if (!f.exists()) {
+ String emsg = "File " + filename + " could not be found.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ } else if (!f.canRead()) {
+ String emsg = "File " + filename + " could not be read.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ } else if (f.length() == 0) {
+ String emsg = "File " + filename + " had no data.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ }
+
+ private InputStream validateMultipleSnapshots(String filenamePrefix) {
+ if (filenamePrefix == null || filenamePrefix.length() == 0) {
+ String emsg = "No snapshot path was provided.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ String targetDir = ".";
+ int lastSeparator = filenamePrefix.lastIndexOf(File.separator);
+
+ LOGGER.info("File separator=[" + File.separator + "] lastSeparator=" + lastSeparator + " filenamePrefix="
+ + filenamePrefix);
+ if (lastSeparator >= 0) {
+ targetDir = filenamePrefix.substring(0, lastSeparator);
+ LOGGER.info("targetDir=" + targetDir);
+ }
+ if (targetDir.length() == 0) {
+ String emsg = "No snapshot directory was found in path:" + filenamePrefix;
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ String prefix = filenamePrefix.substring(lastSeparator + 1);
+ if (prefix == null || prefix.length() == 0) {
+ String emsg = "No snapshot file prefix was provided.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ long timeA = System.nanoTime();
+
+ ArrayList<File> snapFilesArr = new ArrayList<File>();
+ String thisSnapPrefix = prefix + ".P";
+ File fDir = new File(targetDir); // Snapshot directory
+ File[] allFilesArr = fDir.listFiles();
+ for (File snapFile : allFilesArr) {
+ String snapFName = snapFile.getName();
+ if (snapFName.startsWith(thisSnapPrefix)) {
+ snapFilesArr.add(snapFile);
+ }
+ }
+
+ if (snapFilesArr.isEmpty()) {
+ String fullFName = targetDir + AAIConstants.AAI_FILESEP + thisSnapPrefix;
+ String emsg = "Snapshot files " + fullFName + "* could not be found.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+
+ int fCount = snapFilesArr.size();
+ Iterator<File> fItr = snapFilesArr.iterator();
+ Vector<InputStream> inputStreamsV = new Vector<>();
+ for (int i = 0; i < fCount; i++) {
+ File f = snapFilesArr.get(i);
+ String fname = f.getName();
+ if (!f.canRead()) {
+ String emsg = "Snapshot file " + fname + " could not be read.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ } else if (f.length() == 0) {
+ String emsg = "Snapshot file " + fname + " had no data.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ String fullFName = targetDir + AAIConstants.AAI_FILESEP + fname;
+ InputStream fis = null;
+ try {
+ fis = new FileInputStream(fullFName);
+ } catch (FileNotFoundException e) {
+ // should not happen at this point
+ String emsg = "Snapshot file " + fullFName + " could not be found";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ inputStreamsV.add(fis);
+ }
+ // Now add inputStreams.elements() to the Vector,
+ InputStream sis = new SequenceInputStream(inputStreamsV.elements());
+ return (sis);
+ }
+
+ public InMemoryGraph getInMemGraph() {
+ return inMemGraph;
+ }
+
+ public void setInMemGraph(InMemoryGraph inMemGraph) {
+ this.inMemGraph = inMemGraph;
+ }
+}
+
+class CommandLineArgs {
+
+ @Parameter(names = "--help", help = true)
+ public boolean help;
+
+ @Parameter(names = "-d", description = "snapshot file to be loaded", required = true)
+ public String dataSnapshot;
+
+ @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1)
+ public boolean schemaEnabled = true;
+
+ @Parameter(names = "-c", description = "location of configuration file")
+ public String config = "";
+
+ @Parameter(names = "-o", description = "output location")
+ public String output = "";
+
+ @Parameter(names = "-f", description = "format of output")
+ public String format = "PAYLOAD";
+
+ @Parameter(names = "-n", description = "Node input file")
+ public String nodePropertyFile = "";
+
+ @Parameter(names = "-m", description = "multipe snapshots or not", arity = 1)
+ public boolean isMultipleSnapshot = false;
+
+ @Parameter(names = "-i", description = "input filter configuration file")
+ public String inputFilterPropertyFile = "";
+
+ @Parameter(names = "-p", description = "Use the partial graph", arity = 1)
+ public boolean isPartialGraph = true;
+
+}
diff --git a/src/main/java/org/onap/aai/dbgen/GraphSONPartialIO.java b/src/main/java/org/onap/aai/dbgen/GraphSONPartialIO.java
new file mode 100644
index 0000000..915db69
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/GraphSONPartialIO.java
@@ -0,0 +1,158 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen;
+
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.Io;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
+import org.apache.tinkerpop.gremlin.structure.io.Mapper;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Optional;
+import java.util.function.Consumer;
+
+/**
+ * Constructs GraphSON IO implementations given a {@link Graph} and {@link IoRegistry}. Implementers of the {@link Graph}
+ * interfaces should see the {@link GraphSONMapper} for information on the expectations for the {@link IoRegistry}.
+ *
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public final class GraphSONPartialIO implements Io<GraphSONPartialReader.Builder, GraphSONWriter.Builder, GraphSONMapper.Builder> {
+ private final IoRegistry registry;
+ private final Graph graph;
+ private final Optional<Consumer<Mapper.Builder>> onMapper;
+ private final GraphSONVersion version;
+
+ private GraphSONPartialIO(final Builder builder) {
+ this.registry = builder.registry;
+ this.graph = builder.graph;
+ this.onMapper = Optional.ofNullable(builder.onMapper);
+ this.version = builder.version;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public GraphSONPartialReader.Builder reader() {
+ return GraphSONPartialReader.build().mapper(mapper().create());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public GraphSONWriter.Builder writer() {
+ return GraphSONWriter.build().mapper(mapper().create());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public GraphSONMapper.Builder mapper() {
+ final GraphSONMapper.Builder builder = (null == this.registry) ?
+ GraphSONMapper.build().version(version) : GraphSONMapper.build().version(version).addRegistry(this.registry);
+ onMapper.ifPresent(c -> c.accept(builder));
+ return builder;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void writeGraph(final String file) throws IOException {
+ try (final OutputStream out = new FileOutputStream(file)) {
+ writer().create().writeGraph(out, graph);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void readGraph(final String file) throws IOException {
+ try (final InputStream in = new FileInputStream(file)) {
+ reader().create().readGraph(in, graph);
+ }
+ }
+
+ /**
+ * Create a new builder using the default version of GraphSON.
+ */
+ public static Io.Builder<GraphSONPartialIO> build() {
+ return build(GraphSONVersion.V1_0);
+ }
+
+ /**
+ * Create a new builder using the specified version of GraphSON.
+ */
+ public static Io.Builder<GraphSONPartialIO> build(final GraphSONVersion version) {
+ return new Builder(version);
+ }
+
+ public final static class Builder implements Io.Builder<GraphSONPartialIO> {
+
+ private IoRegistry registry = null;
+ private Graph graph;
+ private Consumer<Mapper.Builder> onMapper = null;
+ private final GraphSONVersion version;
+
+ Builder(final GraphSONVersion version) {
+ this.version = version;
+ }
+
+ /**
+ * @deprecated As of release 3.2.2, replaced by {@link #onMapper(Consumer)}.
+ */
+ @Deprecated
+ @Override
+ public Io.Builder<GraphSONPartialIO> registry(final IoRegistry registry) {
+ this.registry = registry;
+ return this;
+ }
+
+ @Override
+ public Io.Builder<? extends Io> onMapper(final Consumer<Mapper.Builder> onMapper) {
+ this.onMapper = onMapper;
+ return this;
+ }
+
+ @Override
+ public Io.Builder<GraphSONPartialIO> graph(final Graph g) {
+ this.graph = g;
+ return this;
+ }
+
+ @Override
+ public GraphSONPartialIO create() {
+ if (null == graph) throw new IllegalArgumentException("The graph argument was not specified");
+ return new GraphSONPartialIO(this);
+ }
+ }
+}
diff --git a/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java b/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java
new file mode 100644
index 0000000..ebe2180
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java
@@ -0,0 +1,354 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen;
+
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
+import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
+import org.apache.tinkerpop.gremlin.structure.io.Mapper;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
+import org.apache.tinkerpop.gremlin.structure.util.Attachable;
+import org.apache.tinkerpop.gremlin.structure.util.Host;
+import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
+import org.apache.tinkerpop.gremlin.util.function.FunctionUtils;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.apache.tinkerpop.shaded.jackson.core.type.TypeReference;
+import org.apache.tinkerpop.shaded.jackson.databind.JsonNode;
+import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper;
+import org.apache.tinkerpop.shaded.jackson.databind.node.JsonNodeType;
+import org.onap.aai.dbmap.InMemoryGraph;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+/**
+ * This is a Wrapper around the GraphsonReader class
+ * The idea is to rewrite methods that are customized for A&AI
+ * GraphsonReader is a final class . hence the use of the Wrapper
+ * instead of inheriting-overwriting
+ *
+ *
+ */
+public final class GraphSONPartialReader implements GraphReader {
+ private final ObjectMapper mapper ;
+ private final long batchSize ;
+ private final GraphSONVersion version ;
+ private boolean unwrapAdjacencyList = false;
+ private final GraphSONReader reader;
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(InMemoryGraph.class);
+
+ final TypeReference<Map<String, Object>> mapTypeReference = new TypeReference<Map<String, Object>>() {
+ };
+
+ private GraphSONPartialReader(final Builder builder) {
+ mapper = builder.mapper.createMapper();
+ batchSize = builder.batchSize;
+ unwrapAdjacencyList = builder.unwrapAdjacencyList;
+ version = ((GraphSONMapper)builder.mapper).getVersion();
+ reader = GraphSONReader.build().create();
+ }
+
+ /**
+ * Read data into a {@link Graph} from output generated by any of the {@link GraphSONWriter} {@code writeVertex} or
+ * {@code writeVertices} methods or by {@link GryoWriter#writeGraph(OutputStream, Graph)}.
+ *
+ * @param inputStream a stream containing an entire graph of vertices and edges as defined by the accompanying
+ * {@link GraphSONWriter#writeGraph(OutputStream, Graph)}.
+ * @param graphToWriteTo the graph to write to when reading from the stream.
+ */
+ @Override
+ public void readGraph(final InputStream inputStream, final Graph graphToWriteTo) throws IOException {
+ // dual pass - create all vertices and store to cache the ids. then create edges. as long as we don't
+ // have vertex labels in the output we can't do this single pass
+ LOGGER.info("Read the Partial Graph");
+ final Map<StarGraph.StarVertex,Vertex> cache = new HashMap<>();
+ final AtomicLong counter = new AtomicLong(0);
+
+ final boolean supportsTx = graphToWriteTo.features().graph().supportsTransactions();
+ final Graph.Features.EdgeFeatures edgeFeatures = graphToWriteTo.features().edge();
+
+ readVertexStrings(inputStream).<Vertex>map(FunctionUtils.wrapFunction(line -> readVertex(new ByteArrayInputStream(line.getBytes()), null, null, Direction.IN))).forEach(vertex -> {
+ try{
+ final Attachable<Vertex> attachable = (Attachable<Vertex>) vertex;
+ cache.put((StarGraph.StarVertex) attachable.get(), attachable.attach(Attachable.Method.create(graphToWriteTo)));
+ if (supportsTx && counter.incrementAndGet() % batchSize == 0)
+ graphToWriteTo.tx().commit();
+ }
+ catch(Exception ex){
+ LOGGER.info("Error in reading vertex from graphson"+vertex.toString());
+ }
+ });
+
+ cache.entrySet().forEach(kv -> kv.getKey().edges(Direction.IN).forEachRemaining(e -> {
+ try{
+ // can't use a standard Attachable attach method here because we have to use the cache for those
+ // graphs that don't support userSuppliedIds on edges. note that outVertex/inVertex methods return
+ // StarAdjacentVertex whose equality should match StarVertex.
+ final Vertex cachedOutV = cache.get(e.outVertex());
+ final Vertex cachedInV = cache.get(e.inVertex());
+
+ if(cachedOutV != null && cachedInV != null){
+
+ final Edge newEdge = edgeFeatures.willAllowId(e.id()) ? cachedOutV.addEdge(e.label(), cachedInV, T.id, e.id()) : cachedOutV.addEdge(e.label(), cachedInV);
+ e.properties().forEachRemaining(p -> newEdge.property(p.key(), p.value()));
+ }
+ else{
+ LOGGER.debug("Ghost edges from "+ cachedOutV + " to "+ cachedInV);
+
+ }
+ if (supportsTx && counter.incrementAndGet() % batchSize == 0)
+ graphToWriteTo.tx().commit();
+ }
+ catch(Exception ex){
+ LOGGER.info("Error in writing vertex into graph"+e.toString());
+ }
+ }));
+
+ if (supportsTx) graphToWriteTo.tx().commit();
+ }
+
+ /**
+ * Read {@link Vertex} objects from output generated by any of the {@link GraphSONWriter} {@code writeVertex} or
+ * {@code writeVertices} methods or by {@link GraphSONWriter#writeGraph(OutputStream, Graph)}.
+ *
+ * @param inputStream a stream containing at least one {@link Vertex} as defined by the accompanying
+ * {@link GraphWriter#writeVertices(OutputStream, Iterator, Direction)} or
+ * {@link GraphWriter#writeVertices(OutputStream, Iterator)} methods.
+ * @param vertexAttachMethod a function that creates re-attaches a {@link Vertex} to a {@link Host} object.
+ * @param edgeAttachMethod a function that creates re-attaches a {@link Edge} to a {@link Host} object.
+ * @param attachEdgesOfThisDirection only edges of this direction are passed to the {@code edgeMaker}.
+ */
+ @Override
+ public Iterator<Vertex> readVertices(final InputStream inputStream,
+ final Function<Attachable<Vertex>, Vertex> vertexAttachMethod,
+ final Function<Attachable<Edge>, Edge> edgeAttachMethod,
+ final Direction attachEdgesOfThisDirection) throws IOException {
+ // return readVertexStrings(inputStream).<Vertex>map(FunctionUtils.wrapFunction(line -> readVertex(new ByteArrayInputStream(line.getBytes()), vertexAttachMethod, edgeAttachMethod, attachEdgesOfThisDirection))).iterator();
+ return reader.readVertices(inputStream, vertexAttachMethod, edgeAttachMethod, attachEdgesOfThisDirection);
+
+ }
+
+ /**
+ * Read a {@link Vertex} from output generated by any of the {@link GraphSONWriter} {@code writeVertex} or
+ * {@code writeVertices} methods or by {@link GraphSONWriter#writeGraph(OutputStream, Graph)}.
+ *
+ * @param inputStream a stream containing at least a single vertex as defined by the accompanying
+ * {@link GraphWriter#writeVertex(OutputStream, Vertex)}.
+ * @param vertexAttachMethod a function that creates re-attaches a {@link Vertex} to a {@link Host} object.
+ */
+ @Override
+ public Vertex readVertex(final InputStream inputStream, final Function<Attachable<Vertex>, Vertex> vertexAttachMethod) throws IOException {
+ return reader.readVertex(inputStream, vertexAttachMethod);
+ }
+
+ /**
+ * Read a {@link Vertex} from output generated by any of the {@link GraphSONWriter} {@code writeVertex} or
+ * {@code writeVertices} methods or by {@link GraphSONWriter#writeGraph(OutputStream, Graph)}.
+ *
+ * @param inputStream a stream containing at least one {@link Vertex} as defined by the accompanying
+ * {@link GraphWriter#writeVertices(OutputStream, Iterator, Direction)} method.
+ * @param vertexAttachMethod a function that creates re-attaches a {@link Vertex} to a {@link Host} object.
+ * @param edgeAttachMethod a function that creates re-attaches a {@link Edge} to a {@link Host} object.
+ * @param attachEdgesOfThisDirection only edges of this direction are passed to the {@code edgeMaker}.
+ */
+ @Override
+ public Vertex readVertex(final InputStream inputStream,
+ final Function<Attachable<Vertex>, Vertex> vertexAttachMethod,
+ final Function<Attachable<Edge>, Edge> edgeAttachMethod,
+ final Direction attachEdgesOfThisDirection) throws IOException {
+
+ return reader.readVertex(inputStream, vertexAttachMethod, edgeAttachMethod, attachEdgesOfThisDirection);
+ }
+
+ /**
+ * Read an {@link Edge} from output generated by {@link GraphSONWriter#writeEdge(OutputStream, Edge)} or via
+ * an {@link Edge} passed to {@link GraphSONWriter#writeObject(OutputStream, Object)}.
+ *
+ * @param inputStream a stream containing at least one {@link Edge} as defined by the accompanying
+ * {@link GraphWriter#writeEdge(OutputStream, Edge)} method.
+ * @param edgeAttachMethod a function that creates re-attaches a {@link Edge} to a {@link Host} object.
+ */
+ @Override
+ public Edge readEdge(final InputStream inputStream, final Function<Attachable<Edge>, Edge> edgeAttachMethod) throws IOException {
+ /*if (version == GraphSONVersion.V1_0) {
+ final Map<String, Object> edgeData = mapper.readValue(inputStream, mapTypeReference);
+
+ final Map<String, Object> edgeProperties = edgeData.containsKey(GraphSONTokens.PROPERTIES) ?
+ (Map<String, Object>) edgeData.get(GraphSONTokens.PROPERTIES) : Collections.EMPTY_MAP;
+ final DetachedEdge edge = new DetachedEdge(edgeData.get(GraphSONTokens.ID),
+ edgeData.get(GraphSONTokens.LABEL).toString(),
+ edgeProperties,
+ Pair.with(edgeData.get(GraphSONTokens.OUT), edgeData.get(GraphSONTokens.OUT_LABEL).toString()),
+ Pair.with(edgeData.get(GraphSONTokens.IN), edgeData.get(GraphSONTokens.IN_LABEL).toString()));
+
+ return edgeAttachMethod.apply(edge);
+ } else {
+ return edgeAttachMethod.apply((DetachedEdge) mapper.readValue(inputStream, Edge.class));
+ }*/
+ return reader.readEdge(inputStream, edgeAttachMethod);
+ }
+
+ /**
+ * Read a {@link VertexProperty} from output generated by
+ * {@link GraphSONWriter#writeVertexProperty(OutputStream, VertexProperty)} or via an {@link VertexProperty} passed
+ * to {@link GraphSONWriter#writeObject(OutputStream, Object)}.
+ *
+ * @param inputStream a stream containing at least one {@link VertexProperty} as written by the accompanying
+ * {@link GraphWriter#writeVertexProperty(OutputStream, VertexProperty)} method.
+ * @param vertexPropertyAttachMethod a function that creates re-attaches a {@link VertexProperty} to a
+ * {@link Host} object.
+ */
+ @Override
+ public VertexProperty readVertexProperty(final InputStream inputStream,
+ final Function<Attachable<VertexProperty>, VertexProperty> vertexPropertyAttachMethod) throws IOException {
+ /*if (version == GraphSONVersion.V1_0) {
+ final Map<String, Object> vpData = mapper.readValue(inputStream, mapTypeReference);
+ final Map<String, Object> metaProperties = (Map<String, Object>) vpData.get(GraphSONTokens.PROPERTIES);
+ final DetachedVertexProperty vp = new DetachedVertexProperty(vpData.get(GraphSONTokens.ID),
+ vpData.get(GraphSONTokens.LABEL).toString(),
+ vpData.get(GraphSONTokens.VALUE), metaProperties);
+ return vertexPropertyAttachMethod.apply(vp);
+ } else {
+ return vertexPropertyAttachMethod.apply((DetachedVertexProperty) mapper.readValue(inputStream, VertexProperty.class));
+ }*/
+ return reader.readVertexProperty(inputStream, vertexPropertyAttachMethod);
+ }
+
+ /**
+ * Read a {@link Property} from output generated by {@link GraphSONWriter#writeProperty(OutputStream, Property)} or
+ * via an {@link Property} passed to {@link GraphSONWriter#writeObject(OutputStream, Object)}.
+ *
+ * @param inputStream a stream containing at least one {@link Property} as written by the accompanying
+ * {@link GraphWriter#writeProperty(OutputStream, Property)} method.
+ * @param propertyAttachMethod a function that creates re-attaches a {@link Property} to a {@link Host} object.
+ */
+ @Override
+ public Property readProperty(final InputStream inputStream,
+ final Function<Attachable<Property>, Property> propertyAttachMethod) throws IOException {
+ /*if (version == GraphSONVersion.V1_0) {
+ final Map<String, Object> propertyData = mapper.readValue(inputStream, mapTypeReference);
+ final DetachedProperty p = new DetachedProperty(propertyData.get(GraphSONTokens.KEY).toString(), propertyData.get(GraphSONTokens.VALUE));
+ return propertyAttachMethod.apply(p);
+ } else {
+ return propertyAttachMethod.apply((DetachedProperty) mapper.readValue(inputStream, Property.class));
+ }*/
+ return reader.readProperty(inputStream, propertyAttachMethod);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <C> C readObject(final InputStream inputStream, final Class<? extends C> clazz) throws IOException {
+ return mapper.readValue(inputStream, clazz);
+ }
+
+ private Stream<String> readVertexStrings(final InputStream inputStream) throws IOException {
+ if (unwrapAdjacencyList) {
+ final JsonNode root = mapper.readTree(inputStream);
+ final JsonNode vertices = root.get(GraphSONTokens.VERTICES);
+ if (!vertices.getNodeType().equals(JsonNodeType.ARRAY)) throw new IOException(String.format("The '%s' key must be an array", GraphSONTokens.VERTICES));
+ return IteratorUtils.stream(vertices.elements()).map(Object::toString);
+ } else {
+ final BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));
+ return br.lines();
+ }
+
+ }
+
+
+ public static Builder build() {
+ return new Builder();
+ }
+
+ public final static class Builder implements ReaderBuilder<GraphSONPartialReader> {
+ private long batchSize = 10000;
+
+ private Mapper<ObjectMapper> mapper = GraphSONMapper.build().create();
+ private boolean unwrapAdjacencyList = false;
+
+
+ private Builder() {}
+
+ /**
+ * Number of mutations to perform before a commit is executed when using
+ * {@link GraphSONPartialReader#readGraph(InputStream, Graph)}.
+ */
+ public Builder batchSize(final long batchSize) {
+ this.batchSize = batchSize;
+ return this;
+ }
+
+ /**
+ * Override all of the {@link GraphSONMapper} builder
+ * options with this mapper. If this value is set to something other than null then that value will be
+ * used to construct the writer.
+ */
+ public Builder mapper(final Mapper<ObjectMapper> mapper) {
+ this.mapper = mapper;
+ return this;
+ }
+
+ /**
+ * If the adjacency list is wrapped in a JSON object, as is done when writing a graph with
+ * {@link GraphSONWriter.Builder#wrapAdjacencyList} set to {@code true}, this setting needs to be set to
+ * {@code true} to properly read it. By default, this value is {@code false} and the adjacency list is
+ * simply read as line delimited vertices.
+ * <p/>
+ * By setting this value to {@code true}, the generated JSON is no longer "splittable" by line and thus not
+ * suitable for OLAP processing. Furthermore, reading this format of the JSON with
+ * {@link GraphSONPartialReader#readGraph(InputStream, Graph)} or
+ * {@link GraphSONPartialReader#readVertices(InputStream, Function, Function, Direction)} requires that the
+ * entire JSON object be read into memory, so it is best saved for "small" graphs.
+ */
+ public Builder unwrapAdjacencyList(final boolean unwrapAdjacencyList) {
+ this.unwrapAdjacencyList = unwrapAdjacencyList;
+ return this;
+ }
+
+ public GraphSONPartialReader create() {
+ return new GraphSONPartialReader(this);
+ }
+ }
+}
diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java
index c0f8ee9..9fc18eb 100644
--- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java
+++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java
@@ -19,28 +19,31 @@
*/
package org.onap.aai.dbgen.schemamod;
-import java.util.Properties;
-
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.onap.aai.config.PropertyPasswordConfiguration;
import org.onap.aai.dbmap.DBConnectionType;
+import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.Loader;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.introspection.ModelType;
-import org.onap.aai.setup.SchemaVersions;
-import org.onap.aai.setup.SchemaVersion;
import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.logging.LoggingContext;
import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.QueryStyle;
import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.ExceptionTranslator;
import org.onap.aai.util.UniquePropertyCheck;
import org.slf4j.MDC;
-
-import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+import java.util.Properties;
+
public class SchemaMod {
private final LoaderFactory loaderFactory;
@@ -159,13 +162,25 @@ public class SchemaMod {
logger.info(msg);
}
- public static void main(String[] args) {
-
- AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
- "org.onap.aai.config",
- "org.onap.aai.setup"
- );
+ public static void main(String[] args) throws AAIException {
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
+ PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
+ initializer.initialize(ctx);
+ try {
+ ctx.scan(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+ ctx.refresh();
+ } catch (Exception e) {
+ AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
+ System.out.println("Problems running SchemaMod "+aai.getMessage());
+ LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
+ throw aai;
+ }
LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
SchemaMod schemaMod = new SchemaMod(loaderFactory, schemaVersions);