diff options
author | ankitbhatt <ankit.bhatt@amdocs.com> | 2018-11-27 18:11:54 +0530 |
---|---|---|
committer | ankitbhatt <ankit.bhatt@amdocs.com> | 2018-12-05 14:47:33 +0530 |
commit | 5777029e3c3880dd9ec39d61a6bfce6497e12e91 (patch) | |
tree | e421abf1ed174b8abeab21aa3bb03f734dfd73a4 /src/main/java | |
parent | 4798a89a09c85adc02da8c60b6678340ba3779a3 (diff) |
Rebuild Edges Functionality
Enhanced EdgeMigrator Class with
new method rebuildEdges,
Added utility to support rebuild All Edges.
Introduced new flag --rebuildEdges in MigrationControllerInternal
Change-Id: Ie60f1a4618451772e4df82378d0a9e7e178c76f5
Issue-ID: AAI-1806
Signed-off-by: ankitbhatt <ankit.bhatt@amdocs.com>
Diffstat (limited to 'src/main/java')
3 files changed, 759 insertions, 532 deletions
diff --git a/src/main/java/org/onap/aai/migration/EdgeMigrator.java b/src/main/java/org/onap/aai/migration/EdgeMigrator.java index 99b4896..bb8acfc 100644 --- a/src/main/java/org/onap/aai/migration/EdgeMigrator.java +++ b/src/main/java/org/onap/aai/migration/EdgeMigrator.java @@ -17,26 +17,38 @@ * limitations under the License. * ============LICENSE_END========================================================= */ -package org.onap.aai.migration; -import java.util.List; +package org.onap.aai.migration; import com.google.common.collect.Multimap; -import org.javatuples.Pair; -import org.apache.tinkerpop.gremlin.structure.Edge; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.javatuples.Pair; import org.onap.aai.db.props.AAIProperties; import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.EdgeRule; import org.onap.aai.edges.EdgeRuleQuery; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.db.exceptions.EdgeMultiplicityException; import org.onap.aai.serialization.engines.TransactionalGraphEngine; -import org.onap.aai.edges.EdgeRule; import org.onap.aai.setup.SchemaVersions; + /** * A migration template for migrating all edge properties between "from" and "to" node from the DbedgeRules.json * @@ -45,101 +57,206 @@ import org.onap.aai.setup.SchemaVersions; @MigrationDangerRating(1) public abstract class EdgeMigrator extends Migrator { - private boolean success = true; - - public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { - super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); - } - - public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions, List<Pair<String, String>> nodePairList) { - super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); - } - - - /** - * Do not override this method as an inheritor of this class - */ - @Override - public void run() { - - executeModifyOperation(); - - } - - /** - * This is where inheritors should add their logic - */ - protected void executeModifyOperation() { - - changeEdgeProperties(); - - } - - protected void changeEdgeLabels() { - //TODO: when json file has edge label as well as edge property changes - } - - - - protected void changeEdgeProperties() { - try { - List<Pair<String, String>> nodePairList = this.getAffectedNodePairTypes(); - for (Pair<String, String> nodePair : nodePairList) { - - String NODE_A = nodePair.getValue0(); - String NODE_B = nodePair.getValue1(); - Multimap<String, EdgeRule> result = edgeIngestor.getRules(new EdgeRuleQuery.Builder(NODE_A, NODE_B).build()); - - GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V(); - /* - * Find Out-Edges from Node A to Node B and change them - * Also Find Out-Edges from Node B to Node A and change them - */ - g.union(__.has(AAIProperties.NODE_TYPE, NODE_A).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_B)), - __.has(AAIProperties.NODE_TYPE, NODE_B).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_A))) - .sideEffect(t -> { - Edge e = t.get(); - try { - Vertex out = e.outVertex(); - Vertex in = e.inVertex(); - if (out == null || in == null) { - logger.error( - e.id() + " invalid because one vertex was null: out=" + out + " in=" + in); - } else { - if (result.containsKey(e.label())) { - EdgeRule rule = result.get(e.label()).iterator().next(); - e.properties().forEachRemaining(prop -> prop.remove()); - edgeSerializer.addProperties(e, rule); - } else { - logger.info("found vertices connected by unkwown label: out=" + out + " label=" - + e.label() + " in=" + in); - } - } - } catch (Exception e1) { - throw new RuntimeException(e1); - } - }).iterate(); - } - - } catch (Exception e) { - logger.error("error encountered", e); - success = false; - } - } + protected int processed = 0; + protected int skipped = 0; + protected Map<String, Integer> edgeMultiplicityExceptionCtr = new HashMap<>(); + protected List<String> edgeMissingParentProperty = new ArrayList<>(); + + private boolean success = true; + + public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, + EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + /** + * Do not override this method as an inheritor of this class. + */ + @Override + public void run() { + + executeModifyOperation(); + + } + + /** + * This is where inheritors should add their logic. + */ + protected void executeModifyOperation() { + + changeEdgeProperties(); + + } + + protected void changeEdgeLabels() { + //TODO: when json file has edge label as well as edge property changes + } + + + + protected void changeEdgeProperties() { + try { + List<Pair<String, String>> nodePairList = this.getAffectedNodePairTypes(); + for (Pair<String, String> nodePair : nodePairList) { + + String NODE_A = nodePair.getValue0(); + String NODE_B = nodePair.getValue1(); + Multimap<String, EdgeRule> result = edgeIngestor.getRules(new EdgeRuleQuery.Builder(NODE_A, NODE_B).build()); + + GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V(); + /* + * Find Out-Edges from Node A to Node B and change them + * Also Find Out-Edges from Node B to Node A and change them + */ + g.union(__.has(AAIProperties.NODE_TYPE, NODE_A).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_B)), + __.has(AAIProperties.NODE_TYPE, NODE_B).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_A))) + .sideEffect(t -> { + Edge e = t.get(); + try { + Vertex out = e.outVertex(); + Vertex in = e.inVertex(); + if (out == null || in == null) { + logger.error( + e.id() + " invalid because one vertex was null: out=" + out + " in=" + in); + } else { + if (result.containsKey(e.label())) { + EdgeRule rule = result.get(e.label()).iterator().next(); + e.properties().forEachRemaining(prop -> prop.remove()); + edgeSerializer.addProperties(e, rule); + } else { + logger.info("found vertices connected by unkwown label: out=" + out + " label=" + + e.label() + " in=" + in); + } + } + } catch (Exception e1) { + throw new RuntimeException(e1); + } + }).iterate(); + } + + } catch (Exception e) { + logger.error("error encountered", e); + success = false; + } + } - @Override - public Status getStatus() { - if (success) { - return Status.SUCCESS; - } else { - return Status.FAILURE; - } - } - - /** - * List of node pairs("from" and "to"), you would like EdgeMigrator to migrate from json files - * @return - */ - public abstract List<Pair<String, String>> getAffectedNodePairTypes() ; - + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + /** + * List of node pairs("from" and "to"), you would like EdgeMigrator to migrate from json files. + * @return + */ + public abstract List<Pair<String, String>> getAffectedNodePairTypes() ; + + /** + * Takes the set of edges, and rebuild them according to current edgeRules schema. + * @param edges takes set of edges as input. + */ + protected void rebuildEdges(Set<Edge> edges) { + Vertex out = null; + Vertex in = null; + String oldEdgeString = null; + edgeMultiplicityExceptionCtr = new HashMap<>(); + edgeMissingParentProperty = new ArrayList<>(); + GraphTraversalSource graphTraversalSource = engine.asAdmin().getTraversalSource(); + try { + for (Edge edge : edges) { + oldEdgeString = toStringForPrinting(edge, 1); + out = edge.outVertex(); + in = edge.inVertex(); + rebuildEdge(edge, graphTraversalSource); + } + } catch (Exception ex) { + logger.error(MIGRATION_ERROR + "exception occurred during migration, failing: out=" + out + " in=" + in + + "edge=" + oldEdgeString, ex); + success = false; + } + } + + private void rebuildEdge(Edge edge, GraphTraversalSource graphTraversalSource) + throws AAIException { + boolean isCousin = false; + Vertex out = edge.outVertex(); + Vertex in = edge.inVertex(); + if (out == null || in == null) { + logger.error(edge.id() + " invalid because one vertex was null: out=" + + edge.outVertex() + " in=" + edge.inVertex()); + skipped++; + return; + } + + if (edge.property("contains-other-v").isPresent()) { + isCousin = "NONE".equals(edge.property("contains-other-v").value()); + } else if (edge.property("isParent").isPresent()) { + isCousin = !(Boolean) edge.property("isParent").value(); + } else { + edgeMissingParentProperty.add(this.toStringForPrinting(edge, 1)); + } + + String inVertexNodeType = in.value(AAIProperties.NODE_TYPE); + String outVertexNodeType = out.value(AAIProperties.NODE_TYPE); + String label = null; + + try { + Collection<EdgeRule> edgeRules = edgeIngestor.getRules(new EdgeRuleQuery.Builder(inVertexNodeType, + outVertexNodeType).build()).values(); + Set<String> edgeLabels = edgeRules.stream().map(EdgeRule::getLabel).collect(Collectors.toSet()); + if (edgeLabels.size() > 1) { + label = selectLabel(edge, edgeLabels); + if (label == null) { + logger.warn("For Multiple EdgeRules between " + "out=" + outVertexNodeType + " in=" + + inVertexNodeType + ": did not find label for edge :" + edge.id()); + } + } + } catch (Exception e) { + logger.error(edge.id() + " did not migrate as no edge rule found for: out=" + outVertexNodeType + + " in=" + inVertexNodeType); + skipped++; + return; + } + + try { + edge.remove(); + if (isCousin) { + edgeSerializer.addEdgeIfPossible(graphTraversalSource, in, out, label); + } else { + edgeSerializer.addTreeEdge(graphTraversalSource, out, in); + } + processed++; + } catch (EdgeMultiplicityException edgeMultiplicityException) { + logger.warn("Edge Multiplicity Exception: " + + "\nInV:\n" + this.toStringForPrinting(in, 1) + + "Edge:\n" + this.toStringForPrinting(edge, 1) + + "OutV:\n" + this.toStringForPrinting(out, 1) + ); + + final String mapKey = "OUT:" + outVertexNodeType + " " + + (isCousin ? EdgeType.COUSIN.toString() : EdgeType.TREE.toString()) + " " + + "IN:" + inVertexNodeType; + if (edgeMultiplicityExceptionCtr.containsKey(mapKey)) { + edgeMultiplicityExceptionCtr.put(mapKey, edgeMultiplicityExceptionCtr.get(mapKey) + 1); + } else { + edgeMultiplicityExceptionCtr.put(mapKey, 1); + } + } + } + + /** + * For selecting label from multiple EdgeLabels + * (where labels got changed between edgeRules' versions), + * you should override this method in inheritor class. + * @param edge Edge + * @param edgeLabels set of edgeLabels + * @return + */ + protected String selectLabel(Edge edge, Set<String> edgeLabels) { + return null; + } } diff --git a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java index 8ef0603..b113f03 100644 --- a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java +++ b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java @@ -17,21 +17,14 @@ * limitations under the License. * ============LICENSE_END========================================================= */ -package org.onap.aai.migration; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; +package org.onap.aai.migration; +import com.att.eelf.configuration.Configuration; +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.exception.ExceptionUtils; @@ -45,24 +38,32 @@ import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Loader; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.introspection.ModelType; -import org.onap.aai.serialization.db.EdgeSerializer; -import org.onap.aai.setup.SchemaVersions; -import org.onap.aai.setup.SchemaVersion; import org.onap.aai.logging.LoggingContext; import org.onap.aai.logging.LoggingContext.StatusCode; -import org.onap.aai.serialization.engines.QueryStyle; +import org.onap.aai.serialization.db.EdgeSerializer; import org.onap.aai.serialization.engines.JanusGraphDBEngine; +import org.onap.aai.serialization.engines.QueryStyle; import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConstants; import org.onap.aai.util.FormatDate; import org.reflections.Reflections; import org.slf4j.MDC; -import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + /** * Runs a series of migrations from a defined directory based on the presence of @@ -72,427 +73,437 @@ import com.beust.jcommander.Parameter; */ public class MigrationControllerInternal { - private EELFLogger logger; - private final int DANGER_ZONE = 10; - public static final String VERTEX_TYPE = "migration-list-1707"; - private final List<String> resultsSummary = new ArrayList<>(); - private final List<NotificationHelper> notifications = new ArrayList<>(); - private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots"; - - private LoaderFactory loaderFactory; - private EdgeIngestor edgeIngestor; - private EdgeSerializer edgeSerializer; - private final SchemaVersions schemaVersions; - - public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){ - this.loaderFactory = loaderFactory; - this.edgeIngestor = edgeIngestor; - this.edgeSerializer = edgeSerializer; - this.schemaVersions = schemaVersions; - } - - /** - * The main method. - * - * @param args - * the arguments - */ - public void run(String[] args) { - // Set the logging file properties to be used by EELFManager - System.setProperty("aai.service.name", MigrationController.class.getSimpleName()); - Properties props = System.getProperties(); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml"); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); - - logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName()); - MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); - - boolean loadSnapshot = false; - - CommandLineArgs cArgs = new CommandLineArgs(); - - JCommander jCommander = new JCommander(cArgs, args); - jCommander.setProgramName(MigrationController.class.getSimpleName()); - - // Set flag to load from snapshot based on the presence of snapshot and - // graph storage backend of inmemory - if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) { - try { - PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config); - if (config.getString("storage.backend").equals("inmemory")) { - loadSnapshot = true; - System.setProperty("load.snapshot.file", "true"); - System.setProperty("snapshot.location", cArgs.dataSnapshot); - } - } catch (ConfigurationException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e)); - return; - } - } - System.setProperty("realtime.db.config", cArgs.config); - logAndPrint("\n\n---------- Connecting to Graph ----------"); - AAIGraph.getInstance(); - - logAndPrint("---------- Connection Established ----------"); - SchemaVersion version = schemaVersions.getDefaultVersion(); - QueryStyle queryStyle = QueryStyle.TRAVERSAL; - ModelType introspectorFactoryType = ModelType.MOXY; - Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); - TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader); - - if (cArgs.help) { - jCommander.usage(); - engine.rollback(); - return; - } - - Reflections reflections = new Reflections("org.onap.aai.migration"); - List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections)); - //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names - if (cArgs.list) { - listMigrationWithStatus(cArgs, migratorClasses, engine); - return; - } - - logAndPrint("---------- Looking for migration scripts to be executed. ----------"); - //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names - if (!cArgs.excludeClasses.isEmpty()) { - migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses); - listMigrationWithStatus(cArgs, migratorClasses, engine); - } - List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses); - - sortList(migratorClassesToRun); - - if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + "."); - logAndPrint("---------- Done ----------"); - LoggingContext.successStatusFields(); - } - - logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts."); - logAndPrint("---------- Executing Migration Scripts ----------"); - - - if (!cArgs.skipPreMigrationSnapShot) { - takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun); - } - - for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { - String name = migratorClass.getSimpleName(); - Migrator migrator; - if (migratorClass.isAnnotationPresent(Enabled.class)) { - - try { - engine.startTransaction(); - if (!cArgs.forced && hasAlreadyRun(name, engine)) { - logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution"); - continue; - } - migrator = migratorClass - .getConstructor( - TransactionalGraphEngine.class, - LoaderFactory.class, - EdgeIngestor.class, - EdgeSerializer.class, - SchemaVersions.class - ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions); - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e)); - LoggingContext.successStatusFields(); - engine.rollback(); - continue; - } - logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script."); - logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs."); - MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName()); - - migrator.run(); - - commitChanges(engine, migrator, cArgs); - } else { - logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled."); - } - } - MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); - for (NotificationHelper notificationHelper : notifications) { - try { - notificationHelper.triggerEvents(); - } catch (AAIException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); - logAndPrint("\tcould not event"); - logger.error("could not event", e); - LoggingContext.successStatusFields(); - } - } - logAndPrint("---------- Done ----------"); - - // Save post migration snapshot if snapshot was loaded - if (!cArgs.skipPostMigrationSnapShot) { - generateSnapshot(engine, "post"); - } - - outputResultsSummary(); - } - - /** - * This method is used to remove excluded classes from migration from the - * script command. - * - * @param excludeClasses - * : Classes to be removed from Migration - * @param migratorClasses - * : Classes to execute migration. - * @return - */ - private List<Class<? extends Migrator>> filterMigrationClasses( - List<String> excludeClasses, - List<Class<? extends Migrator>> migratorClasses) { - - List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses - .stream() - .filter(migratorClass -> !excludeClasses.contains(migratorClass - .getSimpleName())).collect(Collectors.toList()); - - return filteredMigratorClasses; - } - - private void listMigrationWithStatus(CommandLineArgs cArgs, - List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) { - sortList(migratorClasses); - engine.startTransaction(); - System.out.println("---------- List of all migrations ----------"); - migratorClasses.forEach(migratorClass -> { - boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class); - String enabled = enabledAnnotation ? "Enabled" : "Disabled"; - StringBuilder sb = new StringBuilder(); - sb.append(migratorClass.getSimpleName()); - sb.append(" in package "); - sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1)); - sb.append(" is "); - sb.append(enabled); - sb.append(" "); - sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]"); - System.out.println(sb.toString()); - }); - engine.rollback(); - System.out.println("---------- Done ----------"); - } - - private String getDbStatus(String name, TransactionalGraphEngine engine) { - if (hasAlreadyRun(name, engine)) { - return "Already executed in this env"; - } - return "Will be run on next execution if Enabled"; - } - - private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) { - return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext(); - } - private Set<Class<? extends Migrator>> findClasses(Reflections reflections) { - Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class); - /* - * TODO- Change this to make sure only classes in the specific $release are added in the runList - * Or add a annotation like exclude which folks again need to remember to add ?? - */ - - migratorClasses.remove(PropertyMigrator.class); - migratorClasses.remove(EdgeMigrator.class); - return migratorClasses; - } - - - private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) { - - /*int sum = 0; - for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { - if (migratorClass.isAnnotationPresent(Enabled.class)) { - sum += migratorClass.getAnnotation(MigrationPriority.class).value(); - } - } - - if (sum >= DANGER_ZONE) { - - logAndPrint("Entered Danger Zone. Taking snapshot."); - }*/ - - //always take snapshot for now - - generateSnapshot(engine, "pre"); - - } - - - private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs, - List<Class<? extends Migrator>> migratorClasses) { - List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>(); - - for (Class<? extends Migrator> migratorClass : migratorClasses) { - if (!cArgs.scripts.isEmpty() && !cArgs.scripts.contains(migratorClass.getSimpleName())) { - continue; - } else { - migratorClassesToRun.add(migratorClass); - } - } - return migratorClassesToRun; - } - - - private void sortList(List<Class<? extends Migrator>> migratorClasses) { - Collections.sort(migratorClasses, (m1, m2) -> { - try { - if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) { - return 1; - } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) { - return -1; - } else { - return m1.getSimpleName().compareTo(m2.getSimpleName()); - } - } catch (Exception e) { - return 0; - } - }); - } - - - private void generateSnapshot(TransactionalGraphEngine engine, String phase) { - - FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); - String dateStr= fd.getDateTime(); - String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson"; - logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName); - Graph transaction = null; - try { - - Path pathToFile = Paths.get(fileName); - if (!pathToFile.toFile().exists()) { - Files.createDirectories(pathToFile.getParent()); - } - transaction = engine.startTransaction(); - transaction.io(IoCore.graphson()).writeGraph(fileName); - engine.rollback(); - } catch (IOException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); - logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e)); - LoggingContext.successStatusFields(); - engine.rollback(); - } - - logAndPrint( phase + " migration snapshot saved to " + fileName); - } - /** - * Log and print. - * - * @param msg - * the msg - */ - protected void logAndPrint(String msg) { - System.out.println(msg); - logger.info(msg); - } - - /** - * Commit changes. - * - * @param engine - * the graph transaction - * @param migrator - * the migrator - * @param cArgs - */ - protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) { - - String simpleName = migrator.getClass().getSimpleName(); - String message; - if (migrator.getStatus().equals(Status.FAILURE)) { - message = "Migration " + simpleName + " Failed. Rolling back."; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logAndPrint("\t" + message); - LoggingContext.successStatusFields(); - migrator.rollback(); - } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) { - message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back."; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logAndPrint("\t" + message); - LoggingContext.successStatusFields(); - migrator.rollback(); - } else { - MDC.put("logFilenameAppender", simpleName + "/" + simpleName); - - if (cArgs.commit) { - if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) { - engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate(); - } - engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE) - .property(simpleName, true).iterate(); - MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); - notifications.add(migrator.getNotificationHelper()); - migrator.commit(); - message = "Migration " + simpleName + " Succeeded. Changes Committed."; - logAndPrint("\t"+ message +"\t"); - } else { - message = "--commit not specified. Not committing changes for " + simpleName + " to database."; - logAndPrint("\t" + message); - migrator.rollback(); - } - - } - - resultsSummary.add(message); - - } - - private void outputResultsSummary() { - logAndPrint("---------------------------------"); - logAndPrint("-------------Summary-------------"); - for (String result : resultsSummary) { - logAndPrint(result); - } - logAndPrint("---------------------------------"); - logAndPrint("---------------------------------"); - } + private EELFLogger logger; + private final int DANGER_ZONE = 10; + public static final String VERTEX_TYPE = "migration-list-1707"; + private final List<String> resultsSummary = new ArrayList<>(); + private final List<NotificationHelper> notifications = new ArrayList<>(); + private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots"; + + private LoaderFactory loaderFactory; + private EdgeIngestor edgeIngestor; + private EdgeSerializer edgeSerializer; + private final SchemaVersions schemaVersions; + + public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){ + this.loaderFactory = loaderFactory; + this.edgeIngestor = edgeIngestor; + this.edgeSerializer = edgeSerializer; + this.schemaVersions = schemaVersions; + } + + /** + * The main method. + * + * @param args + * the arguments + */ + public void run(String[] args) { + // Set the logging file properties to be used by EELFManager + System.setProperty("aai.service.name", MigrationController.class.getSimpleName()); + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml"); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); + + logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName()); + MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + + boolean loadSnapshot = false; + + CommandLineArgs cArgs = new CommandLineArgs(); + + JCommander jCommander = new JCommander(cArgs, args); + jCommander.setProgramName(MigrationController.class.getSimpleName()); + + // Set flag to load from snapshot based on the presence of snapshot and + // graph storage backend of inmemory + if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) { + try { + PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config); + if (config.getString("storage.backend").equals("inmemory")) { + loadSnapshot = true; + System.setProperty("load.snapshot.file", "true"); + System.setProperty("snapshot.location", cArgs.dataSnapshot); + } + } catch (ConfigurationException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e)); + return; + } + } + System.setProperty("realtime.db.config", cArgs.config); + logAndPrint("\n\n---------- Connecting to Graph ----------"); + AAIGraph.getInstance(); + + logAndPrint("---------- Connection Established ----------"); + SchemaVersion version = schemaVersions.getDefaultVersion(); + QueryStyle queryStyle = QueryStyle.TRAVERSAL; + ModelType introspectorFactoryType = ModelType.MOXY; + Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); + TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader); + + if (cArgs.help) { + jCommander.usage(); + engine.rollback(); + return; + } + + Reflections reflections = new Reflections("org.onap.aai.migration"); + List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections)); + //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names + if (cArgs.list) { + listMigrationWithStatus(cArgs, migratorClasses, engine); + return; + } + + logAndPrint("---------- Looking for migration scripts to be executed. ----------"); + //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names + if (!cArgs.excludeClasses.isEmpty()) { + migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses); + listMigrationWithStatus(cArgs, migratorClasses, engine); + } + List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses); + + sortList(migratorClassesToRun); + + if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); + logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + "."); + logAndPrint("---------- Done ----------"); + LoggingContext.successStatusFields(); + } + + logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts."); + logAndPrint("---------- Executing Migration Scripts ----------"); + + + if (!cArgs.skipPreMigrationSnapShot) { + takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun); + } + + for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { + String name = migratorClass.getSimpleName(); + Migrator migrator; + if (cArgs.runDisabled.contains(name) || migratorClass.isAnnotationPresent(Enabled.class)) {//Check either of enabled annotation or runDisabled flag + + try { + engine.startTransaction(); + if (!cArgs.forced && hasAlreadyRun(name, engine)) { + logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution"); + continue; + } + migrator = migratorClass + .getConstructor( + TransactionalGraphEngine.class, + LoaderFactory.class, + EdgeIngestor.class, + EdgeSerializer.class, + SchemaVersions.class + ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e)); + LoggingContext.successStatusFields(); + engine.rollback(); + continue; + } + logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script."); + logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs."); + MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName()); + + migrator.run(); + + commitChanges(engine, migrator, cArgs); + } else { + logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled."); + } + } + MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + for (NotificationHelper notificationHelper : notifications) { + try { + notificationHelper.triggerEvents(); + } catch (AAIException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); + logAndPrint("\tcould not event"); + logger.error("could not event", e); + LoggingContext.successStatusFields(); + } + } + logAndPrint("---------- Done ----------"); + + // Save post migration snapshot if snapshot was loaded + if (!cArgs.skipPostMigrationSnapShot) { + generateSnapshot(engine, "post"); + } + + outputResultsSummary(); + } + + /** + * This method is used to remove excluded classes from migration from the + * script command. + * + * @param excludeClasses + * : Classes to be removed from Migration + * @param migratorClasses + * : Classes to execute migration. + * @return + */ + private List<Class<? extends Migrator>> filterMigrationClasses( + List<String> excludeClasses, + List<Class<? extends Migrator>> migratorClasses) { + + List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses + .stream() + .filter(migratorClass -> !excludeClasses.contains(migratorClass + .getSimpleName())).collect(Collectors.toList()); + + return filteredMigratorClasses; + } + + private void listMigrationWithStatus(CommandLineArgs cArgs, + List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) { + sortList(migratorClasses); + engine.startTransaction(); + System.out.println("---------- List of all migrations ----------"); + migratorClasses.forEach(migratorClass -> { + boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class); + String enabled = enabledAnnotation ? "Enabled" : "Disabled"; + StringBuilder sb = new StringBuilder(); + sb.append(migratorClass.getSimpleName()); + sb.append(" in package "); + sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1)); + sb.append(" is "); + sb.append(enabled); + sb.append(" "); + sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]"); + System.out.println(sb.toString()); + }); + engine.rollback(); + System.out.println("---------- Done ----------"); + } + + private String getDbStatus(String name, TransactionalGraphEngine engine) { + if (hasAlreadyRun(name, engine)) { + return "Already executed in this env"; + } + return "Will be run on next execution if Enabled"; + } + + private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) { + return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext(); + } + private Set<Class<? extends Migrator>> findClasses(Reflections reflections) { + Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class); + /* + * TODO- Change this to make sure only classes in the specific $release are added in the runList + * Or add a annotation like exclude which folks again need to remember to add ?? + */ + + migratorClasses.remove(PropertyMigrator.class); + migratorClasses.remove(EdgeMigrator.class); + return migratorClasses; + } + + + private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) { + + /*int sum = 0; + for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { + if (migratorClass.isAnnotationPresent(Enabled.class)) { + sum += migratorClass.getAnnotation(MigrationPriority.class).value(); + } + } + + if (sum >= DANGER_ZONE) { + + logAndPrint("Entered Danger Zone. Taking snapshot."); + }*/ + + //always take snapshot for now + + generateSnapshot(engine, "pre"); + + } + + + private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs, + List<Class<? extends Migrator>> migratorClasses) { + List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>(); + if (cArgs.scripts.isEmpty() && cArgs.runDisabled.isEmpty()) { + return migratorClasses; + + } + for (Class<? extends Migrator> migratorClass : migratorClasses) { + if (migratorExplicitlySpecified(cArgs, migratorClass.getSimpleName()) || migratorToRunWhenDisabled(cArgs, migratorClass.getSimpleName())) { + migratorClassesToRun.add(migratorClass); + } + } + return migratorClassesToRun; + } + private boolean migratorExplicitlySpecified(CommandLineArgs cArgs, String migratorName){ + return !cArgs.scripts.isEmpty() && cArgs.scripts.contains(migratorName); + } + private boolean migratorToRunWhenDisabled(CommandLineArgs cArgs, String migratorName){ + return !cArgs.runDisabled.isEmpty() && cArgs.runDisabled.contains(migratorName); + } + + private void sortList(List<Class<? extends Migrator>> migratorClasses) { + Collections.sort(migratorClasses, (m1, m2) -> { + try { + if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) { + return 1; + } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) { + return -1; + } else { + return m1.getSimpleName().compareTo(m2.getSimpleName()); + } + } catch (Exception e) { + return 0; + } + }); + } + + + private void generateSnapshot(TransactionalGraphEngine engine, String phase) { + + FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); + String dateStr= fd.getDateTime(); + String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson"; + logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName); + Graph transaction = null; + try { + + Path pathToFile = Paths.get(fileName); + if (!pathToFile.toFile().exists()) { + Files.createDirectories(pathToFile.getParent()); + } + transaction = engine.startTransaction(); + transaction.io(IoCore.graphson()).writeGraph(fileName); + engine.rollback(); + } catch (IOException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); + logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e)); + LoggingContext.successStatusFields(); + engine.rollback(); + } + + logAndPrint( phase + " migration snapshot saved to " + fileName); + } + /** + * Log and print. + * + * @param msg + * the msg + */ + protected void logAndPrint(String msg) { + System.out.println(msg); + logger.info(msg); + } + + /** + * Commit changes. + * + * @param engine + * the graph transaction + * @param migrator + * the migrator + * @param cArgs + */ + protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) { + + String simpleName = migrator.getClass().getSimpleName(); + String message; + if (migrator.getStatus().equals(Status.FAILURE)) { + message = "Migration " + simpleName + " Failed. Rolling back."; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("\t" + message); + LoggingContext.successStatusFields(); + migrator.rollback(); + } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) { + message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back."; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("\t" + message); + LoggingContext.successStatusFields(); + migrator.rollback(); + } else { + MDC.put("logFilenameAppender", simpleName + "/" + simpleName); + + if (cArgs.commit) { + if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) { + engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate(); + } + engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE) + .property(simpleName, true).iterate(); + MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + notifications.add(migrator.getNotificationHelper()); + migrator.commit(); + message = "Migration " + simpleName + " Succeeded. Changes Committed."; + logAndPrint("\t"+ message +"\t"); + } else { + message = "--commit not specified. Not committing changes for " + simpleName + " to database."; + logAndPrint("\t" + message); + migrator.rollback(); + } + + } + + resultsSummary.add(message); + + } + + private void outputResultsSummary() { + logAndPrint("---------------------------------"); + logAndPrint("-------------Summary-------------"); + for (String result : resultsSummary) { + logAndPrint(result); + } + logAndPrint("---------------------------------"); + logAndPrint("---------------------------------"); + } } class CommandLineArgs { - @Parameter(names = "--help", help = true) - public boolean help; + @Parameter(names = "--help", help = true) + public boolean help; + + @Parameter(names = "-c", description = "location of configuration file") + public String config; + + @Parameter(names = "-m", description = "names of migration scripts") + public List<String> scripts = new ArrayList<>(); - @Parameter(names = "-c", description = "location of configuration file") - public String config; + @Parameter(names = "-l", description = "list the status of migrations") + public boolean list = false; - @Parameter(names = "-m", description = "names of migration scripts") - public List<String> scripts = new ArrayList<>(); + @Parameter(names = "-d", description = "location of data snapshot", hidden = true) + public String dataSnapshot; - @Parameter(names = "-l", description = "list the status of migrations") - public boolean list = false; + @Parameter(names = "-f", description = "force migrations to be rerun") + public boolean forced = false; - @Parameter(names = "-d", description = "location of data snapshot", hidden = true) - public String dataSnapshot; + @Parameter(names = "--commit", description = "commit changes to graph") + public boolean commit = false; - @Parameter(names = "-f", description = "force migrations to be rerun") - public boolean forced = false; + @Parameter(names = "-e", description = "exclude list of migrator classes") + public List<String> excludeClasses = new ArrayList<>(); - @Parameter(names = "--commit", description = "commit changes to graph") - public boolean commit = false; + @Parameter(names = "--skipPreMigrationSnapShot", description = "skips taking the PRE migration snapshot") + public boolean skipPreMigrationSnapShot = false; - @Parameter(names = "-e", description = "exclude list of migrator classes") - public List<String> excludeClasses = new ArrayList<>(); + @Parameter(names = "--skipPostMigrationSnapShot", description = "skips taking the POST migration snapshot") + public boolean skipPostMigrationSnapShot = false; - @Parameter(names = "--skipPreMigrationSnapShot", description = "skips taking the PRE migration snapshot") - public boolean skipPreMigrationSnapShot = false; + @Parameter(names = "--runDisabled", description = "List of migrators which are to be run even when disabled") + public List<String> runDisabled = new ArrayList<>(); - @Parameter(names = "--skipPostMigrationSnapShot", description = "skips taking the POST migration snapshot") - public boolean skipPostMigrationSnapShot = false; } diff --git a/src/main/java/org/onap/aai/migration/RebuildAllEdges.java b/src/main/java/org/onap/aai/migration/RebuildAllEdges.java new file mode 100644 index 0000000..73f5678 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/RebuildAllEdges.java @@ -0,0 +1,99 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.migration; + +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.javatuples.Pair; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +@MigrationPriority(-1) +public class RebuildAllEdges extends EdgeMigrator { + + private static final Map<String, String> LABEL_MAP = getLabelMap(); + + private static Map<String, String> getLabelMap() { + Map<String, String> labelMap = new HashMap<>(); + labelMap.put("isMemberOf", "org.onap.relationships.inventory.MemberOf"); + labelMap.put("isA", "org.onap.relationships.inventory.IsA"); + labelMap.put("has", "org.onap.relationships.inventory.Uses"); + labelMap.put("usesLogicalLink", "tosca.relationships.network.LinksTo"); + labelMap.put("sourceLInterface", "org.onap.relationships.inventory.Source"); + labelMap.put("targetLInterface", "org.onap.relationships.inventory.Destination"); + return labelMap; + } + + public RebuildAllEdges(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, + EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + protected void executeModifyOperation() { + Instant started = Instant.now(); + logger.info("Started at: " + started); + GraphTraversalSource graphTraversalSource = engine.asAdmin().getTraversalSource(); + Set<Edge> edges = graphTraversalSource.E().toSet(); + rebuildEdges(edges); + Instant completed = Instant.now(); + logger.info("Completed at: " + completed + ". Total time taken in ms : " + + (completed.toEpochMilli() - started.toEpochMilli())); + logger.info(MIGRATION_SUMMARY_COUNT + " Total Edges : " + edges.size() + " . Processed count " + processed + + " . Skipped count: " + skipped + "."); + logger.info(MIGRATION_SUMMARY_COUNT + "Edge Missing Parent Property Count: " + + edgeMissingParentProperty.size()); + logger.info(MIGRATION_ERROR + "Edge Multiplicity Exception Count : " + + edgeMultiplicityExceptionCtr.values().stream().mapToInt(Number::intValue).sum()); + logger.info(MIGRATION_ERROR + "Edge Multiplicity Exception Breakdown : " + edgeMultiplicityExceptionCtr); + } + + @Override + protected String selectLabel(Edge edge, Set<String> edgeLabels) { + return ( edgeLabels.contains(LABEL_MAP.get(edge.label())) ) ? LABEL_MAP.get(edge.label()) : null; + } + + @Override + public List<Pair<String, String>> getAffectedNodePairTypes() { + return Collections.emptyList(); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.empty(); + } + + @Override + public String getMigrationName() { + return "RebuildAllEdges"; + } +} |