diff options
Diffstat (limited to 'src/main/java/org/onap/aai/migration')
32 files changed, 4697 insertions, 0 deletions
diff --git a/src/main/java/org/onap/aai/migration/EdgeMigrator.java b/src/main/java/org/onap/aai/migration/EdgeMigrator.java new file mode 100644 index 0000000..99b4896 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/EdgeMigrator.java @@ -0,0 +1,145 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.util.List; + +import com.google.common.collect.Multimap; +import org.javatuples.Pair; + +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.EdgeRuleQuery; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.edges.EdgeRule; +import org.onap.aai.setup.SchemaVersions; + +/** + * A migration template for migrating all edge properties between "from" and "to" node from the DbedgeRules.json + * + */ +@MigrationPriority(0) +@MigrationDangerRating(1) +public abstract class EdgeMigrator extends Migrator { + + private boolean success = true; + + public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions, List<Pair<String, String>> nodePairList) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + + /** + * Do not override this method as an inheritor of this class + */ + @Override + public void run() { + + executeModifyOperation(); + + } + + /** + * This is where inheritors should add their logic + */ + protected void executeModifyOperation() { + + changeEdgeProperties(); + + } + + protected void changeEdgeLabels() { + //TODO: when json file has edge label as well as edge property changes + } + + + + protected void changeEdgeProperties() { + try { + List<Pair<String, String>> nodePairList = this.getAffectedNodePairTypes(); + for (Pair<String, String> nodePair : nodePairList) { + + String NODE_A = nodePair.getValue0(); + String NODE_B = nodePair.getValue1(); + Multimap<String, EdgeRule> result = edgeIngestor.getRules(new EdgeRuleQuery.Builder(NODE_A, NODE_B).build()); + + GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V(); + /* + * Find Out-Edges from Node A to Node B and change them + * Also Find Out-Edges from Node B to Node A and change them + */ + g.union(__.has(AAIProperties.NODE_TYPE, NODE_A).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_B)), + __.has(AAIProperties.NODE_TYPE, NODE_B).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_A))) + .sideEffect(t -> { + Edge e = t.get(); + try { + Vertex out = e.outVertex(); + Vertex in = e.inVertex(); + if (out == null || in == null) { + logger.error( + e.id() + " invalid because one vertex was null: out=" + out + " in=" + in); + } else { + if (result.containsKey(e.label())) { + EdgeRule rule = result.get(e.label()).iterator().next(); + e.properties().forEachRemaining(prop -> prop.remove()); + edgeSerializer.addProperties(e, rule); + } else { + logger.info("found vertices connected by unkwown label: out=" + out + " label=" + + e.label() + " in=" + in); + } + } + } catch (Exception e1) { + throw new RuntimeException(e1); + } + }).iterate(); + } + + } catch (Exception e) { + logger.error("error encountered", e); + success = false; + } + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + /** + * List of node pairs("from" and "to"), you would like EdgeMigrator to migrate from json files + * @return + */ + public abstract List<Pair<String, String>> getAffectedNodePairTypes() ; + +} diff --git a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java new file mode 100644 index 0000000..616ff02 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java @@ -0,0 +1,288 @@ +/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.javatuples.Pair;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+/**
+ * A migration template for "swinging" edges that terminate on an old-node to a new target node.
+ * That is, given an oldNode and a newNode we will swing edges that terminate on the
+ * oldNode and terminate them on the newNode (actually we drop the old edges and add new ones).
+ *
+ *
+ * We allow the passing of some parameters to restrict what edges get swung over:
+ * > otherEndNodeTypeRestriction: only swing edges that terminate on the oldNode if the
+ * node at the other end of the edge is of this nodeType.
+ * > edgeLabelRestriction: Only swing edges that have this edgeLabel
+ * > edgeDirectionRestriction: Only swing edges that go this direction (from the oldNode)
+ * this is a required parameter. valid values are: BOTH, IN, OUT
+ *
+ */
+@MigrationPriority(0)
+@MigrationDangerRating(1)
+public abstract class EdgeSwingMigrator extends Migrator {
+
+ private boolean success = true;
+ private String nodeTypeRestriction = null;
+ private String edgeLabelRestriction = null;
+ private String edgeDirRestriction = null;
+ private List<Pair<Vertex, Vertex>> nodePairList;
+
+
+ public EdgeSwingMigrator(TransactionalGraphEngine engine , LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+
+ /**
+ * Do not override this method as an inheritor of this class
+ */
+ @Override
+ public void run() {
+ executeModifyOperation();
+ cleanupAsAppropriate(this.nodePairList);
+ }
+
+ /**
+ * This is where inheritors should add their logic
+ */
+ protected void executeModifyOperation() {
+
+ try {
+ this.nodeTypeRestriction = this.getNodeTypeRestriction();
+ this.edgeLabelRestriction = this.getEdgeLabelRestriction();
+ this.edgeDirRestriction = this.getEdgeDirRestriction();
+ nodePairList = this.getAffectedNodePairs();
+ for (Pair<Vertex, Vertex> nodePair : nodePairList) {
+ Vertex fromNode = nodePair.getValue0();
+ Vertex toNode = nodePair.getValue1();
+ this.swingEdges(fromNode, toNode,
+ this.nodeTypeRestriction,this.edgeLabelRestriction,this.edgeDirRestriction);
+ }
+ } catch (Exception e) {
+ logger.error("error encountered", e);
+ success = false;
+ }
+ }
+
+
+ protected void swingEdges(Vertex oldNode, Vertex newNode, String nodeTypeRestr, String edgeLabelRestr, String edgeDirRestr) {
+ try {
+ // If the old and new Vertices aren't populated, throw an exception
+ if( oldNode == null ){
+ logger.info ( "null oldNode passed to swingEdges() ");
+ success = false;
+ return;
+ }
+ else if( newNode == null ){
+ logger.info ( "null newNode passed to swingEdges() ");
+ success = false;
+ return;
+ }
+ else if( edgeDirRestr == null ||
+ (!edgeDirRestr.equals("BOTH")
+ && !edgeDirRestr.equals("IN")
+ && !edgeDirRestr.equals("OUT") )
+ ){
+ logger.info ( "invalid direction passed to swingEdges(). valid values are BOTH/IN/OUT ");
+ success = false;
+ return;
+ }
+ else if( edgeLabelRestr != null
+ && (edgeLabelRestr.trim().equals("none") || edgeLabelRestr.trim().equals("")) ){
+ edgeLabelRestr = null;
+ }
+ else if( nodeTypeRestr == null || nodeTypeRestr.trim().equals("") ){
+ nodeTypeRestr = "none";
+ }
+
+ String oldNodeType = oldNode.value(AAIProperties.NODE_TYPE);
+ String oldUri = oldNode.<String> property("aai-uri").isPresent() ? oldNode.<String> property("aai-uri").value() : "URI Not present";
+
+ String newNodeType = newNode.value(AAIProperties.NODE_TYPE);
+ String newUri = newNode.<String> property("aai-uri").isPresent() ? newNode.<String> property("aai-uri").value() : "URI Not present";
+
+ // If the nodeTypes don't match, throw an error
+ if( !oldNodeType.equals(newNodeType) ){
+ logger.info ( "Can not swing edge from a [" + oldNodeType + "] node to a [" +
+ newNodeType + "] node. ");
+ success = false;
+ return;
+ }
+
+ // Find and migrate any applicable OUT edges.
+ if( edgeDirRestr.equals("BOTH") || edgeDirRestr.equals("OUT") ){
+ Iterator <Edge> edgeOutIter = null;
+ if( edgeLabelRestr == null ) {
+ edgeOutIter = oldNode.edges(Direction.OUT);
+ }
+ else {
+ edgeOutIter = oldNode.edges(Direction.OUT, edgeLabelRestr);
+ }
+
+ while( edgeOutIter.hasNext() ){
+ Edge oldOutE = edgeOutIter.next();
+ String eLabel = oldOutE.label();
+ Vertex otherSideNode4ThisEdge = oldOutE.inVertex();
+ String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
+ if( nodeTypeRestr.equals("none") || nodeTypeRestr.toLowerCase().equals(otherSideNodeType) ){
+ Iterator <Property<Object>> propsIter = oldOutE.properties();
+ HashMap<String, String> propMap = new HashMap<String,String>();
+ while( propsIter.hasNext() ){
+ Property <Object> ep = propsIter.next();
+ propMap.put(ep.key(), ep.value().toString());
+ }
+
+ String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present";
+ logger.info ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is ["
+ + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType
+ + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. ");
+ // remove the old edge
+ oldOutE.remove();
+
+ // add the new edge with properties that match the edge that was deleted. We don't want to
+ // change any edge properties - just swinging one end of the edge to a new node.
+ // NOTE - addEdge adds an OUT edge to the vertex passed as a parameter, so we are
+ // adding from the newNode side.
+ Edge newOutE = newNode.addEdge(eLabel, otherSideNode4ThisEdge);
+
+ Iterator it = propMap.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry pair = (Map.Entry)it.next();
+ newOutE.property(pair.getKey().toString(), pair.getValue().toString() );
+ }
+
+ }
+ }
+ }
+
+ // Find and migrate any applicable IN edges.
+ if( edgeDirRestr.equals("BOTH") || edgeDirRestr.equals("IN") ){
+ Iterator <Edge> edgeInIter = null;
+ if( edgeLabelRestr == null ) {
+ edgeInIter = oldNode.edges(Direction.IN);
+ }
+ else {
+ edgeInIter = oldNode.edges(Direction.IN, edgeLabelRestr);
+ }
+
+ while( edgeInIter.hasNext() ){
+ Edge oldInE = edgeInIter.next();
+ String eLabel = oldInE.label();
+ Vertex otherSideNode4ThisEdge = oldInE.outVertex();
+ String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
+ if( nodeTypeRestr.equals("none") || nodeTypeRestr.toLowerCase().equals(otherSideNodeType) ){
+ Iterator <Property<Object>> propsIter = oldInE.properties();
+ HashMap<String, String> propMap = new HashMap<String,String>();
+ while( propsIter.hasNext() ){
+ Property <Object> ep = propsIter.next();
+ propMap.put(ep.key(), ep.value().toString());
+ }
+
+ String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present";
+ logger.info ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is ["
+ + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType
+ + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. ");
+
+ // remove the old edge
+ oldInE.remove();
+
+ // add the new edge with properties that match the edge that was deleted. We don't want to
+ // change any edge properties - just swinging one end of the edge to a new node.
+ // NOTE - addEdge adds an OUT edge to the vertex passed as a parameter, so we are
+ // adding from the node on the other-end of the original edge so we'll get
+ // an IN-edge to the newNode.
+ Edge newInE = otherSideNode4ThisEdge.addEdge(eLabel, newNode);
+
+ Iterator it = propMap.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry pair = (Map.Entry)it.next();
+ newInE.property(pair.getKey().toString(), pair.getValue().toString() );
+ }
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ logger.error("error encountered", e);
+ success = false;
+ }
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+
+ /**
+ * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate from json files
+ * @return
+ */
+ public abstract List<Pair<Vertex, Vertex>> getAffectedNodePairs() ;
+
+
+ /**
+ * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use
+ * @return
+ */
+ public abstract String getNodeTypeRestriction() ;
+
+
+ /**
+ * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use
+ * @return
+ */
+ public abstract String getEdgeLabelRestriction() ;
+
+ /**
+ * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use
+ * @return
+ */
+ public abstract String getEdgeDirRestriction() ;
+
+
+
+ /**
+ * Cleanup (remove) the nodes that edges were moved off of if appropriate
+ * @return
+ */
+ public abstract void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL);
+
+}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/Enabled.java b/src/main/java/org/onap/aai/migration/Enabled.java new file mode 100644 index 0000000..1b7bba3 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/Enabled.java @@ -0,0 +1,35 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * Used to enable a migration to be picked up by the {@link org.onap.aai.migration.MigrationControllerInternal MigrationController} + */ +@Target(ElementType.TYPE) +@Retention(value = RetentionPolicy.RUNTIME) +public @interface Enabled { + +} diff --git a/src/main/java/org/onap/aai/migration/EventAction.java b/src/main/java/org/onap/aai/migration/EventAction.java new file mode 100644 index 0000000..830685b --- /dev/null +++ b/src/main/java/org/onap/aai/migration/EventAction.java @@ -0,0 +1,29 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +/** + * Used to describe the type of DMaaP event you would like to create + */ +public enum EventAction { + CREATE, + UPDATE, + DELETE +} diff --git a/src/main/java/org/onap/aai/migration/MigrationController.java b/src/main/java/org/onap/aai/migration/MigrationController.java new file mode 100644 index 0000000..0e65745 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/MigrationController.java @@ -0,0 +1,78 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.util.UUID; + +import org.onap.aai.dbmap.AAIGraph; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LoggingContext.StatusCode; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConstants; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +/** + * Wrapper class to allow {@link org.onap.aai.migration.MigrationControllerInternal MigrationControllerInternal} + * to be run from a shell script + */ +public class MigrationController { + + /** + * The main method. + * + * @param args + * the arguments + */ + public static void main(String[] args) { + + LoggingContext.init(); + LoggingContext.partnerName("Migration"); + LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS); + LoggingContext.component("MigrationController"); + LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS); + LoggingContext.targetServiceName("main"); + LoggingContext.requestId(UUID.randomUUID().toString()); + LoggingContext.statusCode(StatusCode.COMPLETE); + LoggingContext.responseCode(LoggingContext.SUCCESS); + + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext( + "org.onap.aai.config", + "org.onap.aai.setup" + ); + + LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); + EdgeIngestor edgeIngestor = ctx.getBean(EdgeIngestor.class); + EdgeSerializer edgeSerializer = ctx.getBean(EdgeSerializer.class); + SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class); + + MigrationControllerInternal internal = new MigrationControllerInternal(loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + + try { + internal.run(args); + } catch (Exception e) { + e.printStackTrace(); + } + AAIGraph.getInstance().graphShutdown(); + System.exit(0); + } +} diff --git a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java new file mode 100644 index 0000000..8ef0603 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java @@ -0,0 +1,498 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.io.IoCore; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.dbmap.AAIGraph; +import org.onap.aai.dbmap.DBConnectionType; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LoggingContext.StatusCode; +import org.onap.aai.serialization.engines.QueryStyle; +import org.onap.aai.serialization.engines.JanusGraphDBEngine; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.util.AAIConstants; +import org.onap.aai.util.FormatDate; +import org.reflections.Reflections; +import org.slf4j.MDC; + +import com.att.eelf.configuration.Configuration; +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; + +/** + * Runs a series of migrations from a defined directory based on the presence of + * the {@link org.onap.aai.migration.Enabled Enabled} annotation + * + * It will also write a record of the migrations run to the database. + */ +public class MigrationControllerInternal { + + private EELFLogger logger; + private final int DANGER_ZONE = 10; + public static final String VERTEX_TYPE = "migration-list-1707"; + private final List<String> resultsSummary = new ArrayList<>(); + private final List<NotificationHelper> notifications = new ArrayList<>(); + private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots"; + + private LoaderFactory loaderFactory; + private EdgeIngestor edgeIngestor; + private EdgeSerializer edgeSerializer; + private final SchemaVersions schemaVersions; + + public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){ + this.loaderFactory = loaderFactory; + this.edgeIngestor = edgeIngestor; + this.edgeSerializer = edgeSerializer; + this.schemaVersions = schemaVersions; + } + + /** + * The main method. + * + * @param args + * the arguments + */ + public void run(String[] args) { + // Set the logging file properties to be used by EELFManager + System.setProperty("aai.service.name", MigrationController.class.getSimpleName()); + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml"); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); + + logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName()); + MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + + boolean loadSnapshot = false; + + CommandLineArgs cArgs = new CommandLineArgs(); + + JCommander jCommander = new JCommander(cArgs, args); + jCommander.setProgramName(MigrationController.class.getSimpleName()); + + // Set flag to load from snapshot based on the presence of snapshot and + // graph storage backend of inmemory + if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) { + try { + PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config); + if (config.getString("storage.backend").equals("inmemory")) { + loadSnapshot = true; + System.setProperty("load.snapshot.file", "true"); + System.setProperty("snapshot.location", cArgs.dataSnapshot); + } + } catch (ConfigurationException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e)); + return; + } + } + System.setProperty("realtime.db.config", cArgs.config); + logAndPrint("\n\n---------- Connecting to Graph ----------"); + AAIGraph.getInstance(); + + logAndPrint("---------- Connection Established ----------"); + SchemaVersion version = schemaVersions.getDefaultVersion(); + QueryStyle queryStyle = QueryStyle.TRAVERSAL; + ModelType introspectorFactoryType = ModelType.MOXY; + Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); + TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader); + + if (cArgs.help) { + jCommander.usage(); + engine.rollback(); + return; + } + + Reflections reflections = new Reflections("org.onap.aai.migration"); + List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections)); + //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names + if (cArgs.list) { + listMigrationWithStatus(cArgs, migratorClasses, engine); + return; + } + + logAndPrint("---------- Looking for migration scripts to be executed. ----------"); + //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names + if (!cArgs.excludeClasses.isEmpty()) { + migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses); + listMigrationWithStatus(cArgs, migratorClasses, engine); + } + List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses); + + sortList(migratorClassesToRun); + + if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); + logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + "."); + logAndPrint("---------- Done ----------"); + LoggingContext.successStatusFields(); + } + + logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts."); + logAndPrint("---------- Executing Migration Scripts ----------"); + + + if (!cArgs.skipPreMigrationSnapShot) { + takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun); + } + + for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { + String name = migratorClass.getSimpleName(); + Migrator migrator; + if (migratorClass.isAnnotationPresent(Enabled.class)) { + + try { + engine.startTransaction(); + if (!cArgs.forced && hasAlreadyRun(name, engine)) { + logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution"); + continue; + } + migrator = migratorClass + .getConstructor( + TransactionalGraphEngine.class, + LoaderFactory.class, + EdgeIngestor.class, + EdgeSerializer.class, + SchemaVersions.class + ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e)); + LoggingContext.successStatusFields(); + engine.rollback(); + continue; + } + logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script."); + logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs."); + MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName()); + + migrator.run(); + + commitChanges(engine, migrator, cArgs); + } else { + logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled."); + } + } + MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + for (NotificationHelper notificationHelper : notifications) { + try { + notificationHelper.triggerEvents(); + } catch (AAIException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); + logAndPrint("\tcould not event"); + logger.error("could not event", e); + LoggingContext.successStatusFields(); + } + } + logAndPrint("---------- Done ----------"); + + // Save post migration snapshot if snapshot was loaded + if (!cArgs.skipPostMigrationSnapShot) { + generateSnapshot(engine, "post"); + } + + outputResultsSummary(); + } + + /** + * This method is used to remove excluded classes from migration from the + * script command. + * + * @param excludeClasses + * : Classes to be removed from Migration + * @param migratorClasses + * : Classes to execute migration. + * @return + */ + private List<Class<? extends Migrator>> filterMigrationClasses( + List<String> excludeClasses, + List<Class<? extends Migrator>> migratorClasses) { + + List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses + .stream() + .filter(migratorClass -> !excludeClasses.contains(migratorClass + .getSimpleName())).collect(Collectors.toList()); + + return filteredMigratorClasses; + } + + private void listMigrationWithStatus(CommandLineArgs cArgs, + List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) { + sortList(migratorClasses); + engine.startTransaction(); + System.out.println("---------- List of all migrations ----------"); + migratorClasses.forEach(migratorClass -> { + boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class); + String enabled = enabledAnnotation ? "Enabled" : "Disabled"; + StringBuilder sb = new StringBuilder(); + sb.append(migratorClass.getSimpleName()); + sb.append(" in package "); + sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1)); + sb.append(" is "); + sb.append(enabled); + sb.append(" "); + sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]"); + System.out.println(sb.toString()); + }); + engine.rollback(); + System.out.println("---------- Done ----------"); + } + + private String getDbStatus(String name, TransactionalGraphEngine engine) { + if (hasAlreadyRun(name, engine)) { + return "Already executed in this env"; + } + return "Will be run on next execution if Enabled"; + } + + private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) { + return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext(); + } + private Set<Class<? extends Migrator>> findClasses(Reflections reflections) { + Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class); + /* + * TODO- Change this to make sure only classes in the specific $release are added in the runList + * Or add a annotation like exclude which folks again need to remember to add ?? + */ + + migratorClasses.remove(PropertyMigrator.class); + migratorClasses.remove(EdgeMigrator.class); + return migratorClasses; + } + + + private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) { + + /*int sum = 0; + for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { + if (migratorClass.isAnnotationPresent(Enabled.class)) { + sum += migratorClass.getAnnotation(MigrationPriority.class).value(); + } + } + + if (sum >= DANGER_ZONE) { + + logAndPrint("Entered Danger Zone. Taking snapshot."); + }*/ + + //always take snapshot for now + + generateSnapshot(engine, "pre"); + + } + + + private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs, + List<Class<? extends Migrator>> migratorClasses) { + List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>(); + + for (Class<? extends Migrator> migratorClass : migratorClasses) { + if (!cArgs.scripts.isEmpty() && !cArgs.scripts.contains(migratorClass.getSimpleName())) { + continue; + } else { + migratorClassesToRun.add(migratorClass); + } + } + return migratorClassesToRun; + } + + + private void sortList(List<Class<? extends Migrator>> migratorClasses) { + Collections.sort(migratorClasses, (m1, m2) -> { + try { + if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) { + return 1; + } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) { + return -1; + } else { + return m1.getSimpleName().compareTo(m2.getSimpleName()); + } + } catch (Exception e) { + return 0; + } + }); + } + + + private void generateSnapshot(TransactionalGraphEngine engine, String phase) { + + FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); + String dateStr= fd.getDateTime(); + String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson"; + logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName); + Graph transaction = null; + try { + + Path pathToFile = Paths.get(fileName); + if (!pathToFile.toFile().exists()) { + Files.createDirectories(pathToFile.getParent()); + } + transaction = engine.startTransaction(); + transaction.io(IoCore.graphson()).writeGraph(fileName); + engine.rollback(); + } catch (IOException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); + logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e)); + LoggingContext.successStatusFields(); + engine.rollback(); + } + + logAndPrint( phase + " migration snapshot saved to " + fileName); + } + /** + * Log and print. + * + * @param msg + * the msg + */ + protected void logAndPrint(String msg) { + System.out.println(msg); + logger.info(msg); + } + + /** + * Commit changes. + * + * @param engine + * the graph transaction + * @param migrator + * the migrator + * @param cArgs + */ + protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) { + + String simpleName = migrator.getClass().getSimpleName(); + String message; + if (migrator.getStatus().equals(Status.FAILURE)) { + message = "Migration " + simpleName + " Failed. Rolling back."; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("\t" + message); + LoggingContext.successStatusFields(); + migrator.rollback(); + } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) { + message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back."; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("\t" + message); + LoggingContext.successStatusFields(); + migrator.rollback(); + } else { + MDC.put("logFilenameAppender", simpleName + "/" + simpleName); + + if (cArgs.commit) { + if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) { + engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate(); + } + engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE) + .property(simpleName, true).iterate(); + MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + notifications.add(migrator.getNotificationHelper()); + migrator.commit(); + message = "Migration " + simpleName + " Succeeded. Changes Committed."; + logAndPrint("\t"+ message +"\t"); + } else { + message = "--commit not specified. Not committing changes for " + simpleName + " to database."; + logAndPrint("\t" + message); + migrator.rollback(); + } + + } + + resultsSummary.add(message); + + } + + private void outputResultsSummary() { + logAndPrint("---------------------------------"); + logAndPrint("-------------Summary-------------"); + for (String result : resultsSummary) { + logAndPrint(result); + } + logAndPrint("---------------------------------"); + logAndPrint("---------------------------------"); + } + +} + +class CommandLineArgs { + + @Parameter(names = "--help", help = true) + public boolean help; + + @Parameter(names = "-c", description = "location of configuration file") + public String config; + + @Parameter(names = "-m", description = "names of migration scripts") + public List<String> scripts = new ArrayList<>(); + + @Parameter(names = "-l", description = "list the status of migrations") + public boolean list = false; + + @Parameter(names = "-d", description = "location of data snapshot", hidden = true) + public String dataSnapshot; + + @Parameter(names = "-f", description = "force migrations to be rerun") + public boolean forced = false; + + @Parameter(names = "--commit", description = "commit changes to graph") + public boolean commit = false; + + @Parameter(names = "-e", description = "exclude list of migrator classes") + public List<String> excludeClasses = new ArrayList<>(); + + @Parameter(names = "--skipPreMigrationSnapShot", description = "skips taking the PRE migration snapshot") + public boolean skipPreMigrationSnapShot = false; + + @Parameter(names = "--skipPostMigrationSnapShot", description = "skips taking the POST migration snapshot") + public boolean skipPostMigrationSnapShot = false; +} diff --git a/src/main/java/org/onap/aai/migration/MigrationDangerRating.java b/src/main/java/org/onap/aai/migration/MigrationDangerRating.java new file mode 100644 index 0000000..1d82dc3 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/MigrationDangerRating.java @@ -0,0 +1,41 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * Used to enable a migration to be picked up by the {@link com.openecomp.aai.migration.MigrationControllerInternal MigrationController} + * + * The larger the number, the more danger + * + * Range is 0-10 + */ +@Target(ElementType.TYPE) +@Retention(value = RetentionPolicy.RUNTIME) +public @interface MigrationDangerRating { + + int value(); + +} diff --git a/src/main/java/org/onap/aai/migration/MigrationPriority.java b/src/main/java/org/onap/aai/migration/MigrationPriority.java new file mode 100644 index 0000000..d9e84b8 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/MigrationPriority.java @@ -0,0 +1,41 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * Used to enable a migration to be picked up by the {@link org.onap.aai.migration.MigrationControllerInternal MigrationController} + * + * The priority of the migration. + * + * Lower number has higher priority + */ +@Target(ElementType.TYPE) +@Retention(value = RetentionPolicy.RUNTIME) +public @interface MigrationPriority { + + int value(); + +} diff --git a/src/main/java/org/onap/aai/migration/Migrator.java b/src/main/java/org/onap/aai/migration/Migrator.java new file mode 100644 index 0000000..106d5e4 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/Migrator.java @@ -0,0 +1,385 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Property; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.json.JSONException; +import org.json.JSONObject; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException; +import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; +import org.onap.aai.serialization.db.DBSerializer; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.db.exceptions.NoEdgeRuleFoundException; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.setup.SchemaVersions; + +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; + +/** + * This class defines an A&AI Migration + */ +@MigrationPriority(0) +@MigrationDangerRating(0) +public abstract class Migrator implements Runnable { + + protected EELFLogger logger = null; + + protected DBSerializer serializer = null; + protected Loader loader = null; + + protected TransactionalGraphEngine engine; + protected NotificationHelper notificationHelper; + + protected EdgeSerializer edgeSerializer; + protected EdgeIngestor edgeIngestor; + + protected LoaderFactory loaderFactory; + protected SchemaVersions schemaVersions; + + protected static final String MIGRATION_ERROR = "Migration Error: "; + protected static final String MIGRATION_SUMMARY_COUNT = "Migration Summary Count: "; + + /** + * Instantiates a new migrator. + * + * @param g the g + * @param schemaVersions + */ + public Migrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){ + this.engine = engine; + this.loaderFactory = loaderFactory; + this.edgeIngestor = edgeIngestor; + this.edgeSerializer = edgeSerializer; + this.schemaVersions = schemaVersions; + initDBSerializer(); + this.notificationHelper = new NotificationHelper(loader, serializer, loaderFactory, schemaVersions, engine, "AAI-MIGRATION", this.getMigrationName()); + logger = EELFManager.getInstance().getLogger(this.getClass().getSimpleName()); + logger.info("\tInitilization of " + this.getClass().getSimpleName() + " migration script complete."); + } + + /** + * Gets the status. + * + * @return the status + */ + public abstract Status getStatus(); + + /** + * Rollback. + */ + public void rollback() { + engine.rollback(); + } + + /** + * Commit. + */ + public void commit() { + engine.commit(); + } + + /** + * Create files containing vertices for dmaap Event Generation + * @param dmaapMsgList + */ + public void createDmaapFiles(List<String> dmaapMsgList) { + String fileName = getMigrationName() + "-" + UUID.randomUUID(); + String logDirectory = System.getProperty("AJSC_HOME") + "/logs/migration/dmaapEvents"; + + File f = new File(logDirectory); + f.mkdirs(); + + if (dmaapMsgList.size() > 0) { + try { + Files.write(Paths.get(logDirectory+"/"+fileName), (Iterable<String>)dmaapMsgList.stream()::iterator); + } catch (IOException e) { + logger.error("Unable to generate file with dmaap msgs for MigrateHUBEvcInventory", e); + } + } else { + logger.info("No dmaap msgs detected for MigrateForwardEvcCircuitId"); + } + } + + /** + * Create files containing data for dmaap delete Event Generation + * @param dmaapVertexList + */ + public void createDmaapFilesForDelete(List<Introspector> dmaapDeleteIntrospectorList) {try { + System.out.println("dmaapDeleteIntrospectorList :: " + dmaapDeleteIntrospectorList.size()); + String fileName = "DELETE-"+ getMigrationName() + "-" + UUID.randomUUID(); + String logDirectory = System.getProperty("AJSC_HOME") + "/logs/migration/dmaapEvents/"; + File f = new File(logDirectory); + f.mkdirs(); + + try{ + Files.createFile(Paths.get(logDirectory + "/" + fileName)); + }catch(Exception e) { + e.printStackTrace(); + } + + if (dmaapDeleteIntrospectorList.size() > 0) { + dmaapDeleteIntrospectorList.stream().forEach(svIntr-> { + try { + String str = svIntr.marshal(false); + String finalStr=""; + try { + finalStr=svIntr.getName() + "#@#" + svIntr.getURI() + "#@#" + str+"\n"; + Files.write(Paths.get(logDirectory + "/" + fileName),finalStr.getBytes(),StandardOpenOption.APPEND); + } catch (IOException e) { + logger.error("Unable to generate file with dmaap msgs for "+getMigrationName(), e); + } + + }catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + }); + + //Files.write(Paths.get(logDirectory+"/"+fileName), (Iterable<Vertex>)dmaapVertexList.stream()::iterator); + } + }catch (Exception e) { + e.printStackTrace(); + logger.error("Unable to generate file with dmaap msgs for "+getMigrationName(), e); + }} + + /** + * As string. + * + * @param v the v + * @return the string + */ + protected String asString(Vertex v) { + final JSONObject result = new JSONObject(); + Iterator<VertexProperty<Object>> properties = v.properties(); + Property<Object> pk = null; + try { + while (properties.hasNext()) { + pk = properties.next(); + result.put(pk.key(), pk.value()); + } + } catch (JSONException e) { + logger.error("Warning error reading vertex: " + e); + } + + return result.toString(); + } + + /** + * As string. + * + * @param edge the edge + * @return the string + */ + protected String asString(Edge edge) { + final JSONObject result = new JSONObject(); + Iterator<Property<Object>> properties = edge.properties(); + Property<Object> pk = null; + try { + while (properties.hasNext()) { + pk = properties.next(); + result.put(pk.key(), pk.value()); + } + } catch (JSONException e) { + logger.error("Warning error reading edge: " + e); + } + + return result.toString(); + } + + /** + * + * @param v + * @param numLeadingTabs number of leading \t char's + * @return + */ + protected String toStringForPrinting(Vertex v, int numLeadingTabs) { + String prefix = String.join("", Collections.nCopies(numLeadingTabs, "\t")); + if (v == null) { + return ""; + } + final StringBuilder sb = new StringBuilder(); + sb.append(prefix + v + "\n"); + v.properties().forEachRemaining(prop -> sb.append(prefix + prop + "\n")); + return sb.toString(); + } + + /** + * + * @param e + * @param numLeadingTabs number of leading \t char's + * @return + */ + protected String toStringForPrinting(Edge e, int numLeadingTabs) { + String prefix = String.join("", Collections.nCopies(numLeadingTabs, "\t")); + if (e == null) { + return ""; + } + final StringBuilder sb = new StringBuilder(); + sb.append(prefix + e + "\n"); + sb.append(prefix + e.label() + "\n"); + e.properties().forEachRemaining(prop -> sb.append(prefix + "\t" + prop + "\n")); + return sb.toString(); + } + + /** + * Checks for edge between. + * + * @param a a + * @param b b + * @param d d + * @param edgeLabel the edge label + * @return true, if successful + */ + protected boolean hasEdgeBetween(Vertex a, Vertex b, Direction d, String edgeLabel) { + + if (d.equals(Direction.OUT)) { + return engine.asAdmin().getReadOnlyTraversalSource().V(a).out(edgeLabel).where(__.otherV().hasId(b)).hasNext(); + } else { + return engine.asAdmin().getReadOnlyTraversalSource().V(a).in(edgeLabel).where(__.otherV().hasId(b)).hasNext(); + } + + } + + /** + * Creates the edge + * + * @param type the edge type - COUSIN or TREE + * @param out the out + * @param in the in + * @return the edge + */ + protected Edge createEdge(EdgeType type, Vertex out, Vertex in) throws AAIException { + Edge newEdge = null; + try { + if (type.equals(EdgeType.COUSIN)){ + newEdge = edgeSerializer.addEdge(this.engine.asAdmin().getTraversalSource(), out, in); + } else { + newEdge = edgeSerializer.addTreeEdge(this.engine.asAdmin().getTraversalSource(), out, in); + } + } catch (NoEdgeRuleFoundException e) { + throw new AAIException("AAI_6129", e); + } + return newEdge; + } + + /** + * Creates the edge + * + * @param type the edge type - COUSIN or TREE + * @param out the out + * @param in the in + * @return the edge + */ + protected Edge createPrivateEdge(Vertex out, Vertex in) throws AAIException { + Edge newEdge = null; + try { + newEdge = edgeSerializer.addPrivateEdge(this.engine.asAdmin().getTraversalSource(), out, in, null); + } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) { + throw new AAIException("AAI_6129", e); + } + return newEdge; + } + + /** + * Creates the TREE edge + * + * @param out the out + * @param in the in + * @return the edge + */ + protected Edge createTreeEdge(Vertex out, Vertex in) throws AAIException { + Edge newEdge = createEdge(EdgeType.TREE, out, in); + return newEdge; + } + + /** + * Creates the COUSIN edge + * + * @param out the out + * @param in the in + * @return the edge + */ + protected Edge createCousinEdge(Vertex out, Vertex in) throws AAIException { + Edge newEdge = createEdge(EdgeType.COUSIN, out, in); + return newEdge; + } + + private void initDBSerializer() { + SchemaVersion version = schemaVersions.getDefaultVersion(); + ModelType introspectorFactoryType = ModelType.MOXY; + loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); + try { + this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.getMigrationName()); + } catch (AAIException e) { + throw new RuntimeException("could not create seralizer", e); + } + } + + /** + * These are the node types you would like your traversal to process + * @return + */ + public abstract Optional<String[]> getAffectedNodeTypes(); + + /** + * used as the "fromAppId" when modifying vertices + * @return + */ + public abstract String getMigrationName(); + + /** + * updates all internal vertex properties + * @param v + * @param isNewVertex + */ + protected void touchVertexProperties(Vertex v, boolean isNewVertex) { + this.serializer.touchStandardVertexProperties(v, isNewVertex); + } + + public NotificationHelper getNotificationHelper() { + return this.notificationHelper; + } +} diff --git a/src/main/java/org/onap/aai/migration/NotificationHelper.java b/src/main/java/org/onap/aai/migration/NotificationHelper.java new file mode 100644 index 0000000..ff5c030 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/NotificationHelper.java @@ -0,0 +1,118 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.HashMap; +import java.util.List; + +import javax.ws.rs.core.Response.Status; + +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.rest.ueb.UEBNotification; +import org.onap.aai.serialization.db.DBSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.serialization.engines.query.QueryEngine; + +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; +import org.onap.aai.setup.SchemaVersions; + +/** + * Allows for DMaaP notifications from Migrations + */ +public class NotificationHelper { + + private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(NotificationHelper.class); + protected final DBSerializer serializer; + protected final Loader loader; + protected final TransactionalGraphEngine engine; + protected final String transactionId; + protected final String sourceOfTruth; + protected final UEBNotification notification; + + public NotificationHelper(Loader loader, DBSerializer serializer, LoaderFactory loaderFactory, SchemaVersions schemaVersions, TransactionalGraphEngine engine, String transactionId, String sourceOfTruth) { + this.loader = loader; + this.serializer = serializer; + this.engine = engine; + this.transactionId = transactionId; + this.sourceOfTruth = sourceOfTruth; + this.notification = new UEBNotification(loader, loaderFactory, schemaVersions); + } + + public void addEvent(Vertex v, Introspector obj, EventAction action, URI uri, String basePath) throws UnsupportedEncodingException, AAIException { + HashMap<String, Introspector> relatedObjects = new HashMap<>(); + Status status = mapAction(action); + + if (!obj.isTopLevel()) { + relatedObjects = this.getRelatedObjects(serializer, engine.getQueryEngine(), v); + } + notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, obj, relatedObjects, basePath); + + } + + public void addDeleteEvent(String transactionId, String sourceOfTruth, EventAction action, URI uri, Introspector obj, HashMap relatedObjects,String basePath) throws UnsupportedEncodingException, AAIException { + Status status = mapAction(action); + notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, obj, relatedObjects, basePath); + + } + + private HashMap<String, Introspector> getRelatedObjects(DBSerializer serializer, QueryEngine queryEngine, Vertex v) throws AAIException { + HashMap<String, Introspector> relatedVertices = new HashMap<>(); + List<Vertex> vertexChain = queryEngine.findParents(v); + for (Vertex vertex : vertexChain) { + try { + final Introspector vertexObj = serializer.getVertexProperties(vertex); + relatedVertices.put(vertexObj.getObjectId(), vertexObj); + } catch (AAIUnknownObjectException | UnsupportedEncodingException e) { + LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned"); + } + + } + + return relatedVertices; + } + + private Status mapAction(EventAction action) { + if (EventAction.CREATE.equals(action)) { + return Status.CREATED; + } else if (EventAction.UPDATE.equals(action)) { + return Status.OK; + } else if (EventAction.DELETE.equals(action)) { + return Status.NO_CONTENT; + } else { + return Status.OK; + } + } + + public void triggerEvents() throws AAIException { + notification.triggerEvents(); + } + + public UEBNotification getNotifications() { + return this.notification; + } +} diff --git a/src/main/java/org/onap/aai/migration/PropertyMigrator.java b/src/main/java/org/onap/aai/migration/PropertyMigrator.java new file mode 100644 index 0000000..4599243 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/PropertyMigrator.java @@ -0,0 +1,146 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; + +import org.janusgraph.core.Cardinality; +import org.janusgraph.core.PropertyKey; +import org.janusgraph.core.schema.JanusGraphManagement; +import org.onap.aai.setup.SchemaVersions; + +/** + * A migration template for migrating a property from one name to another + */ +@MigrationPriority(0) +@MigrationDangerRating(1) +public abstract class PropertyMigrator extends Migrator { + + protected String OLD_FIELD; + protected String NEW_FIELD; + protected Integer changedVertexCount; + protected Class<?> fieldType; + protected Cardinality cardinality; + protected final JanusGraphManagement graphMgmt; + + + public PropertyMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){ + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.changedVertexCount = 0; + this.graphMgmt = engine.asAdmin().getManagementSystem(); + } + + public void initialize(String oldName, String newName, Class<?> type, Cardinality cardinality){ + this.OLD_FIELD = oldName; + this.NEW_FIELD = newName; + this.fieldType = type; + this.cardinality = cardinality; + } + + /** + * Do not override this method as an inheritor of this class + */ + @Override + public void run() { + logger.info("-------- Starting PropertyMigrator for node type " + P.within(this.getAffectedNodeTypes().get()) + + " from property " + OLD_FIELD + " to " + NEW_FIELD + " --------"); + modifySchema(); + executeModifyOperation(); + logger.info(Migrator.MIGRATION_SUMMARY_COUNT + changedVertexCount + " vertices modified."); + } + + protected void modifySchema() { + this.addIndex(this.addProperty()); + graphMgmt.commit(); + } + + /** + * This is where inheritors should add their logic + */ + protected void executeModifyOperation() { + changePropertyName(); + } + + protected void changePropertyName() { + GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V(); + if (this.getAffectedNodeTypes().isPresent()) { + g.has(AAIProperties.NODE_TYPE, P.within(this.getAffectedNodeTypes().get())); + } + g.has(OLD_FIELD).sideEffect(t -> { + final Vertex v = t.get(); + logger.info("Migrating property for vertex " + v.toString()); + final String value = v.value(OLD_FIELD); + v.property(OLD_FIELD).remove(); + v.property(NEW_FIELD, value); + this.touchVertexProperties(v, false); + this.changedVertexCount += 1; + logger.info(v.toString() + " : Migrated property " + OLD_FIELD + " to " + NEW_FIELD + " with value = " + value); + }).iterate(); + } + + @Override + public Status getStatus() { + GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V(); + if (this.getAffectedNodeTypes().isPresent()) { + g.has(AAIProperties.NODE_TYPE, P.within(this.getAffectedNodeTypes().get())); + } + long result = g.has(OLD_FIELD).count().next(); + if (result == 0) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + protected Optional<PropertyKey> addProperty() { + + if (!graphMgmt.containsPropertyKey(this.NEW_FIELD)) { + logger.info(" PropertyKey [" + this.NEW_FIELD + "] created in the DB. "); + return Optional.of(graphMgmt.makePropertyKey(this.NEW_FIELD).dataType(this.fieldType).cardinality(this.cardinality) + .make()); + } else { + logger.info(" PropertyKey [" + this.NEW_FIELD + "] already existed in the DB. "); + return Optional.empty(); + } + + } + + protected void addIndex(Optional<PropertyKey> key) { + if (isIndexed() && key.isPresent()) { + if (graphMgmt.containsGraphIndex(key.get().name())) { + logger.debug(" Index [" + key.get().name() + "] already existed in the DB. "); + } else { + logger.info("Add index for PropertyKey: [" + key.get().name() + "]"); + graphMgmt.buildIndex(key.get().name(), Vertex.class).addKey(key.get()).buildCompositeIndex(); + } + } + } + public abstract boolean isIndexed(); + +} diff --git a/src/main/java/org/onap/aai/migration/Status.java b/src/main/java/org/onap/aai/migration/Status.java new file mode 100644 index 0000000..0338594 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/Status.java @@ -0,0 +1,29 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +/** + * Defines the status of the completed migration + */ +public enum Status { + SUCCESS, + CHECK_LOGS, + FAILURE +} diff --git a/src/main/java/org/onap/aai/migration/ValueMigrator.java b/src/main/java/org/onap/aai/migration/ValueMigrator.java new file mode 100644 index 0000000..6d02563 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/ValueMigrator.java @@ -0,0 +1,104 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.util.Map; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.janusgraph.core.schema.JanusGraphManagement; +import org.onap.aai.setup.SchemaVersions; + +/** + * A migration template for filling in default values that are missing or are empty + */ +@MigrationPriority(0) +@MigrationDangerRating(1) +public abstract class ValueMigrator extends Migrator { + + protected final Map<String, Map<String, ?>> propertyValuePairByNodeType; + protected final Boolean updateExistingValues; + protected final JanusGraphManagement graphMgmt; + + /** + * + * @param engine + * @param propertyValuePairByNodeType - format {nodeType: { property: newValue}} + * @param updateExistingValues - if true, updates the value regardless if it already exists + */ + public ValueMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions, Map propertyValuePairByNodeType, Boolean updateExistingValues) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.propertyValuePairByNodeType = propertyValuePairByNodeType; + this.updateExistingValues = updateExistingValues; + this.graphMgmt = engine.asAdmin().getManagementSystem(); + } + + /** + * Do not override this method as an inheritor of this class + */ + @Override + public void run() { + updateValues(); + } + + protected void updateValues() { + for (Map.Entry<String, Map<String, ?>> entry: propertyValuePairByNodeType.entrySet()) { + String nodeType = entry.getKey(); + Map<String, ?> propertyValuePair = entry.getValue(); + for (Map.Entry<String, ?> pair : propertyValuePair.entrySet()) { + String property = pair.getKey(); + Object newValue = pair.getValue(); + try { + GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V() + .has(AAIProperties.NODE_TYPE, nodeType); + while (g.hasNext()) { + Vertex v = g.next(); + if (v.property(property).isPresent() && !updateExistingValues) { + String propertyValue = v.property(property).value().toString(); + if (propertyValue.isEmpty()) { + v.property(property, newValue); + logger.info(String.format("Node Type %s: Property %s is empty, adding value %s", + nodeType, property, newValue.toString())); + this.touchVertexProperties(v, false); + } else { + logger.info(String.format("Node Type %s: Property %s value already exists - skipping", + nodeType, property)); + } + } else { + logger.info(String.format("Node Type %s: Property %s does not exist or " + + "updateExistingValues flag is set to True - adding the property with value %s", + nodeType, property, newValue.toString())); + v.property(property, newValue); + this.touchVertexProperties(v, false); + } + } + } catch (Exception e) { + logger.error(String.format("caught exception updating aai-node-type %s's property %s's value to " + + "%s: %s", nodeType, property, newValue.toString(), e.getMessage())); + logger.error(e.getMessage()); + } + } + } + } +} diff --git a/src/main/java/org/onap/aai/migration/VertexMerge.java b/src/main/java/org/onap/aai/migration/VertexMerge.java new file mode 100644 index 0000000..abf19be --- /dev/null +++ b/src/main/java/org/onap/aai/migration/VertexMerge.java @@ -0,0 +1,255 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + +import java.io.UnsupportedEncodingException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.serialization.db.DBSerializer; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; + +/** + * This class recursively merges two vertices passed in. + * <br> + * You can start with any two vertices, but after the vertices are merged based off the equality of their keys + * + */ +public class VertexMerge { + + private final EELFLogger logger = EELFManager.getInstance().getLogger(this.getClass().getSimpleName()); + + private final GraphTraversalSource g; + private final TransactionalGraphEngine engine; + private final DBSerializer serializer; + private final EdgeSerializer edgeSerializer; + private final Loader loader; + private final NotificationHelper notificationHelper; + private final boolean hasNotifications; + + private VertexMerge(Builder builder) { + this.engine = builder.getEngine(); + this.serializer = builder.getSerializer(); + this.g = engine.asAdmin().getTraversalSource(); + this.edgeSerializer = builder.getEdgeSerializer(); + this.loader = builder.getLoader(); + this.notificationHelper = builder.getHelper(); + this.hasNotifications = builder.isHasNotifications(); + } + + /** + * Merges vertices. forceCopy is a map of the form [{aai-node-type}:{set of properties}] + * @param primary + * @param secondary + * @param forceCopy + * @throws AAIException + * @throws UnsupportedEncodingException + */ + public void performMerge(Vertex primary, Vertex secondary, Map<String, Set<String>> forceCopy, String basePath) throws AAIException, UnsupportedEncodingException { + final Optional<Introspector> secondarySnapshot; + if (this.hasNotifications) { + secondarySnapshot = Optional.of(serializer.getLatestVersionView(secondary)); + } else { + secondarySnapshot = Optional.empty(); + } + mergeProperties(primary, secondary, forceCopy); + + Collection<Vertex> secondaryChildren = this.engine.getQueryEngine().findChildren(secondary); + Collection<Vertex> primaryChildren = this.engine.getQueryEngine().findChildren(primary); + + mergeChildren(primary, secondary, primaryChildren, secondaryChildren, forceCopy); + + Collection<Vertex> secondaryCousins = this.engine.getQueryEngine().findCousinVertices(secondary); + Collection<Vertex> primaryCousins = this.engine.getQueryEngine().findCousinVertices(primary); + + secondaryCousins.removeAll(primaryCousins); + logger.info("removing vertex after merge: " + secondary ); + if (this.hasNotifications && secondarySnapshot.isPresent()) { + this.notificationHelper.addEvent(secondary, secondarySnapshot.get(), EventAction.DELETE, this.serializer.getURIForVertex(secondary, false), basePath); + } + secondary.remove(); + for (Vertex v : secondaryCousins) { + this.edgeSerializer.addEdgeIfPossible(g, v, primary); + } + if (this.hasNotifications) { + final Introspector primarySnapshot = serializer.getLatestVersionView(primary); + this.notificationHelper.addEvent(primary, primarySnapshot, EventAction.UPDATE, this.serializer.getURIForVertex(primary, false), basePath); + } + } + + /** + * This method may go away if we choose to event on each modification performed + * @param primary + * @param secondary + * @param forceCopy + * @throws AAIException + * @throws UnsupportedEncodingException + */ + protected void performMergeHelper(Vertex primary, Vertex secondary, Map<String, Set<String>> forceCopy) throws AAIException, UnsupportedEncodingException { + mergeProperties(primary, secondary, forceCopy); + + Collection<Vertex> secondaryChildren = this.engine.getQueryEngine().findChildren(secondary); + Collection<Vertex> primaryChildren = this.engine.getQueryEngine().findChildren(primary); + + mergeChildren(primary, secondary, primaryChildren, secondaryChildren, forceCopy); + + Collection<Vertex> secondaryCousins = this.engine.getQueryEngine().findCousinVertices(secondary); + Collection<Vertex> primaryCousins = this.engine.getQueryEngine().findCousinVertices(primary); + + secondaryCousins.removeAll(primaryCousins); + secondary.remove(); + for (Vertex v : secondaryCousins) { + this.edgeSerializer.addEdgeIfPossible(g, v, primary); + } + } + + private String getURI(Vertex v) throws UnsupportedEncodingException, AAIException { + Introspector obj = loader.introspectorFromName(v.<String>property(AAIProperties.NODE_TYPE).orElse("")); + this.serializer.dbToObject(Collections.singletonList(v), obj, 0, true, "false"); + return obj.getURI(); + + } + private void mergeChildren(Vertex primary, Vertex secondary, Collection<Vertex> primaryChildren, Collection<Vertex> secondaryChildren, Map<String, Set<String>> forceCopy) throws UnsupportedEncodingException, AAIException { + Map<String, Vertex> primaryMap = uriMap(primaryChildren); + Map<String, Vertex> secondaryMap = uriMap(secondaryChildren); + Set<String> primaryKeys = new HashSet<>(primaryMap.keySet()); + Set<String> secondaryKeys = new HashSet<>(secondaryMap.keySet()); + primaryKeys.retainAll(secondaryKeys); + final Set<String> mergeItems = new HashSet<>(primaryKeys); + primaryKeys = new HashSet<>(primaryMap.keySet()); + secondaryKeys = new HashSet<>(secondaryMap.keySet()); + secondaryKeys.removeAll(primaryKeys); + final Set<String> copyItems = new HashSet<>(secondaryKeys); + + for (String key : mergeItems) { + this.performMergeHelper(primaryMap.get(key), secondaryMap.get(key), forceCopy); + } + + for (String key : copyItems) { + this.edgeSerializer.addTreeEdgeIfPossible(g, secondaryMap.get(key), primary); + this.serializer.getEdgeBetween(EdgeType.TREE, secondary, secondaryMap.get(key)).remove(); + } + + } + + private Map<String, Vertex> uriMap(Collection<Vertex> vertices) throws UnsupportedEncodingException, AAIException { + final Map<String, Vertex> result = new HashMap<>(); + for (Vertex v : vertices) { + result.put(getURI(v), v); + } + return result; + } + + private void mergeProperties(Vertex primary, Vertex secondary, Map<String, Set<String>> forceCopy) throws AAIUnknownObjectException { + final String primaryType = primary.<String>property(AAIProperties.NODE_TYPE).orElse(""); + final String secondaryType = secondary.<String>property(AAIProperties.NODE_TYPE).orElse(""); + + final Introspector secondaryObj = loader.introspectorFromName(secondaryType); + secondary.properties().forEachRemaining(prop -> { + if (!primary.property(prop.key()).isPresent() || forceCopy.getOrDefault(primaryType, new HashSet<String>()).contains(prop.key())) { + primary.property(prop.key(), prop.value()); + } + if (primary.property(prop.key()).isPresent() && secondary.property(prop.key()).isPresent() && secondaryObj.isListType(prop.key())) { + mergeCollection(primary, prop.key(), secondary.values(prop.key())); + } + }); + } + private void mergeCollection(Vertex primary, String propName, Iterator<Object> secondaryValues) { + secondaryValues.forEachRemaining(item -> { + primary.property(propName, item); + }); + } + + + public static class Builder { + private final TransactionalGraphEngine engine; + + private final DBSerializer serializer; + private EdgeSerializer edgeSerializer; + + private final Loader loader; + private NotificationHelper helper = null; + private boolean hasNotifications = false; + public Builder(Loader loader, TransactionalGraphEngine engine, DBSerializer serializer) { + this.loader = loader; + this.engine = engine; + this.serializer = serializer; + } + + public Builder addNotifications(NotificationHelper helper) { + this.helper = helper; + this.hasNotifications = true; + return this; + } + + public Builder edgeSerializer(EdgeSerializer edgeSerializer){ + this.edgeSerializer = edgeSerializer; + return this; + } + + public EdgeSerializer getEdgeSerializer(){ + return edgeSerializer; + } + + public VertexMerge build() { + return new VertexMerge(this); + } + + protected TransactionalGraphEngine getEngine() { + return engine; + } + + protected DBSerializer getSerializer() { + return serializer; + } + + protected Loader getLoader() { + return loader; + } + + protected NotificationHelper getHelper() { + return helper; + } + + protected boolean isHasNotifications() { + return hasNotifications; + } + + } + +} diff --git a/src/main/java/org/onap/aai/migration/v12/ContainmentDeleteOtherVPropertyMigration.java b/src/main/java/org/onap/aai/migration/v12/ContainmentDeleteOtherVPropertyMigration.java new file mode 100644 index 0000000..361e8bc --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/ContainmentDeleteOtherVPropertyMigration.java @@ -0,0 +1,106 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; + +import java.util.Optional; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.enums.EdgeProperty; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.edges.enums.AAIDirection; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +//@Enabled +@MigrationPriority(-100) +@MigrationDangerRating(10) +public class ContainmentDeleteOtherVPropertyMigration extends Migrator { + + private boolean success = true; + + public ContainmentDeleteOtherVPropertyMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + //just for testing using test edge rule files + public ContainmentDeleteOtherVPropertyMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions, String edgeRulesFile) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + public void run() { + try { + engine.asAdmin().getTraversalSource().E().sideEffect(t -> { + Edge e = t.get(); + logger.info("out vertex: " + e.outVertex().property("aai-node-type").value() + + " in vertex: " + e.inVertex().property("aai-node-type").value() + + " label : " + e.label()); + if (e.property(EdgeProperty.CONTAINS.toString()).isPresent() && + e.property(EdgeProperty.DELETE_OTHER_V.toString()).isPresent()) { + //in case of orphans + if (!("constrained-element-set".equals(e.inVertex().property("aai-node-type").value()) + && "model-element".equals(e.outVertex().property("aai-node-type").value()))) { + //skip the weird horrible problem child edge + String containment = (String) e.property(EdgeProperty.CONTAINS.toString()).value(); + if (AAIDirection.OUT.toString().equalsIgnoreCase(containment) || + AAIDirection.IN.toString().equalsIgnoreCase(containment) || + AAIDirection.BOTH.toString().equalsIgnoreCase(containment)) { + logger.info("updating delete-other-v property"); + e.property(EdgeProperty.DELETE_OTHER_V.toString(), containment); + } + } + } + }).iterate(); + } catch (Exception e) { + logger.info("error encountered " + e.getClass() + " " + e.getMessage() + " " + ExceptionUtils.getFullStackTrace(e)); + logger.error("error encountered " + e.getClass() + " " + e.getMessage() + " " + ExceptionUtils.getFullStackTrace(e)); + success = false; + } + + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.empty(); + } + + @Override + public String getMigrationName() { + return "migrate-containment-delete-other-v"; + } + +} diff --git a/src/main/java/org/onap/aai/migration/v12/DeletePInterface.java b/src/main/java/org/onap/aai/migration/v12/DeletePInterface.java new file mode 100644 index 0000000..1089b2f --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/DeletePInterface.java @@ -0,0 +1,131 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +@MigrationPriority(0) +@MigrationDangerRating(0) +public class DeletePInterface extends Migrator { + private boolean success = true; + private final GraphTraversalSource g; + public DeletePInterface(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void run() { + int count = 0; + int skipCount = 0; + int errorCount = 0; + logger.info("---------- Start deleting p-interfaces ----------"); + List<Vertex> pIntfList; + try { + pIntfList = g.V().has(AAIProperties.NODE_TYPE, "p-interface").has("source-of-truth", "AAI-CSVP-INSTARAMS") + .where(this.engine.getQueryBuilder().createEdgeTraversal(EdgeType.TREE, "p-interface", "pnf") + .<GraphTraversal<?, ?>>getQuery()).toList(); + + if (pIntfList != null && !pIntfList.isEmpty()) { + for (Vertex pInterfV : pIntfList) { + try { + Collection<Vertex> cousins = this.engine.getQueryEngine().findCousinVertices(pInterfV); + + Collection<Vertex> children = this.engine.getQueryEngine().findChildren(pInterfV); + if (cousins == null || cousins.isEmpty()) { + if (children == null || children.isEmpty()) { + logger.info("Delete p-interface: " + getVertexURI(pInterfV)); + pInterfV.remove(); + count++; + } else { + skipCount++; + logger.info("skip p-interface " + getVertexURI(pInterfV) + " due to an existing relationship"); + } + } else { + skipCount++; + logger.info("skip p-interface " + getVertexURI(pInterfV) + " due to an existing relationship"); + } + } catch (Exception e) { + success = false; + errorCount++; + logger.error("error occured in deleting p-interface " + getVertexURI(pInterfV) + ", "+ e); + } + } + logger.info ("\n \n ******* Final Summary for deleting p-interfaces Migration ********* \n"); + logger.info("Number of p-interfaces removed: "+ count +"\n"); + logger.info("Number of p-interfaces skipped: "+ skipCount +"\n"); + logger.info("Number of p-interfaces failed to delete due to error : "+ errorCount +"\n"); + } + } catch (AAIException e) { + success = false; + logger.error("error occured in deleting p-interfaces " + e); + } + } + + private String getVertexURI(Vertex v) { + if (v != null) { + if (v.property("aai-uri").isPresent()) { + return v.property("aai-uri").value().toString(); + } else { + return "Vertex ID: " + v.id().toString(); + } + } else { + return ""; + } + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[] { "p-interface" }); + } + + @Override + public String getMigrationName() { + return "DeletePInterface"; + } + +} diff --git a/src/main/java/org/onap/aai/migration/v12/EdgeReportForToscaMigration.java b/src/main/java/org/onap/aai/migration/v12/EdgeReportForToscaMigration.java new file mode 100644 index 0000000..1bdddf3 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/EdgeReportForToscaMigration.java @@ -0,0 +1,162 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +import java.util.*; + +@MigrationPriority(0) +@MigrationDangerRating(0) +public class EdgeReportForToscaMigration extends Migrator { + + private boolean success = true; + + public EdgeReportForToscaMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){ + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public void run() { + Vertex out = null; + Vertex in = null; + String label = ""; + String outURI = ""; + String inURI = ""; + String parentCousinIndicator = "NONE"; + String oldEdgeString = null; + List<String> edgeMissingParentProperty = new ArrayList<>(); + StringBuilder sb = new StringBuilder(); + Set<String> noURI = new HashSet<>(); + sb.append("----------EDGES----------\n"); + + GraphTraversalSource g = engine.asAdmin().getTraversalSource(); + + try { + Set<Edge> edges = g.E().toSet(); + for (Edge edge : edges) { + out = edge.outVertex(); + in = edge.inVertex(); + label = edge.label(); + outURI = this.getVertexURI(out); + inURI = this.getVertexURI(in); + parentCousinIndicator = "NONE"; + oldEdgeString = this.toStringForPrinting(edge, 1); + + if (!outURI.startsWith("/")) { + noURI.add(outURI); + } + if (!inURI.startsWith("/")) { + noURI.add(inURI); + } + + if (out == null || in == null) { + logger.error(edge.id() + " invalid because one vertex was null: out=" + edge.outVertex() + " in=" + edge.inVertex()); + } else { + + if (edge.property("contains-other-v").isPresent()) { + parentCousinIndicator = edge.property("contains-other-v").value().toString(); + } else if (edge.property("isParent").isPresent()) { + if ((Boolean)edge.property("isParent").value()) { + parentCousinIndicator = "OUT"; + } else if (edge.property("isParent-REV").isPresent() && (Boolean)edge.property("isParent-REV").value()) { + parentCousinIndicator = "IN"; + } + } else { + edgeMissingParentProperty.add(this.toStringForPrinting(edge, 1)); + } + + sb.append(outURI + "|" + label + "|" + inURI + "|" + parentCousinIndicator + "\n"); + } + } + } catch(Exception ex){ + logger.error("exception occurred during migration, failing: out=" + out + " in=" + in + "edge=" + oldEdgeString, ex); + success = false; + } + sb.append("--------EDGES END--------\n"); + + logger.info(sb.toString()); + edgeMissingParentProperty.forEach(s -> logger.warn("Edge Missing Parent Property: " + s)); + logger.info("Edge Missing Parent Property Count: " + edgeMissingParentProperty.size()); + logger.info("Vertex Missing URI Property Count: " + noURI.size()); + + } + + private String getVertexURI(Vertex v) { + if (v.property("aai-uri").isPresent()) { + return v.property("aai-uri").value().toString(); + } else { + return v.id().toString() + "(" + v.property("aai-node-type").value().toString() + ")"; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.empty(); + } + + @Override + public String getMigrationName() { + return "edge-report-for-tosca-migration"; + } + + @Override + public void commit() { + engine.rollback(); + } + +} diff --git a/src/main/java/org/onap/aai/migration/v12/MigrateModelVerDistriubutionStatusProperty.java b/src/main/java/org/onap/aai/migration/v12/MigrateModelVerDistriubutionStatusProperty.java new file mode 100644 index 0000000..c09643f --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/MigrateModelVerDistriubutionStatusProperty.java @@ -0,0 +1,85 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +import java.util.Optional; + +@MigrationPriority(20) +@MigrationDangerRating(2) +public class MigrateModelVerDistriubutionStatusProperty extends Migrator{ + + private final String PARENT_NODE_TYPE = "model-ver"; + private boolean success = true; + + public MigrateModelVerDistriubutionStatusProperty(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + + + @Override + public void run() { + + + GraphTraversal<Vertex, Vertex> f = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE,"model-ver"); + + while(f.hasNext()) { + Vertex v = f.next(); + try { + v.property("distribution-status", "DISTRIBUTION_COMPLETE_OK"); + logger.info("changed model-ver.distribution-status property value for model-version-id: " + v.property("model-version-id").value()); + + } catch (Exception e) { + e.printStackTrace(); + success = false; + logger.error("encountered exception for model-version-id:" + v.property("model-version-id").value(), e); + } + } + } + + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{PARENT_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateModelVerDistriubutionStatusProperty"; + } + +} diff --git a/src/main/java/org/onap/aai/migration/v12/MigrateServiceInstanceToConfiguration.java b/src/main/java/org/onap/aai/migration/v12/MigrateServiceInstanceToConfiguration.java new file mode 100644 index 0000000..b4208af --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/MigrateServiceInstanceToConfiguration.java @@ -0,0 +1,193 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; + +import java.io.UnsupportedEncodingException; +import java.util.Iterator; +import java.util.Optional; +import java.util.UUID; + +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +//@Enabled +@MigrationPriority(10) +@MigrationDangerRating(10) +public class MigrateServiceInstanceToConfiguration extends Migrator { + + private boolean success = true; + private final String CONFIGURATION_NODE_TYPE = "configuration"; + private final String SERVICE_INSTANCE_NODE_TYPE = "service-instance"; + private Introspector configObj; + + public MigrateServiceInstanceToConfiguration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + try { + this.configObj = this.loader.introspectorFromName(CONFIGURATION_NODE_TYPE); + } catch (AAIUnknownObjectException e) { + this.configObj = null; + } + } + + @Override + public void run() { + Vertex serviceInstance = null; + Vertex configuration = null; + String serviceInstanceId = "", tunnelBandwidth = ""; + String bandwidthTotal, configType, nodeType; + GraphTraversal<Vertex, Vertex> serviceInstanceItr; + Iterator<Vertex> configurationItr; + + try { + serviceInstanceItr = this.engine.asAdmin().getTraversalSource().V() + .has(AAIProperties.NODE_TYPE, P.within(getAffectedNodeTypes().get())) + .where(this.engine.getQueryBuilder() + .createEdgeTraversal(EdgeType.TREE, "service-instance", "service-subscription") + .getVerticesByProperty("service-type", "DHV") + .<GraphTraversal<?, ?>>getQuery()); + + if (serviceInstanceItr == null || !serviceInstanceItr.hasNext()) { + logger.info("No servince-instance nodes found with service-type of DHV"); + return; + } + + // iterate through all service instances of service-type DHV + while (serviceInstanceItr.hasNext()) { + serviceInstance = serviceInstanceItr.next(); + + if (serviceInstance != null && serviceInstance.property("bandwidth-total").isPresent()) { + serviceInstanceId = serviceInstance.value("service-instance-id"); + logger.info("Processing service instance with id=" + serviceInstanceId); + bandwidthTotal = serviceInstance.value("bandwidth-total"); + + if (bandwidthTotal != null && !bandwidthTotal.isEmpty()) { + + // check for existing edges to configuration nodes + configurationItr = serviceInstance.vertices(Direction.OUT, "has"); + + // create new configuration node if service-instance does not have existing ones + if (!configurationItr.hasNext()) { + logger.info(serviceInstanceId + " has no existing configuration nodes, creating new node"); + createConfigurationNode(serviceInstance, bandwidthTotal); + continue; + } + + // in case if configuration nodes exist, but none are DHV + boolean hasDHVConfig = false; + + // service-instance has existing configuration nodes + while (configurationItr.hasNext()) { + configuration = configurationItr.next(); + nodeType = configuration.value("aai-node-type").toString(); + + if (configuration != null && "configuration".equalsIgnoreCase(nodeType)) { + logger.info("Processing configuration node with id=" + configuration.property("configuration-id").value()); + configType = configuration.value("configuration-type"); + logger.info("Configuration type: " + configType); + + // if configuration-type is DHV, update tunnel-bandwidth to bandwidth-total value + if ("DHV".equalsIgnoreCase(configType)) { + if (configuration.property("tunnel-bandwidth").isPresent()) { + tunnelBandwidth = configuration.value("tunnel-bandwidth"); + } else { + tunnelBandwidth = ""; + } + + logger.info("Existing tunnel-bandwidth: " + tunnelBandwidth); + configuration.property("tunnel-bandwidth", bandwidthTotal); + touchVertexProperties(configuration, false); + logger.info("Updated tunnel-bandwidth: " + configuration.value("tunnel-bandwidth")); + hasDHVConfig = true; + } + } + } + + // create new configuration node if none of existing config nodes are of type DHV + if (!hasDHVConfig) { + logger.info(serviceInstanceId + " has existing configuration nodes, but none are DHV, create new node"); + createConfigurationNode(serviceInstance, bandwidthTotal); + } + } + } + } + } catch (AAIException | UnsupportedEncodingException e) { + logger.error("Caught exception while processing service instance with id=" + serviceInstanceId + " | " + e.toString()); + success = false; + } + } + + private void createConfigurationNode(Vertex serviceInstance, String bandwidthTotal) throws UnsupportedEncodingException, AAIException { + // create new vertex + Vertex configurationNode = serializer.createNewVertex(configObj); + + // configuration-id: UUID format + String configurationUUID = UUID.randomUUID().toString(); + configObj.setValue("configuration-id", configurationUUID); + + // configuration-type: DHV + configObj.setValue("configuration-type", "DHV"); + + // migrate the bandwidth-total property from the service-instance to the + // tunnel-bandwidth property of the related configuration object + configObj.setValue("tunnel-bandwidth", bandwidthTotal); + + // create edge between service instance and configuration: cousinEdge(out, in) + createCousinEdge(serviceInstance, configurationNode); + + // serialize edge & vertex, takes care of everything + serializer.serializeSingleVertex(configurationNode, configObj, "migrations"); + logger.info("Created configuration node with uuid=" + configurationUUID + ", tunnel-bandwidth=" + bandwidthTotal); + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[] {SERVICE_INSTANCE_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "service-instance-to-configuration"; + } +} diff --git a/src/main/java/org/onap/aai/migration/v12/SDWANSpeedChangeMigration.java b/src/main/java/org/onap/aai/migration/v12/SDWANSpeedChangeMigration.java new file mode 100644 index 0000000..b420c57 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/SDWANSpeedChangeMigration.java @@ -0,0 +1,258 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +import java.util.*; + +@MigrationPriority(1) +@MigrationDangerRating(1) +//@Enabled +public class SDWANSpeedChangeMigration extends Migrator { + + private final String PARENT_NODE_TYPE = "alloted-resource"; + private boolean success = true; + + Vertex allottedRsrcVertex; + + Map<String, String> bandwidthMap = new HashMap<>(); + Set<String> bandWidthSet = new HashSet<>(); + + GraphTraversal<Vertex, Vertex> allottedRsrcTraversal; + GraphTraversal<Vertex, Vertex> tunnelXConnectTraversal; + GraphTraversal<Vertex, Vertex> pinterfaceTraversal; + GraphTraversal<Vertex, Vertex> plinkTraversal; + + public SDWANSpeedChangeMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + bandWidthSet.add("bandwidth-up-wan1"); + bandWidthSet.add("bandwidth-down-wan1"); + bandWidthSet.add("bandwidth-up-wan2"); + bandWidthSet.add("bandwidth-down-wan2"); + } + + + @Override + public void run() { + + logger.info("Started the migration "+ getMigrationName()); + + try { + + allottedRsrcTraversal = this.engine.asAdmin().getTraversalSource().V() + .has("aai-node-type", "service-subscription") + .has("service-type", "DHV") + .in("org.onap.relationships.inventory.BelongsTo") + .has("aai-node-type", "service-instance") + .out("org.onap.relationships.inventory.Uses") + .has("aai-node-type", "allotted-resource") + .where( + this.engine.getQueryBuilder() + .createEdgeTraversal(EdgeType.TREE, "allotted-resource", "service-instance") + .createEdgeTraversal(EdgeType.TREE, "service-instance", "service-subscription") + .<GraphTraversal<Vertex, Vertex>>getQuery() + .has("service-type", "VVIG") + ); + + if(!(allottedRsrcTraversal.hasNext())){ + + logger.info("unable to find allotted resource to DHV as cousin and child of VVIG"); + } + + while (allottedRsrcTraversal.hasNext()) { + bandwidthMap.clear(); + + allottedRsrcVertex = allottedRsrcTraversal.next(); + String allottedResourceId = allottedRsrcVertex.property("id").value().toString(); + logger.info("Found an allotted resource with id " + allottedResourceId); + + tunnelXConnectTraversal = this.engine.asAdmin().getTraversalSource() + .V(allottedRsrcVertex) + .in("org.onap.relationships.inventory.BelongsTo") + .has("aai-node-type", "tunnel-xconnect"); + + if (tunnelXConnectTraversal != null && tunnelXConnectTraversal.hasNext()) { + Vertex xConnect = tunnelXConnectTraversal.next(); + String tunnelId = xConnect.property("id").value().toString(); + logger.info("Found an tunnelxconnect object with id " + tunnelId); + extractBandwidthProps(xConnect); + modifyPlink(allottedRsrcVertex); + } else { + logger.info("Unable to find the tunnel connect for the current allotted resource traversal"); + } + + } + } catch (AAIException e) { + e.printStackTrace(); + success = false; + } + + logger.info("Successfully finished the " + getMigrationName()); + } + + public void extractBandwidthProps(Vertex vertex) { + logger.info("Trying to extract bandwith props"); + bandWidthSet.stream().forEach((key) -> { + if (vertex.property(key).isPresent()) { + bandwidthMap.put(key, vertex.property(key).value().toString()); + } + }); + logger.info("Extracted bandwith props for tunnelXConnect " +vertex.value("id")); + } + + public void modifyPlink(Vertex v) { + + try { + pinterfaceTraversal = this.engine.asAdmin().getTraversalSource().V(v) + .in("org.onap.relationships.inventory.Uses").has("aai-node-type", "service-instance") + .where( + __.out("org.onap.relationships.inventory.BelongsTo") + .has("aai-node-type", "service-subscription") + .has("service-type", "DHV") + ) + .out("org.onap.relationships.inventory.ComposedOf").has("aai-node-type", "generic-vnf") + .out("tosca.relationships.HostedOn").has("aai-node-type", "vserver") + .out("tosca.relationships.HostedOn").has("aai-node-type", "pserver") + .in("tosca.relationships.network.BindsTo").has("aai-node-type", "p-interface"); + } catch (Exception e) { + logger.info("error trying to find p interfaces"); + } + + + while (pinterfaceTraversal.hasNext()) { + + Vertex pInterfaceVertex = pinterfaceTraversal.next(); + + String pinterfaceName = pInterfaceVertex.property("interface-name").value().toString(); + logger.info("p-interface "+ pinterfaceName + " found from traversal from allotted-resource " +v.value("id")); + String[] parts = pinterfaceName.split("/"); + + if (parts[parts.length - 1].equals("10")) { + + logger.info("Found the pinterface with the interface name ending with /10"); + + try { + plinkTraversal = this.engine.asAdmin().getTraversalSource() + .V(pInterfaceVertex) + .out("tosca.relationships.network.LinksTo") + .has("aai-node-type", "physical-link"); + } catch (Exception e) { + logger.info("error trying to find the p Link for /10"); + } + if (plinkTraversal != null && plinkTraversal.hasNext()) { + Vertex pLink = plinkTraversal.next(); + + + if ( bandwidthMap.containsKey("bandwidth-up-wan1") + && bandwidthMap.containsKey("bandwidth-down-wan1") + && !(("").equals(bandwidthMap.get("bandwidth-up-wan1").replaceAll("[^0-9]", "").trim())) + && !(("").equals(bandwidthMap.get("bandwidth-down-wan1").replaceAll("[^0-9]", "").trim()))) + { + + pLink.property("service-provider-bandwidth-up-value", Integer.valueOf(bandwidthMap.get("bandwidth-up-wan1").replaceAll("[^0-9]", "").trim())); + pLink.property("service-provider-bandwidth-up-units", "Mbps"); + pLink.property("service-provider-bandwidth-down-value", Integer.valueOf(bandwidthMap.get("bandwidth-down-wan1").replaceAll("[^0-9]", "").trim())); + pLink.property("service-provider-bandwidth-down-units", "Mbps"); + logger.info("Successfully modified the plink with link name ", pLink.property("link-name").value().toString()); + this.touchVertexProperties(pLink, false); + } else { + logger.info("missing up and down vals for the plink with link name ", pLink.property("link-name").value().toString()); + } + + + } else { + logger.info("missing plink for p interface" + pinterfaceName); + } + + } + + if (parts[parts.length - 1].equals("11")) { + + logger.info("Found the pinterface with the interface name ending with /11"); + try { + plinkTraversal = this.engine.asAdmin() + .getTraversalSource() + .V(pInterfaceVertex) + .out("tosca.relationships.network.LinksTo") + .has("aai-node-type", "physical-link"); + } catch (Exception e) { + logger.info("error trying to find the p Link for /11"); + } + + if (plinkTraversal != null && plinkTraversal.hasNext()) { + Vertex pLink = plinkTraversal.next(); + + + if ( bandwidthMap.containsKey("bandwidth-up-wan2") + && bandwidthMap.containsKey("bandwidth-down-wan2") + && !(("").equals(bandwidthMap.get("bandwidth-up-wan2").replaceAll("[^0-9]", "").trim())) + && !(("").equals(bandwidthMap.get("bandwidth-down-wan2").replaceAll("[^0-9]", "").trim()))) + { + pLink.property("service-provider-bandwidth-up-value", Integer.valueOf(bandwidthMap.get("bandwidth-up-wan2").replaceAll("[^0-9]", "").trim())); + pLink.property("service-provider-bandwidth-up-units", "Mbps"); + pLink.property("service-provider-bandwidth-down-value", Integer.valueOf(bandwidthMap.get("bandwidth-down-wan2").replaceAll("[^0-9]", "").trim())); + pLink.property("service-provider-bandwidth-down-units", "Mbps"); + logger.info("Successfully modified the plink with link name ", pLink.property("link-name").value().toString()); + this.touchVertexProperties(pLink, false); + } else { + logger.error("missing up and down vals for the plink with link name ", pLink.property("link-name").value().toString()); + } + + } else { + logger.info("missing plink for p interface" + pinterfaceName); + } + } + } + } + + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + + return Optional.of(new String[]{PARENT_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "SDWANSpeedChangeMigration"; + } + + +} diff --git a/src/main/java/org/onap/aai/migration/v12/UpdateAaiUriIndexMigration.java b/src/main/java/org/onap/aai/migration/v12/UpdateAaiUriIndexMigration.java new file mode 100644 index 0000000..33689b5 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/UpdateAaiUriIndexMigration.java @@ -0,0 +1,328 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; + +import org.janusgraph.core.Cardinality; +import org.janusgraph.core.PropertyKey; +import org.janusgraph.core.schema.SchemaAction; +import org.janusgraph.core.schema.SchemaStatus; +import org.janusgraph.core.schema.JanusGraphIndex; +import org.janusgraph.core.schema.JanusGraphManagement; +import org.janusgraph.graphdb.database.management.ManagementSystem; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.dbmap.AAIGraph; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.migration.*; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.setup.SchemaVersions; + +import java.time.temporal.ChronoUnit; +import java.util.*; + +/** + * Remove old aai-uri index per + * https://github.com/JanusGraph/janusgraph/wiki/Indexing + */ + +@Enabled + +@MigrationPriority(500) +@MigrationDangerRating(1000) +public class UpdateAaiUriIndexMigration extends Migrator { + + private final SchemaVersion version; + private final ModelType introspectorFactoryType; + private GraphTraversalSource g; + private JanusGraphManagement graphMgmt; + private Status status = Status.SUCCESS; + + private String retiredName = AAIProperties.AAI_URI + "-RETIRED-" + System.currentTimeMillis(); + + /** + * Instantiates a new migrator. + * + * @param engine + */ + public UpdateAaiUriIndexMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) throws AAIException { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + version = schemaVersions.getDefaultVersion(); + introspectorFactoryType = ModelType.MOXY; + loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); + g = this.engine.asAdmin().getTraversalSource(); + this.engine.rollback(); + graphMgmt = engine.asAdmin().getManagementSystem(); + + } + + @Override + public Status getStatus() { + return status; + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.empty(); + } + + @Override + public String getMigrationName() { + return "UpdateAaiUriIndex"; + } + + @Override + public void run() { + + // close all but current open titan instances + closeAllButCurrentInstances(); + + // get all indexes containing aai-uri + Set<IndexDetails> indexes = getIndexesWithAaiUri(); + logger.info("Found " + indexes.size() + " aai uri index."); + indexes.stream().map(s -> "\t" + s.getIndexName() + " : " + s.getPropertyName() + " : " + s.getStatus() ).forEach(System.out::println); + + renameAaiUriIndex(indexes); + + // remove all of the aai-uri indexes that are in the list + removeIndexes(indexes); + + //retire old property + verifyGraphManagementIsOpen(); + PropertyKey aaiUri = graphMgmt.getPropertyKey(AAIProperties.AAI_URI); + if (aaiUri != null) { + graphMgmt.changeName(aaiUri, retiredName); + } + graphMgmt.commit(); + + //remove all aai uri keys + logger.info("Remove old keys."); + dropAllKeyProperties(indexes); + + // add aai-uri unique index + logger.info("Create new unique aai-uri index"); + createUniqueAaiUriIndex(); + + + // change index status to ENABLED STATE + logger.info("Enable index"); + enableIndex(); + + this.engine.startTransaction(); + + logger.info("Checking and dropping retired properties."); + g = this.engine.asAdmin().getTraversalSource(); + g.V().has(retiredName).properties(retiredName).drop().iterate(); + logger.info("Done."); + } + + + protected void createUniqueAaiUriIndex() { + verifyGraphManagementIsOpen(); + // create new aaiuri property + PropertyKey aaiUriProperty = graphMgmt.getPropertyKey(AAIProperties.AAI_URI); + if (aaiUriProperty == null) { + logger.info("Creating new aai-uri property."); + aaiUriProperty = graphMgmt.makePropertyKey(AAIProperties.AAI_URI).dataType(String.class) + .cardinality(Cardinality.SINGLE).make(); + } + logger.info("Creating new aai-uri index."); + graphMgmt.buildIndex(AAIProperties.AAI_URI, Vertex.class).addKey(aaiUriProperty).unique().buildCompositeIndex(); + graphMgmt.commit(); + } + + private void dropAllKeyProperties(Set<IndexDetails> indexes) { + indexes.stream().map(e -> e.getPropertyName()).distinct().forEach(p -> { + verifyGraphManagementIsOpen(); + if (graphMgmt.getPropertyKey(p) != null) { + graphMgmt.getPropertyKey(p).remove(); + } + graphMgmt.commit(); + }); + } + + private void renameAaiUriIndex(Set<IndexDetails> indexes) { + verifyGraphManagementIsOpen(); + indexes.stream().filter(s -> s.getIndexName().equals(AAIProperties.AAI_URI)).forEach( s -> { + JanusGraphIndex index = graphMgmt.getGraphIndex(s.getIndexName()); + graphMgmt.changeName(index, retiredName); + s.setIndexName(retiredName); + }); + graphMgmt.commit(); + } + + private void removeIndexes(Set<IndexDetails> indexes) { + + for (IndexDetails index : indexes) { + verifyGraphManagementIsOpen(); + + JanusGraphIndex aaiUriIndex = graphMgmt.getGraphIndex(index.getIndexName()); + + if (!index.getStatus().equals(SchemaStatus.DISABLED)) { + logger.info("Disabling index: " + index.getIndexName()); + logger.info("\tCurrent state: " + aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(index.getPropertyName()))); + + graphMgmt.updateIndex(aaiUriIndex, SchemaAction.DISABLE_INDEX); + graphMgmt.commit(); + try { + ManagementSystem.awaitGraphIndexStatus(AAIGraph.getInstance().getGraph(), index.getIndexName()) + .timeout(10, ChronoUnit.MINUTES) + .status(SchemaStatus.DISABLED) + .call(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + verifyGraphManagementIsOpen(); + aaiUriIndex = graphMgmt.getGraphIndex(index.getIndexName()); + if (aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(index.getPropertyName())).equals(SchemaStatus.DISABLED)) { + logger.info("Removing index: " + index.getIndexName()); + graphMgmt.updateIndex(aaiUriIndex, SchemaAction.REMOVE_INDEX); + graphMgmt.commit(); + } + if(graphMgmt.isOpen()) { + graphMgmt.commit(); + } + } + + } + + protected Set<IndexDetails> getIndexesWithAaiUri() { + verifyGraphManagementIsOpen(); + Set<IndexDetails> aaiUriIndexName = new HashSet<>(); + + Iterator<JanusGraphIndex> titanIndexes = graphMgmt.getGraphIndexes(Vertex.class).iterator(); + JanusGraphIndex titanIndex; + while (titanIndexes.hasNext()) { + titanIndex = titanIndexes.next(); + if (titanIndex.name().contains(AAIProperties.AAI_URI) && titanIndex.getFieldKeys().length > 0) { + logger.info("Found aai-uri index: " + titanIndex.name()); + aaiUriIndexName.add(new IndexDetails(titanIndex.name(), titanIndex.getIndexStatus(titanIndex.getFieldKeys()[0]), titanIndex.getFieldKeys()[0].name())); + } + } + graphMgmt.rollback(); + return aaiUriIndexName; + } + + private void closeAllButCurrentInstances() { + verifyGraphManagementIsOpen(); + logger.info("Closing all but current titan instances."); + graphMgmt.getOpenInstances().stream().filter(s -> !s.contains("(current)")).forEach(s -> { + logger.info("\t"+s); + graphMgmt.forceCloseInstance(s); + }); + graphMgmt.commit(); + } + + + private void verifyGraphManagementIsOpen() { + if (!graphMgmt.isOpen()) { + graphMgmt = this.engine.asAdmin().getManagementSystem(); + } + } + + private void enableIndex() { + verifyGraphManagementIsOpen(); + JanusGraphIndex aaiUriIndex = graphMgmt.getGraphIndex(AAIProperties.AAI_URI); + SchemaStatus schemaStatus = aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(AAIProperties.AAI_URI)); + if (schemaStatus.equals(SchemaStatus.INSTALLED)) { + logger.info("Registering index: " + AAIProperties.AAI_URI); + logger.info("\tCurrent state: " + schemaStatus); + + graphMgmt.updateIndex(aaiUriIndex, SchemaAction.REGISTER_INDEX); + graphMgmt.commit(); + try { + ManagementSystem.awaitGraphIndexStatus(AAIGraph.getInstance().getGraph(), AAIProperties.AAI_URI) + .timeout(10, ChronoUnit.MINUTES) + .status(SchemaStatus.REGISTERED) + .call(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + verifyGraphManagementIsOpen(); + aaiUriIndex = graphMgmt.getGraphIndex(AAIProperties.AAI_URI); + schemaStatus = aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(AAIProperties.AAI_URI)); + if (schemaStatus.equals(SchemaStatus.REGISTERED)) { + logger.info("Enabling index: " + AAIProperties.AAI_URI); + logger.info("\tCurrent state: " + schemaStatus); + + graphMgmt.updateIndex(aaiUriIndex, SchemaAction.ENABLE_INDEX); + graphMgmt.commit(); + try { + ManagementSystem.awaitGraphIndexStatus(AAIGraph.getInstance().getGraph(), AAIProperties.AAI_URI) + .timeout(10, ChronoUnit.MINUTES) + .status(SchemaStatus.ENABLED) + .call(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + verifyGraphManagementIsOpen(); + aaiUriIndex = graphMgmt.getGraphIndex(AAIProperties.AAI_URI); + schemaStatus = aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(AAIProperties.AAI_URI)); + logger.info("Final state: " + schemaStatus); + graphMgmt.rollback(); + } + + private class IndexDetails { + private String indexName; + private SchemaStatus status; + private String propertyName; + + public IndexDetails(String indexName, SchemaStatus status, String propertyName) { + this.indexName = indexName; + this.status = status; + this.propertyName = propertyName; + } + + public String getIndexName() { + return indexName; + } + + public SchemaStatus getStatus() { + return status; + } + + public String getPropertyName() { + return propertyName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public void setStatus(SchemaStatus status) { + this.status = status; + } + + public void setPropertyName(String propertyName) { + this.propertyName = propertyName; + } + } +} diff --git a/src/main/java/org/onap/aai/migration/v12/UriMigration.java b/src/main/java/org/onap/aai/migration/v12/UriMigration.java new file mode 100644 index 0000000..cb0926e --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/UriMigration.java @@ -0,0 +1,180 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.enums.EdgeProperty; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.migration.*; +import org.onap.aai.edges.enums.AAIDirection; +import org.onap.aai.serialization.db.DBSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.setup.SchemaVersions; +import org.springframework.web.util.UriUtils; + +import javax.ws.rs.core.UriBuilder; +import java.io.UnsupportedEncodingException; +import java.util.*; +import java.util.stream.Collectors; + +@Enabled + +@MigrationPriority(1000) +@MigrationDangerRating(1000) +public class UriMigration extends Migrator { + + private final SchemaVersion version; + private final ModelType introspectorFactoryType; + private GraphTraversalSource g; + + private Map<String, UriBuilder> nodeTypeToUri; + private Map<String, Set<String>> nodeTypeToKeys; + + protected Set<Object> seen = new HashSet<>(); + + /** + * Instantiates a new migrator. + * + * @param engine + */ + public UriMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) throws AAIException { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + version = schemaVersions.getDefaultVersion(); + introspectorFactoryType = ModelType.MOXY; + loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); + g = this.engine.asAdmin().getTraversalSource(); + this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.getMigrationName()); + + } + + @Override + public void run() { + long start = System.currentTimeMillis(); + nodeTypeToUri = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect( + Collectors.toMap( + e -> e.getKey(), + e -> UriBuilder.fromPath(e.getValue().getFullGenericURI().replaceAll("\\{"+ e.getKey() + "-", "{")) + )); + + nodeTypeToKeys = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect( + Collectors.toMap( + e -> e.getKey(), + e -> e.getValue().getKeys() + )); + + Set<String> topLevelNodeTypes = loader.getAllObjects().entrySet().stream() + .filter(e -> e.getValue().isTopLevel()).map(Map.Entry::getKey) + .collect(Collectors.toSet()); + + logger.info("Top level count : " + topLevelNodeTypes.size()); + topLevelNodeTypes.stream().forEach(topLevelNodeType -> { + Set<Vertex> parentSet = g.V().has(AAIProperties.NODE_TYPE, topLevelNodeType).toSet(); + logger.info(topLevelNodeType + " : " + parentSet.size()); + try { + this.verifyOrAddUri("", parentSet); + } catch (AAIUnknownObjectException e) { + e.printStackTrace(); + } catch (AAIException e) { + e.printStackTrace(); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + }); + logger.info("RUNTIME: " + (System.currentTimeMillis() - start)); + logger.info("NO URI: " + g.V().hasNot(AAIProperties.AAI_URI).count().next()); + logger.info("NUM VERTEXES SEEN: " + seen.size()); + seen = new HashSet<>(); + + } + + protected void verifyOrAddUri(String parentUri, Set<Vertex> vertexSet) throws UnsupportedEncodingException, AAIException { + String correctUri; + for (Vertex v : vertexSet) { + seen.add(v.id()); + //if there is an issue generating the uri catch, log and move on; + try { + correctUri = parentUri + this.getUriForVertex(v); + } catch (Exception e) { + logger.error("Vertex has issue generating uri " + e.getMessage() + "\n\t" + this.asString(v)); + continue; + } + try { + v.property(AAIProperties.AAI_URI, correctUri); + } catch (Exception e) { + logger.info(e.getMessage() + "\n\t" + this.asString(v)); + } + if (!v.property(AAIProperties.AAI_UUID).isPresent()) { + v.property(AAIProperties.AAI_UUID, UUID.randomUUID().toString()); + } + this.verifyOrAddUri(correctUri, getChildren(v)); + } + } + + protected Set<Vertex> getChildren(Vertex v) { + + Set<Vertex> children = g.V(v).bothE().not(__.has(EdgeProperty.CONTAINS.toString(), AAIDirection.NONE.toString())).otherV().toSet(); + + return children.stream().filter(child -> !seen.contains(child.id())).collect(Collectors.toSet()); + } + + protected String getUriForVertex(Vertex v) { + String aaiNodeType = v.property(AAIProperties.NODE_TYPE).value().toString(); + + + Map<String, String> parameters = this.nodeTypeToKeys.get(aaiNodeType).stream().collect(Collectors.toMap( + key -> key, + key -> encodeProp(v.property(key).value().toString()) + )); + + return this.nodeTypeToUri.get(aaiNodeType).buildFromEncodedMap(parameters).toString(); + } + + private static String encodeProp(String s) { + try { + return UriUtils.encode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + return ""; + } + } + + @Override + public Status getStatus() { + return Status.SUCCESS; + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.empty(); + } + + @Override + public String getMigrationName() { + return UriMigration.class.getSimpleName(); + } +} diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateBooleanDefaultsToFalse.java b/src/main/java/org/onap/aai/migration/v13/MigrateBooleanDefaultsToFalse.java new file mode 100644 index 0000000..89a9459 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateBooleanDefaultsToFalse.java @@ -0,0 +1,114 @@ +/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Status;
+import org.onap.aai.migration.ValueMigrator;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(1)
+@MigrationDangerRating(1)
+public class MigrateBooleanDefaultsToFalse extends ValueMigrator {
+ protected static final String VNF_NODE_TYPE = "generic-vnf";
+ protected static final String VSERVER_NODE_TYPE = "vserver";
+ protected static final String VNFC_NODE_TYPE = "vnfc";
+ protected static final String L3NETWORK_NODE_TYPE = "l3-network";
+ protected static final String SUBNET_NODE_TYPE = "subnet";
+ protected static final String LINTERFACE_NODE_TYPE = "l-interface";
+ protected static final String VFMODULE_NODE_TYPE = "vf-module";
+
+ private static Map<String, Map> map;
+ private static Map<String, Boolean> pair1;
+ private static Map<String, Boolean> pair2;
+ private static Map<String, Boolean> pair3;
+ private static Map<String, Boolean> pair4;
+ private static Map<String, Boolean> pair5;
+ private static Map<String, Boolean> pair6;
+
+ public MigrateBooleanDefaultsToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setBooleanDefaultsToFalse(), false);
+
+ }
+
+ private static Map<String, Map> setBooleanDefaultsToFalse(){
+ map = new HashMap<>();
+ pair1 = new HashMap<>();
+ pair2 = new HashMap<>();
+ pair3 = new HashMap<>();
+ pair4 = new HashMap<>();
+ pair5 = new HashMap<>();
+ pair6 = new HashMap<>();
+
+
+ pair1.put("is-closed-loop-disabled", false);
+ map.put("generic-vnf", pair1);
+ map.put("vnfc", pair1);
+ map.put("vserver", pair1);
+
+ pair2.put("is-bound-to-vpn", false);
+ pair2.put("is-provider-network", false);
+ pair2.put("is-shared-network", false);
+ pair2.put("is-external-network", false);
+ map.put("l3-network", pair2);
+
+ pair3.put("dhcp-enabled", false);
+ map.put("subnet", pair3);
+
+ pair4.put("is-port-mirrored", false);
+ pair4.put("is-ip-unnumbered", false);
+ map.put("l-interface", pair4);
+
+ pair5.put("is-base-vf-module", false);
+ map.put("vf-module", pair5);
+
+ pair6.put("is-ip-unnumbered", false);
+ map.put("vlan", pair6);
+
+ return map;
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{VNF_NODE_TYPE,VSERVER_NODE_TYPE,VNFC_NODE_TYPE,L3NETWORK_NODE_TYPE,SUBNET_NODE_TYPE,LINTERFACE_NODE_TYPE,VFMODULE_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateBooleanDefaultsToFalse";
+ }
+
+}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInMaintDefaultToFalse.java b/src/main/java/org/onap/aai/migration/v13/MigrateInMaintDefaultToFalse.java new file mode 100644 index 0000000..1773038 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateInMaintDefaultToFalse.java @@ -0,0 +1,98 @@ +/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Status;
+import org.onap.aai.migration.ValueMigrator;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(1)
+@MigrationDangerRating(1)
+public class MigrateInMaintDefaultToFalse extends ValueMigrator {
+
+ protected static final String VNF_NODE_TYPE = "generic-vnf";
+ protected static final String LINTERFACE_NODE_TYPE = "l-interface";
+ protected static final String LAG_INTERFACE_NODE_TYPE = "lag-interface";
+ protected static final String LOGICAL_LINK_NODE_TYPE = "logical-link";
+ protected static final String PINTERFACE_NODE_TYPE = "p-interface";
+ protected static final String VLAN_NODE_TYPE = "vlan";
+ protected static final String VNFC_NODE_TYPE = "vnfc";
+ protected static final String VSERVER_NODE_TYPE = "vserver";
+ protected static final String PSERVER_NODE_TYPE = "pserver";
+ protected static final String PNF_NODE_TYPE = "pnf";
+ protected static final String NOS_SERVER_NODE_TYPE = "nos-server";
+
+ private static Map<String, Map> map;
+ private static Map<String, Boolean> pair;
+
+ public MigrateInMaintDefaultToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setInMaintToFalse(), false);
+ }
+
+ private static Map<String, Map> setInMaintToFalse(){
+ map = new HashMap<>();
+ pair = new HashMap<>();
+
+ pair.put("in-maint", false);
+
+ map.put("generic-vnf", pair);
+ map.put("l-interface", pair);
+ map.put("lag-interface", pair);
+ map.put("logical-link", pair);
+ map.put("p-interface", pair);
+ map.put("vlan", pair);
+ map.put("vnfc", pair);
+ map.put("vserver", pair);
+ map.put("pserver", pair);
+ map.put("pnf", pair);
+ map.put("nos-server", pair);
+
+ return map;
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{VNF_NODE_TYPE,LINTERFACE_NODE_TYPE,LAG_INTERFACE_NODE_TYPE,LOGICAL_LINK_NODE_TYPE,PINTERFACE_NODE_TYPE,VLAN_NODE_TYPE,VNFC_NODE_TYPE,VSERVER_NODE_TYPE,PSERVER_NODE_TYPE,PNF_NODE_TYPE,NOS_SERVER_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateInMaintDefaultToFalse";
+ }
+
+}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantId.java b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantId.java new file mode 100644 index 0000000..1244c59 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantId.java @@ -0,0 +1,85 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import org.janusgraph.core.Cardinality; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.PropertyMigrator; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +import java.util.Optional; + + +@MigrationPriority(19) +@MigrationDangerRating(2) +@Enabled +public class MigrateInstanceGroupModelInvariantId extends PropertyMigrator { + + private static final String INSTANCE_GROUP_NODE_TYPE = "instance-group"; + private static final String INSTANCE_GROUP_MODEL_INVARIANT_ID_PROPERTY = "model-invariant-id"; + private static final String INSTANCE_GROUP_MODEL_INVARIANT_ID_LOCAL_PROPERTY = "model-invariant-id-local"; + + public MigrateInstanceGroupModelInvariantId(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.initialize(INSTANCE_GROUP_MODEL_INVARIANT_ID_PROPERTY, INSTANCE_GROUP_MODEL_INVARIANT_ID_LOCAL_PROPERTY, + String.class, Cardinality.SINGLE); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{this.INSTANCE_GROUP_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateInstanceGroupModelInvariantId"; + } + + @Override + public boolean isIndexed() { + return true; + } +} diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionId.java b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionId.java new file mode 100644 index 0000000..64341ba --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionId.java @@ -0,0 +1,84 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import java.util.Optional; +import org.janusgraph.core.Cardinality; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.PropertyMigrator; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(19) +@MigrationDangerRating(2) +@Enabled +public class MigrateInstanceGroupModelVersionId extends PropertyMigrator { + + private static final String INSTANCE_GROUP_NODE_TYPE = "instance-group"; + private static final String INSTANCE_GROUP_MODEL_VERSION_ID_PROPERTY = "model-version-id"; + private static final String INSTANCE_GROUP_MODEL_VERSION_ID_LOCAL_PROPERTY = "model-version-id-local"; + + public MigrateInstanceGroupModelVersionId(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.initialize(INSTANCE_GROUP_MODEL_VERSION_ID_PROPERTY, INSTANCE_GROUP_MODEL_VERSION_ID_LOCAL_PROPERTY, + String.class, Cardinality.SINGLE); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{this.INSTANCE_GROUP_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateInstanceGroupModelVersionId"; + } + + @Override + public boolean isIndexed() { + return true; + } +} diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupSubType.java b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupSubType.java new file mode 100644 index 0000000..6823da8 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupSubType.java @@ -0,0 +1,64 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; +import java.util.Optional; + +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.PropertyMigrator; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; + +import org.janusgraph.core.Cardinality; +import org.onap.aai.setup.SchemaVersions; + +@MigrationPriority(20) +@MigrationDangerRating(2) +@Enabled +public class MigrateInstanceGroupSubType extends PropertyMigrator{ + + protected static final String SUB_TYPE_PROPERTY = "sub-type"; + protected static final String INSTANCE_GROUP_ROLE_PROPERTY = "instance-group-role"; + protected static final String INSTANCE_GROUP_NODE_TYPE = "instance-group"; + + public MigrateInstanceGroupSubType(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.initialize(SUB_TYPE_PROPERTY , INSTANCE_GROUP_ROLE_PROPERTY, String.class,Cardinality.SINGLE); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{INSTANCE_GROUP_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateInstanceGroupSubType"; + } + + @Override + public boolean isIndexed() { + return true; + } + +} diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupType.java b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupType.java new file mode 100644 index 0000000..f3cd669 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupType.java @@ -0,0 +1,64 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; +import java.util.Optional; + +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.PropertyMigrator; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; + +import org.janusgraph.core.Cardinality; +import org.onap.aai.setup.SchemaVersions; + +@MigrationPriority(20) +@MigrationDangerRating(2) +@Enabled +public class MigrateInstanceGroupType extends PropertyMigrator{ + + protected static final String TYPE_PROPERTY = "type"; + protected static final String INSTANCE_GROUP_TYPE_PROPERTY = "instance-group-type"; + protected static final String INSTANCE_GROUP_NODE_TYPE = "instance-group"; + + public MigrateInstanceGroupType(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.initialize(TYPE_PROPERTY , INSTANCE_GROUP_TYPE_PROPERTY, String.class,Cardinality.SINGLE); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{INSTANCE_GROUP_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateInstanceGroupType"; + } + + @Override + public boolean isIndexed() { + return true; + } + +} diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateModelVer.java b/src/main/java/org/onap/aai/migration/v13/MigrateModelVer.java new file mode 100644 index 0000000..7bc9a7d --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateModelVer.java @@ -0,0 +1,229 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +@MigrationPriority(20) +@MigrationDangerRating(2) +@Enabled +public class MigrateModelVer extends Migrator{ + + protected static final String MODELINVARIANTID = "model-invariant-id"; + protected static final String MODELVERSIONID = "model-version-id"; + protected static final String MODELINVARIANTIDLOCAL = "model-invariant-id-local"; + protected static final String MODELVERSIONIDLOCAL = "model-version-id-local"; + + protected static final String MODELVER = "model-ver"; + protected static final String MODEL = "model"; + + protected static final String CONNECTOR_NODETYPE = "connector"; + protected static final String SERVICEINSTANCE_NODETYPE = "service-instance"; + protected static final String CONFIGURATION_NODETYPE = "configuration"; + protected static final String LOGICALLINK_NODETYPE = "logical-link"; + protected static final String VNFC_NODETYPE = "vnfc"; + protected static final String L3NETWORK_NODETYPE = "l3-network"; + protected static final String GENERICVNF_NODETYPE = "generic-vnf"; + protected static final String PNF_NODETYPE = "pnf"; + protected static final String VFMODULE_NODETYPE = "vf-module"; + protected static final String INSTANCEGROUP_NODETYPE = "instance-group"; + protected static final String ALLOTTEDRESOURCE_NODETYPE = "allotted-resource"; + protected static final String COLLECTION_NODETYPE = "collection"; + + private boolean success = true; + + private static Map<String, String> NODETYPEKEYMAP = new HashMap<String, String>(); + + static { + NODETYPEKEYMAP.put(CONNECTOR_NODETYPE,"resource-instance-id"); + NODETYPEKEYMAP.put(SERVICEINSTANCE_NODETYPE,"service-instance-id"); + NODETYPEKEYMAP.put(CONFIGURATION_NODETYPE, "configuration-id"); + NODETYPEKEYMAP.put(LOGICALLINK_NODETYPE,"link-name"); + NODETYPEKEYMAP.put(VNFC_NODETYPE, "vnfc-name"); + NODETYPEKEYMAP.put(L3NETWORK_NODETYPE, "network-id"); + NODETYPEKEYMAP.put(GENERICVNF_NODETYPE,"vnf-id"); + NODETYPEKEYMAP.put(PNF_NODETYPE,"pnf-name"); + NODETYPEKEYMAP.put(VFMODULE_NODETYPE,"vf-module-id"); + NODETYPEKEYMAP.put(INSTANCEGROUP_NODETYPE,"id"); + NODETYPEKEYMAP.put(ALLOTTEDRESOURCE_NODETYPE,"id"); + NODETYPEKEYMAP.put(COLLECTION_NODETYPE,"collection-id"); + } + + public MigrateModelVer(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + public void run() { + + List<Vertex> vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, CONNECTOR_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, CONNECTOR_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, SERVICEINSTANCE_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, SERVICEINSTANCE_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, CONFIGURATION_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, CONFIGURATION_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, LOGICALLINK_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, LOGICALLINK_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VNFC_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, VNFC_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, L3NETWORK_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, L3NETWORK_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, GENERICVNF_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, GENERICVNF_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, PNF_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, PNF_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VFMODULE_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, VFMODULE_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, INSTANCEGROUP_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, INSTANCEGROUP_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, ALLOTTEDRESOURCE_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, ALLOTTEDRESOURCE_NODETYPE); + + vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, COLLECTION_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList(); + createEdges(vertextList, COLLECTION_NODETYPE); + } + + private void createEdges(List<Vertex> sourceVertexList, String nodeTypeString) + { + int modelVerEdgeCount = 0; + int modelVerEdgeErrorCount = 0; + + logger.info("---------- Start Creating an Edge for " + nodeTypeString + " nodes with Model Invariant Id and Model Version Id to the model-ver ----------"); + Map<String, Vertex> modelVerUriVtxIdMap = new HashMap<String, Vertex>(); + for (Vertex vertex : sourceVertexList) { + String currentValueModelVersionID = null; + String currrentValueModelInvariantID = null; + try { + currentValueModelVersionID = getModelVersionIdNodeValue(vertex); + currrentValueModelInvariantID = getModelInvariantIdNodeValue(vertex); + + String uri = String.format("/service-design-and-creation/models/model/%s/model-vers/model-ver/%s", currrentValueModelInvariantID, currentValueModelVersionID); + String propertyKey = NODETYPEKEYMAP.get(nodeTypeString); + String propertyValue = vertex.value(propertyKey).toString(); + logger.info("Processing "+nodeTypeString+ " vertex with key "+vertex.value(propertyKey).toString()); + Vertex modelVerVertex = null; + + if (modelVerUriVtxIdMap.containsKey(uri)){ + modelVerVertex = modelVerUriVtxIdMap.get(uri); + } else { + List<Vertex> modelverList = this.engine.asAdmin().getTraversalSource().V().has(MODELINVARIANTID,currrentValueModelInvariantID).has(AAIProperties.NODE_TYPE, MODEL).in() + .has(AAIProperties.NODE_TYPE, "model-ver" ).has("aai-uri", uri).toList(); + if (modelverList != null && !modelverList.isEmpty()) { + modelVerVertex = modelverList.get(0); + modelVerUriVtxIdMap.put(uri, modelVerVertex); + } + } + + if (modelVerVertex != null && modelVerVertex.property("model-version-id").isPresent() ) { + boolean edgePresent = false; + //Check if edge already exists for each of the source vertex + List<Vertex> outVertexList = this.engine.asAdmin().getTraversalSource().V(modelVerVertex).in().has("aai-node-type", nodeTypeString).has(propertyKey, propertyValue).toList(); + Iterator<Vertex> vertexItr = outVertexList.iterator(); + if (outVertexList != null && !outVertexList.isEmpty() && vertexItr.hasNext()){ + logger.info("\t Edge already exists from " + nodeTypeString + " node to models-ver with model-invariant-id :" + currrentValueModelInvariantID + " and model-version-id :" + currentValueModelVersionID); + edgePresent = true; + continue; + } + // Build edge from vertex to modelVerVertex + if (!edgePresent) { + this.createPrivateEdge(vertex, modelVerVertex); + modelVerEdgeCount++; + } + } else + { + modelVerEdgeErrorCount++; + logger.info("\t" + MIGRATION_ERROR + "Unable to create edge. No model-ver vertex found with model-invariant-id :" + currrentValueModelInvariantID + " and model-version-id :" + currentValueModelVersionID); + + } + } catch (Exception e) { + success = false; + modelVerEdgeErrorCount++; + logger.error("\t" + MIGRATION_ERROR + "encountered exception from " + nodeTypeString + " node when trying to create edge to models-ver with model-invariant-id :" + currrentValueModelInvariantID + " and model-version-id :" + currentValueModelVersionID, e); + } + } + + logger.info ("\n \n ******* Summary " + nodeTypeString + " Nodes: Finished creating an Edge for " + nodeTypeString + " nodes with Model Invariant Id and Model Version Id to the model-ver Migration ********* \n"); + logger.info(MIGRATION_SUMMARY_COUNT+"Number of ModelVer edge created from " + nodeTypeString + " nodes: " + modelVerEdgeCount +"\n"); + logger.info(MIGRATION_SUMMARY_COUNT+"Number of ModelVer edge failed to create the edge from the " + nodeTypeString + " nodes due to error : "+ modelVerEdgeErrorCount +"\n"); + + + } + private String getModelInvariantIdNodeValue(Vertex vertex) { + String propertyValue = ""; + if(vertex != null && vertex.property(MODELINVARIANTIDLOCAL).isPresent()){ + propertyValue = vertex.value(MODELINVARIANTIDLOCAL).toString(); + } + return propertyValue; + } + + private String getModelVersionIdNodeValue(Vertex vertex) { + String propertyValue = ""; + if(vertex != null && vertex.property(MODELVERSIONIDLOCAL).isPresent()){ + propertyValue = vertex.value(MODELVERSIONIDLOCAL).toString(); + } + return propertyValue; + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{MODELVER}); + } + + @Override + public String getMigrationName() { + return "MigrateModelVer"; + } + +} diff --git a/src/main/java/org/onap/aai/migration/v13/MigratePserverAndPnfEquipType.java b/src/main/java/org/onap/aai/migration/v13/MigratePserverAndPnfEquipType.java new file mode 100644 index 0000000..6788d7f --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigratePserverAndPnfEquipType.java @@ -0,0 +1,157 @@ +/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Migrator;
+import org.onap.aai.migration.Status;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+@Enabled
+public class MigratePserverAndPnfEquipType extends Migrator{
+
+ protected static final String EQUIP_TYPE_PROPERTY = "equip-type";
+ protected static final String HOSTNAME_PROPERTY = "hostname";
+ protected static final String PNF_NAME_PROPERTY = "pnf-name";
+ protected static final String PNF_NODE_TYPE = "pnf";
+ protected static final String PSERVER_NODE_TYPE = "pserver";
+ private boolean success = true;
+
+ public MigratePserverAndPnfEquipType(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+
+
+ @Override
+ public void run() {
+ int pserverCount = 0;
+ int pnfCount = 0;
+ int pserverErrorCount = 0;
+ int pnfErrorCount = 0;
+ logger.info("---------- Start Updating equip-type for Pserver and Pnf ----------");
+
+ List<Vertex> pserverList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, PSERVER_NODE_TYPE).toList();
+ List<Vertex> pnfList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE).toList();
+
+ for (Vertex vertex : pserverList) {
+ String currentValueOfEquipType = null;
+ String hostName = null;
+ try {
+ currentValueOfEquipType = getEquipTypeNodeValue(vertex);
+ hostName = getHostNameNodeValue(vertex);
+ if("Server".equals(currentValueOfEquipType) ||"server".equals(currentValueOfEquipType) ){
+ vertex.property(EQUIP_TYPE_PROPERTY, "SERVER");
+ this.touchVertexProperties(vertex, false);
+ logger.info("changed Pserver equip-type from " + currentValueOfEquipType + " to SERVER having hostname : " + hostName);
+ pserverCount++;
+ }
+ } catch (Exception e) {
+ success = false;
+ pserverErrorCount++;
+ logger.error(MIGRATION_ERROR + "encountered exception for equip-type:" + currentValueOfEquipType + " having hostName :" + hostName, e);
+ }
+ }
+
+ for (Vertex vertex : pnfList) {
+ String currentValueOfEquipType = null;
+ String pnfName = null;
+ try {
+ currentValueOfEquipType = getEquipTypeNodeValue(vertex);
+ pnfName = getPnfNameNodeValue(vertex);
+ if("Switch".equals(currentValueOfEquipType)||"switch".equals(currentValueOfEquipType)){
+ vertex.property(EQUIP_TYPE_PROPERTY, "SWITCH");
+ this.touchVertexProperties(vertex, false);
+ logger.info("changed Pnf equip-type from "+ currentValueOfEquipType +" to SWITCH having pnf-name :" + pnfName);
+ pnfCount++;
+ }
+
+ } catch (Exception e) {
+ success = false;
+ pnfErrorCount++;
+ logger.error(MIGRATION_ERROR + "encountered exception for equip-type:" + currentValueOfEquipType +" having pnf-name : "+ pnfName , e);
+ }
+ }
+
+ logger.info ("\n \n ******* Final Summary Updated equip-type for Pserver and Pnf Migration ********* \n");
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of Pservers updated: "+ pserverCount +"\n");
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of Pservers failed to update due to error : "+ pserverErrorCount +"\n");
+
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of Pnf updated: "+ pnfCount +"\n");
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of Pnf failed to update due to error : "+ pnfErrorCount +"\n");
+
+ }
+
+ private String getEquipTypeNodeValue(Vertex vertex) {
+ String propertyValue = "";
+ if(vertex != null && vertex.property(EQUIP_TYPE_PROPERTY).isPresent()){
+ propertyValue = vertex.property(EQUIP_TYPE_PROPERTY).value().toString();
+ }
+ return propertyValue;
+ }
+
+ private String getHostNameNodeValue(Vertex vertex) {
+ String propertyValue = "";
+ if(vertex != null && vertex.property(HOSTNAME_PROPERTY).isPresent()){
+ propertyValue = vertex.property(HOSTNAME_PROPERTY).value().toString();
+ }
+ return propertyValue;
+ }
+
+ private String getPnfNameNodeValue(Vertex vertex) {
+ String propertyValue = "";
+ if(vertex != null && vertex.property(PNF_NAME_PROPERTY).isPresent()){
+ propertyValue = vertex.property(PNF_NAME_PROPERTY).value().toString();
+ }
+ return propertyValue;
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{PSERVER_NODE_TYPE,PNF_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigratePserverAndPnfEquipType";
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantId.java b/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantId.java new file mode 100644 index 0000000..a643842 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantId.java @@ -0,0 +1,84 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import org.janusgraph.core.Cardinality; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.PropertyMigrator; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +import java.util.Optional; + + +@MigrationPriority(19) +@MigrationDangerRating(2) +@Enabled +public class MigrateVnfcModelInvariantId extends PropertyMigrator { + + private static final String VNFC_NODE_TYPE = "vnfc"; + private static final String VNFC_MODEL_INVARIANT_ID_PROPERTY = "model-invariant-id"; + private static final String VNFC_MODEL_INVARIANT_ID_LOCAL_PROPERTY = "model-invariant-id-local"; + + public MigrateVnfcModelInvariantId(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.initialize(VNFC_MODEL_INVARIANT_ID_PROPERTY, VNFC_MODEL_INVARIANT_ID_LOCAL_PROPERTY, String.class, Cardinality.SINGLE); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{this.VNFC_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateVnfcModelInvariantId"; + } + + @Override + public boolean isIndexed() { + return true; + } +} diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionId.java b/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionId.java new file mode 100644 index 0000000..13cdb80 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionId.java @@ -0,0 +1,83 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import java.util.Optional; +import org.janusgraph.core.Cardinality; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.PropertyMigrator; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(19) +@MigrationDangerRating(2) +@Enabled +public class MigrateVnfcModelVersionId extends PropertyMigrator { + + private static final String VNFC_NODE_TYPE = "vnfc"; + private static final String VNFC_MODEL_VERSION_ID_PROPERTY = "model-version-id"; + private static final String VNFC_MODEL_VERSION_ID_LOCAL_PROPERTY = "model-version-id-local"; + + public MigrateVnfcModelVersionId(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.initialize(VNFC_MODEL_VERSION_ID_PROPERTY, VNFC_MODEL_VERSION_ID_LOCAL_PROPERTY, String.class, Cardinality.SINGLE); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{this.VNFC_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateVnfcModelVersionId"; + } + + @Override + public boolean isIndexed() { + return true; + } +} |