diff options
Diffstat (limited to 'src/main/java/org/onap/aai/migration')
27 files changed, 8090 insertions, 716 deletions
diff --git a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java index 616ff02..b3faec8 100644 --- a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java +++ b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java @@ -1,288 +1,339 @@ -/**
- * ============LICENSE_START=======================================================
- * org.onap.aai
- * ================================================================================
- * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END=========================================================
- */
-package org.onap.aai.migration;
-
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Property;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.javatuples.Pair;
-import org.onap.aai.db.props.AAIProperties;
-import org.onap.aai.edges.EdgeIngestor;
-import org.onap.aai.introspection.LoaderFactory;
-import org.onap.aai.serialization.db.EdgeSerializer;
-import org.onap.aai.serialization.engines.TransactionalGraphEngine;
-import org.onap.aai.setup.SchemaVersions;
-
-/**
- * A migration template for "swinging" edges that terminate on an old-node to a new target node.
- * That is, given an oldNode and a newNode we will swing edges that terminate on the
- * oldNode and terminate them on the newNode (actually we drop the old edges and add new ones).
- *
- *
- * We allow the passing of some parameters to restrict what edges get swung over:
- * > otherEndNodeTypeRestriction: only swing edges that terminate on the oldNode if the
- * node at the other end of the edge is of this nodeType.
- * > edgeLabelRestriction: Only swing edges that have this edgeLabel
- * > edgeDirectionRestriction: Only swing edges that go this direction (from the oldNode)
- * this is a required parameter. valid values are: BOTH, IN, OUT
- *
- */
-@MigrationPriority(0)
-@MigrationDangerRating(1)
-public abstract class EdgeSwingMigrator extends Migrator {
-
- private boolean success = true;
- private String nodeTypeRestriction = null;
- private String edgeLabelRestriction = null;
- private String edgeDirRestriction = null;
- private List<Pair<Vertex, Vertex>> nodePairList;
-
-
- public EdgeSwingMigrator(TransactionalGraphEngine engine , LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
- super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
- }
-
-
- /**
- * Do not override this method as an inheritor of this class
- */
- @Override
- public void run() {
- executeModifyOperation();
- cleanupAsAppropriate(this.nodePairList);
- }
-
- /**
- * This is where inheritors should add their logic
- */
- protected void executeModifyOperation() {
-
- try {
- this.nodeTypeRestriction = this.getNodeTypeRestriction();
- this.edgeLabelRestriction = this.getEdgeLabelRestriction();
- this.edgeDirRestriction = this.getEdgeDirRestriction();
- nodePairList = this.getAffectedNodePairs();
- for (Pair<Vertex, Vertex> nodePair : nodePairList) {
- Vertex fromNode = nodePair.getValue0();
- Vertex toNode = nodePair.getValue1();
- this.swingEdges(fromNode, toNode,
- this.nodeTypeRestriction,this.edgeLabelRestriction,this.edgeDirRestriction);
- }
- } catch (Exception e) {
- logger.error("error encountered", e);
- success = false;
- }
- }
-
-
- protected void swingEdges(Vertex oldNode, Vertex newNode, String nodeTypeRestr, String edgeLabelRestr, String edgeDirRestr) {
- try {
- // If the old and new Vertices aren't populated, throw an exception
- if( oldNode == null ){
- logger.info ( "null oldNode passed to swingEdges() ");
- success = false;
- return;
- }
- else if( newNode == null ){
- logger.info ( "null newNode passed to swingEdges() ");
- success = false;
- return;
- }
- else if( edgeDirRestr == null ||
- (!edgeDirRestr.equals("BOTH")
- && !edgeDirRestr.equals("IN")
- && !edgeDirRestr.equals("OUT") )
- ){
- logger.info ( "invalid direction passed to swingEdges(). valid values are BOTH/IN/OUT ");
- success = false;
- return;
- }
- else if( edgeLabelRestr != null
- && (edgeLabelRestr.trim().equals("none") || edgeLabelRestr.trim().equals("")) ){
- edgeLabelRestr = null;
- }
- else if( nodeTypeRestr == null || nodeTypeRestr.trim().equals("") ){
- nodeTypeRestr = "none";
- }
-
- String oldNodeType = oldNode.value(AAIProperties.NODE_TYPE);
- String oldUri = oldNode.<String> property("aai-uri").isPresent() ? oldNode.<String> property("aai-uri").value() : "URI Not present";
-
- String newNodeType = newNode.value(AAIProperties.NODE_TYPE);
- String newUri = newNode.<String> property("aai-uri").isPresent() ? newNode.<String> property("aai-uri").value() : "URI Not present";
-
- // If the nodeTypes don't match, throw an error
- if( !oldNodeType.equals(newNodeType) ){
- logger.info ( "Can not swing edge from a [" + oldNodeType + "] node to a [" +
- newNodeType + "] node. ");
- success = false;
- return;
- }
-
- // Find and migrate any applicable OUT edges.
- if( edgeDirRestr.equals("BOTH") || edgeDirRestr.equals("OUT") ){
- Iterator <Edge> edgeOutIter = null;
- if( edgeLabelRestr == null ) {
- edgeOutIter = oldNode.edges(Direction.OUT);
- }
- else {
- edgeOutIter = oldNode.edges(Direction.OUT, edgeLabelRestr);
- }
-
- while( edgeOutIter.hasNext() ){
- Edge oldOutE = edgeOutIter.next();
- String eLabel = oldOutE.label();
- Vertex otherSideNode4ThisEdge = oldOutE.inVertex();
- String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
- if( nodeTypeRestr.equals("none") || nodeTypeRestr.toLowerCase().equals(otherSideNodeType) ){
- Iterator <Property<Object>> propsIter = oldOutE.properties();
- HashMap<String, String> propMap = new HashMap<String,String>();
- while( propsIter.hasNext() ){
- Property <Object> ep = propsIter.next();
- propMap.put(ep.key(), ep.value().toString());
- }
-
- String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present";
- logger.info ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is ["
- + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType
- + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. ");
- // remove the old edge
- oldOutE.remove();
-
- // add the new edge with properties that match the edge that was deleted. We don't want to
- // change any edge properties - just swinging one end of the edge to a new node.
- // NOTE - addEdge adds an OUT edge to the vertex passed as a parameter, so we are
- // adding from the newNode side.
- Edge newOutE = newNode.addEdge(eLabel, otherSideNode4ThisEdge);
-
- Iterator it = propMap.entrySet().iterator();
- while (it.hasNext()) {
- Map.Entry pair = (Map.Entry)it.next();
- newOutE.property(pair.getKey().toString(), pair.getValue().toString() );
- }
-
- }
- }
- }
-
- // Find and migrate any applicable IN edges.
- if( edgeDirRestr.equals("BOTH") || edgeDirRestr.equals("IN") ){
- Iterator <Edge> edgeInIter = null;
- if( edgeLabelRestr == null ) {
- edgeInIter = oldNode.edges(Direction.IN);
- }
- else {
- edgeInIter = oldNode.edges(Direction.IN, edgeLabelRestr);
- }
-
- while( edgeInIter.hasNext() ){
- Edge oldInE = edgeInIter.next();
- String eLabel = oldInE.label();
- Vertex otherSideNode4ThisEdge = oldInE.outVertex();
- String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
- if( nodeTypeRestr.equals("none") || nodeTypeRestr.toLowerCase().equals(otherSideNodeType) ){
- Iterator <Property<Object>> propsIter = oldInE.properties();
- HashMap<String, String> propMap = new HashMap<String,String>();
- while( propsIter.hasNext() ){
- Property <Object> ep = propsIter.next();
- propMap.put(ep.key(), ep.value().toString());
- }
-
- String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present";
- logger.info ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is ["
- + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType
- + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. ");
-
- // remove the old edge
- oldInE.remove();
-
- // add the new edge with properties that match the edge that was deleted. We don't want to
- // change any edge properties - just swinging one end of the edge to a new node.
- // NOTE - addEdge adds an OUT edge to the vertex passed as a parameter, so we are
- // adding from the node on the other-end of the original edge so we'll get
- // an IN-edge to the newNode.
- Edge newInE = otherSideNode4ThisEdge.addEdge(eLabel, newNode);
-
- Iterator it = propMap.entrySet().iterator();
- while (it.hasNext()) {
- Map.Entry pair = (Map.Entry)it.next();
- newInE.property(pair.getKey().toString(), pair.getValue().toString() );
- }
- }
- }
- }
-
- } catch (Exception e) {
- logger.error("error encountered", e);
- success = false;
- }
- }
-
- @Override
- public Status getStatus() {
- if (success) {
- return Status.SUCCESS;
- } else {
- return Status.FAILURE;
- }
- }
-
-
- /**
- * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate from json files
- * @return
- */
- public abstract List<Pair<Vertex, Vertex>> getAffectedNodePairs() ;
-
-
- /**
- * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use
- * @return
- */
- public abstract String getNodeTypeRestriction() ;
-
-
- /**
- * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use
- * @return
- */
- public abstract String getEdgeLabelRestriction() ;
-
- /**
- * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use
- * @return
- */
- public abstract String getEdgeDirRestriction() ;
-
-
-
- /**
- * Cleanup (remove) the nodes that edges were moved off of if appropriate
- * @return
- */
- public abstract void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL);
-
+/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration; + + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Property; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.javatuples.Pair; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.enums.EdgeProperty; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +/** + * A migration template for "swinging" edges that terminate on an old-node to a new target node. + * That is, given an oldNode and a newNode we will swing edges that terminate on the + * oldNode and terminate them on the newNode (actually we drop the old edges and add new ones). + * + * + * We allow the passing of some parameters to restrict what edges get swung over: + * > otherEndNodeTypeRestriction: only swing edges that terminate on the oldNode if the + * node at the other end of the edge is of this nodeType. + * > edgeLabelRestriction: Only swing edges that have this edgeLabel + * > edgeDirectionRestriction: Only swing edges that go this direction (from the oldNode) + * this is a required parameter. valid values are: BOTH, IN, OUT + * + */ +@MigrationPriority(0) +@MigrationDangerRating(1) +public abstract class EdgeSwingMigrator extends Migrator { + + private boolean success = true; + private String nodeTypeRestriction = null; + private String edgeLabelRestriction = null; + private String edgeDirRestriction = null; + private List<Pair<Vertex, Vertex>> nodePairList; + + + public EdgeSwingMigrator(TransactionalGraphEngine engine , LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + + /** + * Do not override this method as an inheritor of this class + */ + @Override + public void run() { + executeModifyOperation(); + cleanupAsAppropriate(this.nodePairList); + } + + /** + * This is where inheritors should add their logic + */ + protected void executeModifyOperation() { + + try { + this.nodeTypeRestriction = this.getNodeTypeRestriction(); + this.edgeLabelRestriction = this.getEdgeLabelRestriction(); + this.edgeDirRestriction = this.getEdgeDirRestriction(); + nodePairList = this.getAffectedNodePairs(); + for (Pair<Vertex, Vertex> nodePair : nodePairList) { + Vertex fromNode = nodePair.getValue0(); + Vertex toNode = nodePair.getValue1(); + this.swingEdges(fromNode, toNode, + this.nodeTypeRestriction,this.edgeLabelRestriction,this.edgeDirRestriction); + } + } catch (Exception e) { + logger.error("error encountered", e); + success = false; + } + } + + + protected void swingEdges(Vertex oldNode, Vertex newNode, String nodeTypeRestr, String edgeLabelRestr, String edgeDirRestr) { + try { + // If the old and new Vertices aren't populated, throw an exception + if( oldNode == null ){ + logger.info ( "null oldNode passed to swingEdges() "); + success = false; + return; + } + else if( newNode == null ){ + logger.info ( "null newNode passed to swingEdges() "); + success = false; + return; + } + else if( edgeDirRestr == null || + (!edgeDirRestr.equals("BOTH") + && !edgeDirRestr.equals("IN") + && !edgeDirRestr.equals("OUT") ) + ){ + logger.info ( "invalid direction passed to swingEdges(). valid values are BOTH/IN/OUT "); + success = false; + return; + } + else if( edgeLabelRestr != null + && (edgeLabelRestr.trim().equals("none") || edgeLabelRestr.trim().equals("")) ){ + edgeLabelRestr = null; + } + else if( nodeTypeRestr == null || nodeTypeRestr.trim().equals("") ){ + nodeTypeRestr = "none"; + } + + String oldNodeType = oldNode.value(AAIProperties.NODE_TYPE); + String oldUri = oldNode.<String> property("aai-uri").isPresent() ? oldNode.<String> property("aai-uri").value() : "URI Not present"; + + String newNodeType = newNode.value(AAIProperties.NODE_TYPE); + String newUri = newNode.<String> property("aai-uri").isPresent() ? newNode.<String> property("aai-uri").value() : "URI Not present"; + + // If the nodeTypes don't match, throw an error + if( !oldNodeType.equals(newNodeType) ){ + logger.info ( "Can not swing edge from a [" + oldNodeType + "] node to a [" + + newNodeType + "] node. "); + success = false; + return; + } + + // Find and migrate any applicable OUT edges. + if( edgeDirRestr.equals("BOTH") || edgeDirRestr.equals("OUT") ){ + Iterator <Edge> edgeOutIter = null; + Iterator <Edge> newNodeEdgeOutIter = null; + + if( edgeLabelRestr == null ) { + edgeOutIter = oldNode.edges(Direction.OUT); + newNodeEdgeOutIter = newNode.edges(Direction.OUT); + + } + else { + edgeOutIter = oldNode.edges(Direction.OUT, edgeLabelRestr); + newNodeEdgeOutIter = newNode.edges(Direction.OUT, edgeLabelRestr); + } + + List<Vertex> newNodeOtherEndVertexList = new ArrayList<Vertex>(); + while (newNodeEdgeOutIter.hasNext()){ + Edge newNodeOutE = newNodeEdgeOutIter.next(); + Vertex otherSideNode4ThisEdgeOfNewNode = newNodeOutE.inVertex(); + newNodeOtherEndVertexList.add(otherSideNode4ThisEdgeOfNewNode); + } + + while( edgeOutIter.hasNext() ){ + Edge oldOutE = edgeOutIter.next(); + String eLabel = oldOutE.label(); + Vertex otherSideNode4ThisEdge = oldOutE.inVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + if( nodeTypeRestr.equals("none") || nodeTypeRestr.toLowerCase().equals(otherSideNodeType) ){ + Iterator <Property<Object>> propsIter = oldOutE.properties(); + HashMap<String, String> propMap = new HashMap<String,String>(); + while( propsIter.hasNext() ){ + Property <Object> ep = propsIter.next(); + propMap.put(ep.key(), ep.value().toString()); + } + + String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present"; + logger.info ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is [" + + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType + + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. "); + // remove the old edge + oldOutE.remove(); + + // add the new edge with properties that match the edge that was deleted. We don't want to + // change any edge properties - just swinging one end of the edge to a new node. + // NOTE - addEdge adds an OUT edge to the vertex passed as a parameter, so we are + // adding from the newNode side. + + EdgeType edgeType = getEdgeType(propMap); + if (edgeType != null && !newNodeOtherEndVertexList.contains(otherSideNode4ThisEdge)){ +// Edge newOutE = newNode.addEdge(eLabel, otherSideNode4ThisEdge); + Edge newOutE = createEdgeIfPossible(edgeType, newNode, otherSideNode4ThisEdge); + if (newOutE != null){ + Iterator it = propMap.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry)it.next(); + newOutE.property(pair.getKey().toString(), pair.getValue().toString() ); + } + }else { + logger.info("\n Edge was not swung due to Multiplicity Rule Violation..."); + } + } + } + } + } + + // Find and migrate any applicable IN edges. + if( edgeDirRestr.equals("BOTH") || edgeDirRestr.equals("IN") ){ + Iterator <Edge> edgeInIter = null; + Iterator <Edge> newNodeEdgeOutIter = null; + if( edgeLabelRestr == null ) { + edgeInIter = oldNode.edges(Direction.IN); + newNodeEdgeOutIter = newNode.edges(Direction.IN); + } + else { + edgeInIter = oldNode.edges(Direction.IN, edgeLabelRestr); + newNodeEdgeOutIter = newNode.edges(Direction.IN, edgeLabelRestr); + } + + List<Vertex> newNodeOtherEndVertexList = new ArrayList<Vertex>(); + while (newNodeEdgeOutIter.hasNext()){ + Edge newNodeOutE = newNodeEdgeOutIter.next(); + Vertex otherSideNode4ThisEdgeOfNewNode = newNodeOutE.outVertex(); + newNodeOtherEndVertexList.add(otherSideNode4ThisEdgeOfNewNode); + } + + while( edgeInIter.hasNext() ){ + Edge oldInE = edgeInIter.next(); + String eLabel = oldInE.label(); + Vertex otherSideNode4ThisEdge = oldInE.outVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + if( nodeTypeRestr.equals("none") || nodeTypeRestr.toLowerCase().equals(otherSideNodeType) ){ + Iterator <Property<Object>> propsIter = oldInE.properties(); + HashMap<String, String> propMap = new HashMap<String,String>(); + while( propsIter.hasNext() ){ + Property <Object> ep = propsIter.next(); + propMap.put(ep.key(), ep.value().toString()); + } + + String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present"; + logger.info ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is [" + + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType + + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. "); + + // remove the old edge + oldInE.remove(); + + // add the new edge with properties that match the edge that was deleted. We don't want to + // change any edge properties - just swinging one end of the edge to a new node. + // NOTE - addEdge adds an OUT edge to the vertex passed as a parameter, so we are + // adding from the node on the other-end of the original edge so we'll get + // an IN-edge to the newNode. + EdgeType edgeType = getEdgeType(propMap); + if (edgeType != null && !newNodeOtherEndVertexList.contains(otherSideNode4ThisEdge)){ +// Edge newInE = otherSideNode4ThisEdge.addEdge(eLabel, newNode); + Edge newInE = createEdgeIfPossible(edgeType, otherSideNode4ThisEdge, newNode); + if (newInE != null){ + Iterator it = propMap.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry)it.next(); + newInE.property(pair.getKey().toString(), pair.getValue().toString() ); + } + } else { + logger.info("\t Edge was not swung due to Multiplicity Rule Violation..."); + } + } + } + } + } + + } catch (Exception e) { + logger.error("error encountered", e); + success = false; + } + } + + private EdgeType getEdgeType(HashMap edgePropMap) { + EdgeType type = null; + String containsValue = edgePropMap.get(EdgeProperty.CONTAINS.toString()).toString(); + if ("NONE".equalsIgnoreCase(containsValue)){ + type = EdgeType.COUSIN; + } else { + type = EdgeType.TREE; + } + return type; + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + + /** + * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate from json files + * @return + */ + public abstract List<Pair<Vertex, Vertex>> getAffectedNodePairs() ; + + + /** + * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use + * @return + */ + public abstract String getNodeTypeRestriction() ; + + + /** + * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use + * @return + */ + public abstract String getEdgeLabelRestriction() ; + + /** + * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use + * @return + */ + public abstract String getEdgeDirRestriction() ; + + + + /** + * Cleanup (remove) the nodes that edges were moved off of if appropriate + * @return + */ + public abstract void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL); + }
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/MigrationController.java b/src/main/java/org/onap/aai/migration/MigrationController.java index 0e65745..ecc0434 100644 --- a/src/main/java/org/onap/aai/migration/MigrationController.java +++ b/src/main/java/org/onap/aai/migration/MigrationController.java @@ -19,18 +19,22 @@ */ package org.onap.aai.migration; -import java.util.UUID; - +import org.onap.aai.config.PropertyPasswordConfiguration; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.logging.LoggingContext; import org.onap.aai.logging.LoggingContext.StatusCode; import org.onap.aai.serialization.db.EdgeSerializer; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConstants; +import org.onap.aai.util.ExceptionTranslator; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import java.util.UUID; + /** * Wrapper class to allow {@link org.onap.aai.migration.MigrationControllerInternal MigrationControllerInternal} * to be run from a shell script @@ -43,7 +47,7 @@ public class MigrationController { * @param args * the arguments */ - public static void main(String[] args) { + public static void main(String[] args) throws AAIException { LoggingContext.init(); LoggingContext.partnerName("Migration"); @@ -55,11 +59,23 @@ public class MigrationController { LoggingContext.statusCode(StatusCode.COMPLETE); LoggingContext.responseCode(LoggingContext.SUCCESS); - AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext( - "org.onap.aai.config", - "org.onap.aai.setup" - ); - + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); + PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); + initializer.initialize(ctx); + try { + ctx.scan( + "org.onap.aai.config", + "org.onap.aai.setup" + ); + ctx.refresh(); + } catch (Exception e) { + AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); + System.out.println("Problems running tool "+aai.getMessage()); + LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); + throw aai; + } LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); EdgeIngestor edgeIngestor = ctx.getBean(EdgeIngestor.class); EdgeSerializer edgeSerializer = ctx.getBean(EdgeSerializer.class); diff --git a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java index b113f03..b94460a 100644 --- a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java +++ b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java @@ -20,16 +20,25 @@ package org.onap.aai.migration; -import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.io.IoCore; +import org.onap.aai.datasnapshot.DataSnapshot; import org.onap.aai.db.props.AAIProperties; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.dbmap.DBConnectionType; @@ -38,32 +47,24 @@ import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Loader; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.introspection.ModelType; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.setup.SchemaVersion; import org.onap.aai.logging.LoggingContext; import org.onap.aai.logging.LoggingContext.StatusCode; -import org.onap.aai.serialization.db.EdgeSerializer; -import org.onap.aai.serialization.engines.JanusGraphDBEngine; import org.onap.aai.serialization.engines.QueryStyle; +import org.onap.aai.serialization.engines.JanusGraphDBEngine; import org.onap.aai.serialization.engines.TransactionalGraphEngine; -import org.onap.aai.setup.SchemaVersion; -import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConstants; import org.onap.aai.util.FormatDate; import org.reflections.Reflections; import org.slf4j.MDC; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; - +import com.att.eelf.configuration.Configuration; +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; /** * Runs a series of migrations from a defined directory based on the presence of @@ -73,262 +74,280 @@ import java.util.stream.Collectors; */ public class MigrationControllerInternal { - private EELFLogger logger; - private final int DANGER_ZONE = 10; - public static final String VERTEX_TYPE = "migration-list-1707"; - private final List<String> resultsSummary = new ArrayList<>(); - private final List<NotificationHelper> notifications = new ArrayList<>(); - private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots"; - - private LoaderFactory loaderFactory; - private EdgeIngestor edgeIngestor; - private EdgeSerializer edgeSerializer; - private final SchemaVersions schemaVersions; - - public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){ - this.loaderFactory = loaderFactory; - this.edgeIngestor = edgeIngestor; - this.edgeSerializer = edgeSerializer; - this.schemaVersions = schemaVersions; - } - - /** - * The main method. - * - * @param args - * the arguments - */ - public void run(String[] args) { - // Set the logging file properties to be used by EELFManager - System.setProperty("aai.service.name", MigrationController.class.getSimpleName()); - Properties props = System.getProperties(); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml"); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); - - logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName()); - MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); - - boolean loadSnapshot = false; - - CommandLineArgs cArgs = new CommandLineArgs(); - - JCommander jCommander = new JCommander(cArgs, args); - jCommander.setProgramName(MigrationController.class.getSimpleName()); - - // Set flag to load from snapshot based on the presence of snapshot and - // graph storage backend of inmemory - if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) { - try { - PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config); - if (config.getString("storage.backend").equals("inmemory")) { - loadSnapshot = true; - System.setProperty("load.snapshot.file", "true"); - System.setProperty("snapshot.location", cArgs.dataSnapshot); - } - } catch (ConfigurationException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e)); - return; - } - } - System.setProperty("realtime.db.config", cArgs.config); - logAndPrint("\n\n---------- Connecting to Graph ----------"); - AAIGraph.getInstance(); - - logAndPrint("---------- Connection Established ----------"); - SchemaVersion version = schemaVersions.getDefaultVersion(); - QueryStyle queryStyle = QueryStyle.TRAVERSAL; - ModelType introspectorFactoryType = ModelType.MOXY; - Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); - TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader); - - if (cArgs.help) { - jCommander.usage(); - engine.rollback(); - return; - } - - Reflections reflections = new Reflections("org.onap.aai.migration"); - List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections)); - //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names - if (cArgs.list) { - listMigrationWithStatus(cArgs, migratorClasses, engine); - return; - } - - logAndPrint("---------- Looking for migration scripts to be executed. ----------"); - //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names - if (!cArgs.excludeClasses.isEmpty()) { - migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses); - listMigrationWithStatus(cArgs, migratorClasses, engine); - } - List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses); - - sortList(migratorClassesToRun); - - if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + "."); - logAndPrint("---------- Done ----------"); - LoggingContext.successStatusFields(); - } - - logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts."); - logAndPrint("---------- Executing Migration Scripts ----------"); - - - if (!cArgs.skipPreMigrationSnapShot) { - takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun); - } - - for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { - String name = migratorClass.getSimpleName(); - Migrator migrator; - if (cArgs.runDisabled.contains(name) || migratorClass.isAnnotationPresent(Enabled.class)) {//Check either of enabled annotation or runDisabled flag - - try { - engine.startTransaction(); - if (!cArgs.forced && hasAlreadyRun(name, engine)) { - logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution"); - continue; - } - migrator = migratorClass - .getConstructor( - TransactionalGraphEngine.class, - LoaderFactory.class, - EdgeIngestor.class, - EdgeSerializer.class, - SchemaVersions.class - ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions); - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e)); - LoggingContext.successStatusFields(); - engine.rollback(); - continue; - } - logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script."); - logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs."); - MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName()); - - migrator.run(); - - commitChanges(engine, migrator, cArgs); - } else { - logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled."); - } - } - MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); - for (NotificationHelper notificationHelper : notifications) { - try { - notificationHelper.triggerEvents(); - } catch (AAIException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); - logAndPrint("\tcould not event"); - logger.error("could not event", e); - LoggingContext.successStatusFields(); - } - } - logAndPrint("---------- Done ----------"); - - // Save post migration snapshot if snapshot was loaded - if (!cArgs.skipPostMigrationSnapShot) { - generateSnapshot(engine, "post"); - } - - outputResultsSummary(); - } - - /** - * This method is used to remove excluded classes from migration from the - * script command. - * - * @param excludeClasses - * : Classes to be removed from Migration - * @param migratorClasses - * : Classes to execute migration. - * @return - */ - private List<Class<? extends Migrator>> filterMigrationClasses( - List<String> excludeClasses, - List<Class<? extends Migrator>> migratorClasses) { - - List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses - .stream() - .filter(migratorClass -> !excludeClasses.contains(migratorClass - .getSimpleName())).collect(Collectors.toList()); - - return filteredMigratorClasses; - } - - private void listMigrationWithStatus(CommandLineArgs cArgs, - List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) { - sortList(migratorClasses); - engine.startTransaction(); - System.out.println("---------- List of all migrations ----------"); - migratorClasses.forEach(migratorClass -> { - boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class); - String enabled = enabledAnnotation ? "Enabled" : "Disabled"; - StringBuilder sb = new StringBuilder(); - sb.append(migratorClass.getSimpleName()); - sb.append(" in package "); - sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1)); - sb.append(" is "); - sb.append(enabled); - sb.append(" "); - sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]"); - System.out.println(sb.toString()); - }); - engine.rollback(); - System.out.println("---------- Done ----------"); - } - - private String getDbStatus(String name, TransactionalGraphEngine engine) { - if (hasAlreadyRun(name, engine)) { - return "Already executed in this env"; - } - return "Will be run on next execution if Enabled"; - } - - private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) { - return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext(); - } - private Set<Class<? extends Migrator>> findClasses(Reflections reflections) { - Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class); - /* - * TODO- Change this to make sure only classes in the specific $release are added in the runList - * Or add a annotation like exclude which folks again need to remember to add ?? - */ - - migratorClasses.remove(PropertyMigrator.class); - migratorClasses.remove(EdgeMigrator.class); - return migratorClasses; - } - - - private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) { - - /*int sum = 0; - for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { - if (migratorClass.isAnnotationPresent(Enabled.class)) { - sum += migratorClass.getAnnotation(MigrationPriority.class).value(); - } - } - - if (sum >= DANGER_ZONE) { - - logAndPrint("Entered Danger Zone. Taking snapshot."); - }*/ - - //always take snapshot for now - - generateSnapshot(engine, "pre"); - - } + private EELFLogger logger; + private final int DANGER_ZONE = 10; + public static final String VERTEX_TYPE = "migration-list-1707"; + private final List<String> resultsSummary = new ArrayList<>(); + private final List<NotificationHelper> notifications = new ArrayList<>(); + private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots"; + + private LoaderFactory loaderFactory; + private EdgeIngestor edgeIngestor; + private EdgeSerializer edgeSerializer; + private final SchemaVersions schemaVersions; + + public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){ + this.loaderFactory = loaderFactory; + this.edgeIngestor = edgeIngestor; + this.edgeSerializer = edgeSerializer; + this.schemaVersions = schemaVersions; + + } + + /** + * The main method. + * + * @param args + * the arguments + */ + public void run(String[] args) { + // Set the logging file properties to be used by EELFManager + System.setProperty("aai.service.name", MigrationController.class.getSimpleName()); + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml"); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); + + logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName()); + MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + + boolean loadSnapshot = false; + + CommandLineArgs cArgs = new CommandLineArgs(); + + JCommander jCommander = new JCommander(cArgs, args); + jCommander.setProgramName(MigrationController.class.getSimpleName()); + + // Set flag to load from snapshot based on the presence of snapshot and + // graph storage backend of inmemory + if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) { + try { + PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config); + if (config.getString("storage.backend").equals("inmemory")) { + loadSnapshot = true; +// System.setProperty("load.snapshot.file", "true"); + System.setProperty("snapshot.location", cArgs.dataSnapshot); + String snapshotLocation =cArgs.dataSnapshot; + String snapshotDir; + String snapshotFile; + int index = snapshotLocation.lastIndexOf("\\"); + if (index == -1){ + //Use default directory path + snapshotDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "snapshots"; + snapshotFile = snapshotLocation; + } else { + snapshotDir = snapshotLocation.substring(0, index+1); + snapshotFile = snapshotLocation.substring(index+1, snapshotLocation.length()) ; + } + String [] dataSnapShotArgs = {"-c","MULTITHREAD_RELOAD","-f", snapshotFile, "-oldFileDir",snapshotDir, "-caller","migration"}; + DataSnapshot dataSnapshot = new DataSnapshot(); + dataSnapshot.executeCommand(dataSnapShotArgs, true, false, null, "MULTITHREAD_RELOAD", snapshotFile); + } + } catch (ConfigurationException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e)); + return; + } + } + else { + System.setProperty("realtime.db.config", cArgs.config); + logAndPrint("\n\n---------- Connecting to Graph ----------"); + AAIGraph.getInstance(); + } + + logAndPrint("---------- Connection Established ----------"); + SchemaVersion version = schemaVersions.getDefaultVersion(); + QueryStyle queryStyle = QueryStyle.TRAVERSAL; + ModelType introspectorFactoryType = ModelType.MOXY; + Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); + TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader); + + if (cArgs.help) { + jCommander.usage(); + engine.rollback(); + return; + } + + Reflections reflections = new Reflections("org.onap.aai.migration"); + List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections)); + //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names + if (cArgs.list) { + listMigrationWithStatus(cArgs, migratorClasses, engine); + return; + } + + logAndPrint("---------- Looking for migration scripts to be executed. ----------"); + //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names + if (!cArgs.excludeClasses.isEmpty()) { + migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses); + listMigrationWithStatus(cArgs, migratorClasses, engine); + } + List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses); + + sortList(migratorClassesToRun); + + if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); + logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + "."); + logAndPrint("---------- Done ----------"); + LoggingContext.successStatusFields(); + } + + logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts."); + logAndPrint("---------- Executing Migration Scripts ----------"); + + + if (!cArgs.skipPreMigrationSnapShot) { + takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun); + } + + for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { + String name = migratorClass.getSimpleName(); + Migrator migrator; + if (cArgs.runDisabled.contains(name) || migratorClass.isAnnotationPresent(Enabled.class)) { + + try { + engine.startTransaction(); + if (!cArgs.forced && hasAlreadyRun(name, engine)) { + logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution"); + continue; + } + migrator = migratorClass + .getConstructor( + TransactionalGraphEngine.class, + LoaderFactory.class, + EdgeIngestor.class, + EdgeSerializer.class, + SchemaVersions.class + ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e)); + LoggingContext.successStatusFields(); + engine.rollback(); + continue; + } + logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script."); + logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs."); + MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName()); + + migrator.run(); + + commitChanges(engine, migrator, cArgs); + } else { + logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled."); + } + } + MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + for (NotificationHelper notificationHelper : notifications) { + try { + notificationHelper.triggerEvents(); + } catch (AAIException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); + logAndPrint("\tcould not event"); + logger.error("could not event", e); + LoggingContext.successStatusFields(); + } + } + logAndPrint("---------- Done ----------"); + + // Save post migration snapshot if snapshot was loaded + if (!cArgs.skipPostMigrationSnapShot) { + generateSnapshot(engine, "post"); + } + + outputResultsSummary(); + } + + /** + * This method is used to remove excluded classes from migration from the + * script command. + * + * @param excludeClasses + * : Classes to be removed from Migration + * @param migratorClasses + * : Classes to execute migration. + * @return + */ + private List<Class<? extends Migrator>> filterMigrationClasses( + List<String> excludeClasses, + List<Class<? extends Migrator>> migratorClasses) { + + List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses + .stream() + .filter(migratorClass -> !excludeClasses.contains(migratorClass + .getSimpleName())).collect(Collectors.toList()); + + return filteredMigratorClasses; + } + + private void listMigrationWithStatus(CommandLineArgs cArgs, + List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) { + sortList(migratorClasses); + engine.startTransaction(); + System.out.println("---------- List of all migrations ----------"); + migratorClasses.forEach(migratorClass -> { + boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class); + String enabled = enabledAnnotation ? "Enabled" : "Disabled"; + StringBuilder sb = new StringBuilder(); + sb.append(migratorClass.getSimpleName()); + sb.append(" in package "); + sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1)); + sb.append(" is "); + sb.append(enabled); + sb.append(" "); + sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]"); + System.out.println(sb.toString()); + }); + engine.rollback(); + System.out.println("---------- Done ----------"); + } + + private String getDbStatus(String name, TransactionalGraphEngine engine) { + if (hasAlreadyRun(name, engine)) { + return "Already executed in this env"; + } + return "Will be run on next execution if Enabled"; + } + + private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) { + return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext(); + } + private Set<Class<? extends Migrator>> findClasses(Reflections reflections) { + Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class); + /* + * TODO- Change this to make sure only classes in the specific $release are added in the runList + * Or add a annotation like exclude which folks again need to remember to add ?? + */ + + migratorClasses.remove(PropertyMigrator.class); + migratorClasses.remove(EdgeMigrator.class); + return migratorClasses; + } + + + private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) { + + /*int sum = 0; + for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { + if (migratorClass.isAnnotationPresent(Enabled.class)) { + sum += migratorClass.getAnnotation(MigrationPriority.class).value(); + } + } + + if (sum >= DANGER_ZONE) { + + logAndPrint("Entered Danger Zone. Taking snapshot."); + }*/ + + //always take snapshot for now + + generateSnapshot(engine, "pre"); + + } private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs, @@ -345,6 +364,7 @@ public class MigrationControllerInternal { } return migratorClassesToRun; } + private boolean migratorExplicitlySpecified(CommandLineArgs cArgs, String migratorName){ return !cArgs.scripts.isEmpty() && cArgs.scripts.contains(migratorName); } @@ -352,122 +372,125 @@ public class MigrationControllerInternal { return !cArgs.runDisabled.isEmpty() && cArgs.runDisabled.contains(migratorName); } - private void sortList(List<Class<? extends Migrator>> migratorClasses) { - Collections.sort(migratorClasses, (m1, m2) -> { - try { - if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) { - return 1; - } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) { - return -1; - } else { - return m1.getSimpleName().compareTo(m2.getSimpleName()); - } - } catch (Exception e) { - return 0; - } - }); - } - - - private void generateSnapshot(TransactionalGraphEngine engine, String phase) { - - FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); - String dateStr= fd.getDateTime(); - String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson"; - logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName); - Graph transaction = null; - try { - - Path pathToFile = Paths.get(fileName); - if (!pathToFile.toFile().exists()) { - Files.createDirectories(pathToFile.getParent()); - } - transaction = engine.startTransaction(); - transaction.io(IoCore.graphson()).writeGraph(fileName); - engine.rollback(); - } catch (IOException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); - logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e)); - LoggingContext.successStatusFields(); - engine.rollback(); - } - - logAndPrint( phase + " migration snapshot saved to " + fileName); - } - /** - * Log and print. - * - * @param msg - * the msg - */ - protected void logAndPrint(String msg) { - System.out.println(msg); - logger.info(msg); - } - - /** - * Commit changes. - * - * @param engine - * the graph transaction - * @param migrator - * the migrator - * @param cArgs - */ - protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) { - - String simpleName = migrator.getClass().getSimpleName(); - String message; - if (migrator.getStatus().equals(Status.FAILURE)) { - message = "Migration " + simpleName + " Failed. Rolling back."; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logAndPrint("\t" + message); - LoggingContext.successStatusFields(); - migrator.rollback(); - } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) { - message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back."; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - logAndPrint("\t" + message); - LoggingContext.successStatusFields(); - migrator.rollback(); - } else { - MDC.put("logFilenameAppender", simpleName + "/" + simpleName); - - if (cArgs.commit) { - if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) { - engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate(); - } - engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE) - .property(simpleName, true).iterate(); - MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); - notifications.add(migrator.getNotificationHelper()); - migrator.commit(); - message = "Migration " + simpleName + " Succeeded. Changes Committed."; - logAndPrint("\t"+ message +"\t"); - } else { - message = "--commit not specified. Not committing changes for " + simpleName + " to database."; - logAndPrint("\t" + message); - migrator.rollback(); - } - - } - - resultsSummary.add(message); - - } - - private void outputResultsSummary() { - logAndPrint("---------------------------------"); - logAndPrint("-------------Summary-------------"); - for (String result : resultsSummary) { - logAndPrint(result); - } - logAndPrint("---------------------------------"); - logAndPrint("---------------------------------"); - } + private void sortList(List<Class<? extends Migrator>> migratorClasses) { + Collections.sort(migratorClasses, (m1, m2) -> { + try { + if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) { + return 1; + } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) { + return -1; + } else { + return m1.getSimpleName().compareTo(m2.getSimpleName()); + } + } catch (Exception e) { + return 0; + } + }); + } + + + private void generateSnapshot(TransactionalGraphEngine engine, String phase) { + + FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); + String dateStr= fd.getDateTime(); + String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson"; + logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName); + Graph transaction = null; + try { + + Path pathToFile = Paths.get(fileName); + if (!pathToFile.toFile().exists()) { + Files.createDirectories(pathToFile.getParent()); + } + String [] dataSnapshotArgs = {"-c","THREADED_SNAPSHOT", "-fileName",fileName, "-caller","migration"}; + DataSnapshot dataSnapshot = new DataSnapshot(); + dataSnapshot.executeCommand(dataSnapshotArgs, true, false, null, "THREADED_SNAPSHOT", null); +// transaction = engine.startTransaction(); +// transaction.io(IoCore.graphson()).writeGraph(fileName); +// engine.rollback(); + } catch (IOException e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); + logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e)); + LoggingContext.successStatusFields(); + engine.rollback(); + } + + logAndPrint( phase + " migration snapshot saved to " + fileName); + } + /** + * Log and print. + * + * @param msg + * the msg + */ + protected void logAndPrint(String msg) { + System.out.println(msg); + logger.info(msg); + } + + /** + * Commit changes. + * + * @param engine + * the graph transaction + * @param migrator + * the migrator + * @param cArgs + */ + protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) { + + String simpleName = migrator.getClass().getSimpleName(); + String message; + if (migrator.getStatus().equals(Status.FAILURE)) { + message = "Migration " + simpleName + " Failed. Rolling back."; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("\t" + message); + LoggingContext.successStatusFields(); + migrator.rollback(); + } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) { + message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back."; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint("\t" + message); + LoggingContext.successStatusFields(); + migrator.rollback(); + } else { + MDC.put("logFilenameAppender", simpleName + "/" + simpleName); + + if (cArgs.commit) { + if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) { + engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate(); + } + engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE) + .property(simpleName, true).iterate(); + MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + notifications.add(migrator.getNotificationHelper()); + migrator.commit(); + message = "Migration " + simpleName + " Succeeded. Changes Committed."; + logAndPrint("\t"+ message +"\t"); + } else { + message = "--commit not specified. Not committing changes for " + simpleName + " to database."; + logAndPrint("\t" + message); + migrator.rollback(); + } + + } + + resultsSummary.add(message); + + } + + private void outputResultsSummary() { + logAndPrint("---------------------------------"); + logAndPrint("-------------Summary-------------"); + for (String result : resultsSummary) { + logAndPrint(result); + } + logAndPrint("---------------------------------"); + logAndPrint("---------------------------------"); + } } diff --git a/src/main/java/org/onap/aai/migration/Migrator.java b/src/main/java/org/onap/aai/migration/Migrator.java index 106d5e4..791fec0 100644 --- a/src/main/java/org/onap/aai/migration/Migrator.java +++ b/src/main/java/org/onap/aai/migration/Migrator.java @@ -135,11 +135,11 @@ public abstract class Migrator implements Runnable { if (dmaapMsgList.size() > 0) { try { Files.write(Paths.get(logDirectory+"/"+fileName), (Iterable<String>)dmaapMsgList.stream()::iterator); - } catch (IOException e) { - logger.error("Unable to generate file with dmaap msgs for MigrateHUBEvcInventory", e); + } catch (IOException e) { + logger.error("Unable to generate file with dmaap msgs for " + getMigrationName(), e); } } else { - logger.info("No dmaap msgs detected for MigrateForwardEvcCircuitId"); + logger.info("No dmaap msgs detected for " + getMigrationName()); } } @@ -304,6 +304,28 @@ public abstract class Migrator implements Runnable { } return newEdge; } + + /** + * Creates the edge + * + * @param type the edge type - COUSIN or TREE + * @param out the out + * @param in the in + * @return the edge + */ + protected Edge createEdgeIfPossible(EdgeType type, Vertex out, Vertex in) throws AAIException { + Edge newEdge = null; + try { + if (type.equals(EdgeType.COUSIN)){ + newEdge = edgeSerializer.addEdgeIfPossible(this.engine.asAdmin().getTraversalSource(), out, in); + } else { + newEdge = edgeSerializer.addTreeEdgeIfPossible(this.engine.asAdmin().getTraversalSource(), out, in); + } + } catch (NoEdgeRuleFoundException e) { + throw new AAIException("AAI_6129", e); + } + return newEdge; + } /** * Creates the edge diff --git a/src/main/java/org/onap/aai/migration/ValueMigrator.java b/src/main/java/org/onap/aai/migration/ValueMigrator.java index 6d02563..458796a 100644 --- a/src/main/java/org/onap/aai/migration/ValueMigrator.java +++ b/src/main/java/org/onap/aai/migration/ValueMigrator.java @@ -19,7 +19,12 @@ */ package org.onap.aai.migration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Map.Entry; + import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.onap.aai.db.props.AAIProperties; @@ -38,9 +43,16 @@ import org.onap.aai.setup.SchemaVersions; public abstract class ValueMigrator extends Migrator { protected final Map<String, Map<String, ?>> propertyValuePairByNodeType; + protected Map<String, List<?>> conditionsMap; protected final Boolean updateExistingValues; - protected final JanusGraphManagement graphMgmt; - + protected final JanusGraphManagement graphMgmt; + + private int migrationSuccess = 0; + private Map<String, String> nodeTotalSuccess = new HashMap<>(); + private int subTotal = 0; + + private static List<String> dmaapMsgList = new ArrayList<String>(); + /** * * @param engine @@ -53,6 +65,23 @@ public abstract class ValueMigrator extends Migrator { this.updateExistingValues = updateExistingValues; this.graphMgmt = engine.asAdmin().getManagementSystem(); } + + //Migrate with property conditions + public ValueMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions, Map propertyValuePairByNodeType, Map conditionsMap, Boolean updateExistingValues) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.propertyValuePairByNodeType = propertyValuePairByNodeType; + this.updateExistingValues = updateExistingValues; + this.conditionsMap = conditionsMap; + this.graphMgmt = engine.asAdmin().getManagementSystem(); + } + + @Override + public void commit() { + engine.commit(); + if(isUpdateDmaap()){ + createDmaapFiles(this.dmaapMsgList); + } + } /** * Do not override this method as an inheritor of this class @@ -64,41 +93,95 @@ public abstract class ValueMigrator extends Migrator { protected void updateValues() { for (Map.Entry<String, Map<String, ?>> entry: propertyValuePairByNodeType.entrySet()) { - String nodeType = entry.getKey(); + String nodeType = entry.getKey(); + this.subTotal = 0; + Map<String, ?> propertyValuePair = entry.getValue(); for (Map.Entry<String, ?> pair : propertyValuePair.entrySet()) { - String property = pair.getKey(); + String property = pair.getKey(); Object newValue = pair.getValue(); try { GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V() - .has(AAIProperties.NODE_TYPE, nodeType); + .has(AAIProperties.NODE_TYPE, nodeType); while (g.hasNext()) { Vertex v = g.next(); - if (v.property(property).isPresent() && !updateExistingValues) { - String propertyValue = v.property(property).value().toString(); - if (propertyValue.isEmpty()) { - v.property(property, newValue); - logger.info(String.format("Node Type %s: Property %s is empty, adding value %s", - nodeType, property, newValue.toString())); - this.touchVertexProperties(v, false); - } else { - logger.info(String.format("Node Type %s: Property %s value already exists - skipping", - nodeType, property)); - } - } else { - logger.info(String.format("Node Type %s: Property %s does not exist or " + - "updateExistingValues flag is set to True - adding the property with value %s", - nodeType, property, newValue.toString())); - v.property(property, newValue); - this.touchVertexProperties(v, false); - } + + if (this.conditionsMap !=null){ + checkConditions( v, property, newValue, nodeType); + }else{ + migrateValues( v, property, newValue, nodeType); + } } } catch (Exception e) { logger.error(String.format("caught exception updating aai-node-type %s's property %s's value to " + "%s: %s", nodeType, property, newValue.toString(), e.getMessage())); logger.error(e.getMessage()); } + } + this.nodeTotalSuccess.put(nodeType, Integer.toString(this.subTotal)); + } + + logger.info ("\n \n ******* Final Summary for " + " " + getMigrationName() +" ********* \n"); + for (Map.Entry<String, String> migratedNode: nodeTotalSuccess.entrySet()) { + logger.info("Total Migrated Records for " + migratedNode.getKey() +": " + migratedNode.getValue()); + + } + logger.info(this.MIGRATION_SUMMARY_COUNT + "Total Migrated Records: "+ migrationSuccess); + + } + + private void migrateValues (Vertex v, String property, Object newValue, String nodeType) throws Exception{ + + if (v.property(property).isPresent() && !updateExistingValues) { + String propertyValue = v.property(property).value().toString(); + if (propertyValue.isEmpty()) { + v.property(property, newValue); + logger.info(String.format("Node Type %s: Property %s is empty, adding value %s", + nodeType, property, newValue.toString())); + this.touchVertexProperties(v, false); + updateDmaapList(v); + this.migrationSuccess++; + this.subTotal++; + } else { + logger.info(String.format("Node Type %s: Property %s value already exists - skipping", + nodeType, property)); } + } else { + logger.info(String.format("Node Type %s: Property %s does not exist or " + + "updateExistingValues flag is set to True - adding the property with value %s", + nodeType, property, newValue.toString())); + v.property(property, newValue); + this.touchVertexProperties(v, false); + updateDmaapList(v); + this.migrationSuccess++; + this.subTotal++; } } -} + + private void checkConditions(Vertex v, String property, Object newValue, String nodeType) throws Exception{ + + for (Map.Entry<String, List<?>> entry: conditionsMap.entrySet()){ + String conditionType = entry.getKey(); + List <?> conditionsValueList = conditionsMap.get(conditionType); + + if(v.property(conditionType).isPresent()){ + for (int i = 0; i < conditionsValueList.size(); i++){ + if (v.property(conditionType).value().equals(conditionsValueList.get(i))){ + migrateValues( v, property, newValue, nodeType); + break; + } + } + } + } + } + + private void updateDmaapList(Vertex v){ + String dmaapMsg = System.nanoTime() + "_" + v.id().toString() + "_" + v.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + logger.info("\tAdding Updated Vertex " + v.id().toString() + " to dmaapMsgList...."); + } + + public boolean isUpdateDmaap(){ + return false; + } +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v12/ALTSLicenseEntitlementMigration.java b/src/main/java/org/onap/aai/migration/v12/ALTSLicenseEntitlementMigration.java new file mode 100644 index 0000000..ef45209 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/ALTSLicenseEntitlementMigration.java @@ -0,0 +1,200 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.*; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(11) +@MigrationDangerRating(0) +public class ALTSLicenseEntitlementMigration extends Migrator{ + + private final String LICENSE_NODE_TYPE = "license"; + private final String ENTITLEMENT_NODE_TYPE = "entitlement"; + private boolean success = true; + private final GraphTraversalSource g; + private int headerLength; + + + public ALTSLicenseEntitlementMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void run() { + logger.info("---------- Update ALTS Entitlements and Licenses resource-uuid in generic-vnf ----------"); + String homeDir = System.getProperty("AJSC_HOME"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + int fileLineCounter = 0; + String fileName = homeDir + "/" + configDir + "/" + "migration-input-files/ALTS-migration-data/ALTS-migration-input.csv"; + Map<String, Set<String>> history = new HashMap<>(); + logger.info(fileName); + logger.info("---------- Processing VNFs from file ----------"); + try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { + String vnfLine; + while ((vnfLine = br.readLine()) != null) { + vnfLine = vnfLine.replace("\n", "").replace("\r", ""); + logger.info("\n"); + if (!vnfLine.isEmpty()) { + if (fileLineCounter != 0) { + String[] fields = vnfLine.split("\\s*,\\s*", -1); + if (fields.length != this.headerLength) { + logger.info("ERROR: Vnf line should contain " + this.headerLength + " columns, contains " + fields.length + " instead."); + success = false; + continue; + } + String newResourceUuid = fields[0]; + String groupUuid = fields[1]; + String vnfId = fields[19]; + logger.info("---------- Processing Line " + vnfLine + "----------"); + logger.info("newResourceUuid = " + newResourceUuid + " vnfId = " + vnfId + " group uuid = " + groupUuid); + if (history.containsKey(vnfId)){ + if (history.get(vnfId).contains(groupUuid)){ + logger.info("ERROR: duplicate groupUuid in vnf - skipping"); + fileLineCounter++; + continue; + } + else{ + history.get(vnfId).add(groupUuid); + } + } + else { + Set newSet = new HashSet(); + newSet.add(groupUuid); + history.put(vnfId, newSet); + } + List<Vertex> entitlements = g.V().has(AAIProperties.NODE_TYPE, "entitlement").has("group-uuid", groupUuid) + .where(this.engine.getQueryBuilder().createEdgeTraversal(EdgeType.TREE, "entitlement", "generic-vnf").getVerticesByProperty("vnf-id", vnfId) + .<GraphTraversal<?, ?>>getQuery()).toList(); + + List<Vertex> licenses = g.V().has(AAIProperties.NODE_TYPE, "license").has("group-uuid", groupUuid) + .where(this.engine.getQueryBuilder().createEdgeTraversal(EdgeType.TREE, "license", "generic-vnf").getVerticesByProperty("vnf-id", vnfId) + .<GraphTraversal<?, ?>>getQuery()).toList(); + + this.ChangeResourceUuid(entitlements, newResourceUuid, "entitlements", vnfId, groupUuid); + this.ChangeResourceUuid(licenses, newResourceUuid, "license", vnfId, groupUuid); + + } else { + this.headerLength = vnfLine.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength); + if (this.headerLength < 22){ + logger.info("ERROR: Input file should have 22 columns"); + this.success = false; + return; + } + } + } + fileLineCounter++; + } + } catch (FileNotFoundException e) { + logger.info("ERROR: Could not find file " + fileName, e); + success = false; + } catch (IOException e) { + logger.info("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info("encountered exception", e); + e.printStackTrace(); + success = false; + } + } + + private void ChangeResourceUuid(List<Vertex> vertices, String newResourceUuid, String nodeType, String vnfId, String groupUuid){ + if (vertices.size() > 1) { + logger.info("\t More than 1 " + nodeType + "found, skipping"); + return; + } + else if (vertices.size() == 1) { + try { + logger.info(String.format("Updating %s with groupUuid %s from generic-vnf with vnfId %s with newResourceUuid %s", nodeType, groupUuid, vnfId, newResourceUuid)); + Vertex v = vertices.get(0); + String resourceUuid = v.<String>property("resource-uuid").value(); + logger.info("\tOriginal resource-uuid: " + resourceUuid); + v.property("resource-uuid", newResourceUuid); + + String aaiUri = v.<String>property(AAIProperties.AAI_URI).value(); + if (aaiUri != null) { + logger.info("\tOriginal aaiUri: " + aaiUri); + aaiUri = aaiUri.replaceFirst("[^/]*"+resourceUuid + "$", newResourceUuid); + v.property(AAIProperties.AAI_URI, aaiUri); + logger.info("\tNew aaiUri: " + v.value(AAIProperties.AAI_URI).toString()); + } + + this.touchVertexProperties(v, false); + logger.info("\tNew resource-uuid: " + newResourceUuid); + } + catch (Exception e){ + logger.info("\t ERROR: caught exception: " + e.getMessage()); + } + } + else { + logger.info("\t No " + nodeType + " found with group-uuid "+ groupUuid + " for generic-vnf " +vnfId); + return; + } + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{LICENSE_NODE_TYPE, ENTITLEMENT_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "ALTSLicenseEntitlementMigration"; + } + +} diff --git a/src/main/java/org/onap/aai/migration/v12/MigrateDataFromASDCToConfiguration.java b/src/main/java/org/onap/aai/migration/v12/MigrateDataFromASDCToConfiguration.java new file mode 100644 index 0000000..819c7d4 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/MigrateDataFromASDCToConfiguration.java @@ -0,0 +1,138 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConstants; + +import java.io.*; +import java.util.Optional; + +@MigrationPriority(20) +@MigrationDangerRating(2) +//@Enabled +public class MigrateDataFromASDCToConfiguration extends Migrator { + private final String PARENT_NODE_TYPE = "generic-vnf"; + private boolean success = true; + private String entitlementPoolUuid = ""; + private String VNT = ""; + + + public MigrateDataFromASDCToConfiguration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + + @Override + public void run() { + + String homeDir = System.getProperty("AJSC_HOME"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + + String csvFile = homeDir + AAIConstants.AAI_FILESEP + configDir + + AAIConstants.AAI_FILESEP + "migration-input-files" + + AAIConstants.AAI_FILESEP + "VNT-migration-data" + + AAIConstants.AAI_FILESEP + "VNT-migration-input.csv"; + + logger.info("Reading Csv file: " + csvFile); + BufferedReader br = null; + String line = ""; + String cvsSplitBy = "\t"; + try { + + br = new BufferedReader(new FileReader(new File(csvFile))); + while ((line = br.readLine()) != null) { + line = line.replaceAll("\"", ""); + String[] temp = line.split(cvsSplitBy); + if ("entitlement-pool-uuid".equals(temp[0]) || "vendor-allowed-max-bandwidth (VNT)".equals(temp[1])) { + continue; + } + entitlementPoolUuid = temp[0]; + VNT = temp[1]; + GraphTraversal<Vertex, Vertex> f = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, "entitlement").has("group-uuid", entitlementPoolUuid) + .out("org.onap.relationships.inventory.BelongsTo").has(AAIProperties.NODE_TYPE, "generic-vnf") + .has("vnf-type", "HN").in("org.onap.relationships.inventory.ComposedOf").has(AAIProperties.NODE_TYPE, "service-instance").out("org.onap.relationships.inventory.Uses").has(AAIProperties.NODE_TYPE, "configuration"); + + modify(f); + } + + } catch (FileNotFoundException e) { + success = false; + logger.error("Found Exception" , e); + } catch (IOException e) { + success = false; + logger.error("Found Exception" , e); + } catch (Exception a) { + success= false; + logger.error("Found Exception" , a); + } finally { + try { + br.close(); + } catch (IOException e) { + success = false; + logger.error("Found Exception" , e); + } + } + + } + + public void modify(GraphTraversal<Vertex, Vertex> g) { + int count = 0; + while (g.hasNext()) { + Vertex v = g.next(); + logger.info("Found node type " + v.property("aai-node-type").value().toString() + " with configuration id: " + v.property("configuration-id").value().toString()); + v.property("vendor-allowed-max-bandwidth", VNT); + logger.info("VNT val after migration: " + v.property("vendor-allowed-max-bandwidth").value().toString()); + count++; + } + + logger.info("modified " + count + " configuration nodes related to Entitlement UUID: " +entitlementPoolUuid); + + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{PARENT_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateDataFromASDCToConfiguration"; + } + + +} diff --git a/src/main/java/org/onap/aai/migration/v12/MigrateHUBEvcInventory.java b/src/main/java/org/onap/aai/migration/v12/MigrateHUBEvcInventory.java new file mode 100644 index 0000000..0b3103b --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/MigrateHUBEvcInventory.java @@ -0,0 +1,293 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; + + +@MigrationPriority(31) +@MigrationDangerRating(100) +//@Enabled +public class MigrateHUBEvcInventory extends Migrator { + + private static final String FORWARDER_EVC_NODE_TYPE = "forwarder-evc"; + + private static boolean success = true; + private static boolean checkLog = false; + private static GraphTraversalSource g = null; + private int headerLength; + + private static int processedEvcsCount = 0; + private static int falloutRowsCount = 0; + private static List<String> processedEvcsList = new ArrayList<String>(); + private static Map<String, String> falloutLinesMap = new HashMap<String, String>(); + + private static final String homeDir = System.getProperty("AJSC_HOME"); + private static List<String> dmaapMsgList = new ArrayList<String>(); + + + public MigrateHUBEvcInventory(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void run() { + logger.info("---------- Start migration of HUB EVC Inventory ----------"); + String homeDir = System.getProperty("AJSC_HOME"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + + String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/sarea-inventory/"; + int fileLineCounter = 0; + String fileName = feedDir+ "hub.csv"; + logger.info(fileName); + logger.info("---------- Processing HUB Entries from file ----------"); + try { + String line; + List<String> lines = Files.readAllLines(Paths.get(fileName)); + Iterator<String> lineItr = lines.iterator(); + while (lineItr.hasNext()){ + line = lineItr.next(); + logger.info("\n"); + if (!line.isEmpty()) { + if (fileLineCounter != 0) { + String[] colList = line.split("\\s*,\\s*", -1); +// if (colList.length != headerLength) { +// logger.info("ERROR: HUB line entry should contain " + headerLength + " columns, contains " + colList.length + " instead."); +// success = false; +// continue; +// } + Map<String, String> hubColValues = new HashMap<String, String>(); + hubColValues.put("ivlan", colList[1]); + hubColValues.put("nniSvlan", colList[3]); + hubColValues.put("evcName", colList[4]); + + String evcName = hubColValues.get("evcName"); + String ivlan = hubColValues.get("ivlan"); + String nniSvlan = hubColValues.get("nniSvlan"); + if (!AAIConfig.isEmpty(evcName)) { + logger.info("---------- Processing Line " + line + "----------"); + logger.info("\t Evc Name = " + evcName ); + + List<Vertex> forwarderEvcList = g.V().has ("forwarding-path-id", evcName).has(AAIProperties.NODE_TYPE, "forwarding-path") + .in("org.onap.relationships.inventory.BelongsTo").has("aai-node-type", "forwarder") + .out("org.onap.relationships.inventory.Uses").has("aai-node-type", "configuration") + .in("org.onap.relationships.inventory.BelongsTo").has("aai-node-type", "forwarder-evc").toList(); + + + if (forwarderEvcList == null || forwarderEvcList.isEmpty()){ + logger.info("\t ERROR: Forwarder-evc does not exist for evc-id = " + evcName + " - skipping"); + falloutLinesMap.put(String.valueOf(fileLineCounter+1), "["+evcName+"] - Forwarder-evc does not exist" ); + falloutRowsCount++; + } + else if (forwarderEvcList!= null && !forwarderEvcList.isEmpty()) { + Iterator<Vertex> listItr = forwarderEvcList.iterator(); + while (listItr.hasNext()){ + Vertex forwarderEvcVtx = listItr.next(); + if (forwarderEvcVtx != null && forwarderEvcVtx.property("forwarder-evc-id").isPresent() && !AAIConfig.isEmpty(ivlan )) { + boolean isUpdated = updateIvlanOnForwarder(forwarderEvcVtx, ivlan, nniSvlan ); + if (!isUpdated){ + falloutLinesMap.put(String.valueOf(fileLineCounter+1), "["+evcName+"] - Forwarder-evc does not have svlan populated" ); + falloutRowsCount++; + } + } + } + if (!processedEvcsList.contains(evcName)) { + processedEvcsList.add(evcName); + processedEvcsCount++; + } + + } + } + } else { + this.headerLength = line.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength); + if (this.headerLength < 5){ + logger.info("ERROR: Input file should have 5 columns"); + MigrateHUBEvcInventory.success = false; + return; + } + } + } + + fileLineCounter++; + } + + logger.info ("\n \n ******* Final Summary for HUB FILE Migration ********* \n"); + logger.info("Evcs processed: "+processedEvcsCount); + logger.info("Total Rows Count: "+(fileLineCounter + 1)); + logger.info("Fallout Rows Count : "+falloutRowsCount +"\n"); + if (!falloutLinesMap.isEmpty()) { + logger.info("------ Fallout Details: ------"); + falloutLinesMap.forEach((lineNumber, errorMsg) -> { + logger.info(errorMsg + ": on row "+lineNumber.toString()); + }); + } + + } catch (FileNotFoundException e) { + logger.info("ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + checkLog = true; + } catch (IOException e) { + logger.info("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info("encountered exception", e.getMessage()); + success = false; + } + } + + private boolean updateIvlanOnForwarder(Vertex forwarderEvcVtx, String ivlan, String nniSvlan) throws Exception { + + boolean isUpdated = true; + String forwarderEvcId = forwarderEvcVtx.value("forwarder-evc-id"); + + String forwarderSvlan = null; + if( forwarderEvcVtx.property("svlan").isPresent()) { + forwarderSvlan = forwarderEvcVtx.value("svlan"); + } + if (forwarderSvlan != null && !forwarderSvlan.isEmpty()) { + int forwarderSvlanValue = Integer.parseInt(forwarderSvlan); + int nniSvlanValue = Integer.parseInt(nniSvlan); + if (forwarderSvlan != null && nniSvlan != null && (forwarderSvlanValue == nniSvlanValue)) { + if (ivlan != null && !ivlan.isEmpty()) { + if (forwarderEvcVtx.property("ivlan").isPresent()) { + String forwarderIvlan = forwarderEvcVtx.value("ivlan"); + if (forwarderIvlan != null && !forwarderIvlan.isEmpty()) { + if (Integer.parseInt(forwarderIvlan) == Integer.parseInt(ivlan)) { + logger.info("\t Skipped update ivlan for forwarder-evc[" + forwarderEvcId + + "], ivlan already set to expected value"); + } else { + logger.info("\t Start ivlan update for forwarder-evc[" + forwarderEvcId + "]"); + updateIvlan(forwarderEvcVtx, ivlan, forwarderEvcId); + } + } + } else { + updateIvlan(forwarderEvcVtx, ivlan, forwarderEvcId); + } + } + } + } else { + logger.info("Skipping ivlan update, svlan is not present on the forwarder-evc ["+forwarderEvcId +"]" ); + isUpdated = false; + } + return isUpdated; + } + + private void updateIvlan(Vertex forwarderEvcVtx, String ivlan, String forwarderEvcId) { + forwarderEvcVtx.property("ivlan", ivlan); + this.touchVertexProperties(forwarderEvcVtx, false); + logger.info("\t Updated ivlan to "+ ivlan + " on forwarder-evc[" + + forwarderEvcId + "]"); + String dmaapMsg = System.nanoTime() + "_" + forwarderEvcVtx.id().toString() + "_" + forwarderEvcVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); +// try { +// final Introspector evcIntrospector = serializer.getLatestVersionView(forwarderEvcVtx); +// this.notificationHelper.addEvent(forwarderEvcVtx, evcIntrospector, EventAction.UPDATE, +// this.serializer.getURIForVertex(forwarderEvcVtx, false)); +// } catch (UnsupportedEncodingException e) { +// logger.info("\t ERROR: Could not update ivlan on forwader-evc " + forwarderEvcVtx, e.getMessage()); +// } catch (AAIException e) { +// logger.info("\t ERROR: Could not update ivlan on forwarder-evc "+ forwarderEvcVtx, e.getMessage()); +// } + } + + + @Override + public Status getStatus() { + if (checkLog) { + return Status.CHECK_LOGS; + } + else if (success) { + return Status.SUCCESS; + } + else { + return Status.FAILURE; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{MigrateHUBEvcInventory.FORWARDER_EVC_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateHUBEvcInventory"; + } +} diff --git a/src/main/java/org/onap/aai/migration/v12/MigrateINVEvcInventory.java b/src/main/java/org/onap/aai/migration/v12/MigrateINVEvcInventory.java new file mode 100644 index 0000000..a9fce6a --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/MigrateINVEvcInventory.java @@ -0,0 +1,242 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.List; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; + + +@MigrationPriority(28) +@MigrationDangerRating(100) +public class MigrateINVEvcInventory extends Migrator { + + private static final String PROPERTY_EVC_ID = "evc-id"; + private static final String EVC_NODE_TYPE = "evc"; + + private static boolean success = true; + private static boolean checkLog = false; + private static GraphTraversalSource g = null; + private int headerLength; + + private static int processedEvcsCount = 0; + private static int falloutEvcsCount = 0; + private static Map<String, String> falloutEvcsMap = new HashMap<String, String>(); + + private static final String homeDir = System.getProperty("AJSC_HOME"); + private static List<String> dmaapMsgList = new ArrayList<String>(); + + + public MigrateINVEvcInventory(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void run() { + logger.info("---------- Start migration of INV EVC Inventory ----------"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + + String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/sarea-inventory/"; + int fileLineCounter = 0; + String fileName = feedDir+ "inv.csv"; + logger.info(fileName); + logger.info("---------- Processing INV Entries from file ----------"); + try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { + String line; + while ((line = br.readLine()) != null) { + line = line.replace("\n", "").replace("\r", ""); + logger.info("\n"); + if (!line.isEmpty()) { + if (fileLineCounter != 0) { + String[] colList = line.split("\\s*,\\s*", -1); + if (colList.length != headerLength) { + logger.info("ERROR: INV line should contain " + headerLength + " columns, contains " + colList.length + " instead."); + continue; + } + Map<String, String> invColValues = new HashMap<String, String>(); + invColValues.put("evcName", colList[22]); + invColValues.put("collectorInterconnectType", colList[17]); + + String evcName = invColValues.get("evcName"); + String interconnectType = invColValues.get("collectorInterconnectType"); + if (!AAIConfig.isEmpty(evcName) && !AAIConfig.isEmpty(interconnectType) ) { + logger.info("---------- Processing Line " + line + "----------"); + logger.info("\t Evc Name = " + evcName ); + + // For each provided evc-name, check if the evc already exists + List<Vertex> existingEvcList = g.V().has(PROPERTY_EVC_ID, evcName).has(AAIProperties.NODE_TYPE, EVC_NODE_TYPE).toList(); + if (existingEvcList == null || existingEvcList.size() == 0){ + logger.info("\t ERROR: Evc does not exist with evc-id = " + evcName + " - skipping"); + falloutEvcsCount++; + falloutEvcsMap.put((fileLineCounter+1)+"", "["+evcName+"] - Evc does not exist" ); + } + else if (existingEvcList!= null && existingEvcList.size() == 1) { + Vertex evcVtx = existingEvcList.get(0); + if (evcVtx != null && !AAIConfig.isEmpty(interconnectType )) { + updateEvcInterconnectType(evcVtx, interconnectType ); + } + processedEvcsCount++; + } + else if (existingEvcList!= null && existingEvcList.size() > 1) { + logger.info("\t ERROR: More than one EVC exist with evc-id = " + evcName + " - skipping"); + falloutEvcsCount++; + falloutEvcsMap.put((fileLineCounter+1)+"", "["+evcName+"] - More than one EVC exist with evc-id" ); + } + } else { + logger.info("---------- Processing Line " + line + "----------"); + logger.info("Invalid line entry : evcName: "+evcName + " interConnectType: "+ interconnectType); + continue; + } + } else { + this.headerLength = line.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength); + if (this.headerLength < 23){ + logger.info("ERROR: Input file should have 23 columns"); + this.success = false; + return; + } + } + } + + fileLineCounter++; + } + + logger.info ("\n \n ******* Final Summary for INV FILE Migration ********* \n"); + logger.info("Evcs processed: "+processedEvcsCount); + logger.info("Fallout Evcs count: "+falloutEvcsCount); + if (!falloutEvcsMap.isEmpty()) { + logger.info("------ Fallout Details: ------"); + falloutEvcsMap.forEach((lineNumber, errorMsg) -> { + logger.info(errorMsg + ": on row "+lineNumber.toString()); + }); + } + } catch (FileNotFoundException e) { + logger.info("ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + checkLog = true; + } catch (IOException e) { + logger.info("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info("encountered exception", e); + e.printStackTrace(); + success = false; + } + } + + + private void updateEvcInterconnectType(Vertex evcVtx, String interconnectType) { + + String evcId = evcVtx.value("evc-id"); + if (interconnectType != null && !interconnectType.isEmpty()){ + evcVtx.property("inter-connect-type-ingress", interconnectType); + this.touchVertexProperties(evcVtx, false); + logger.info("\t Updated inter-connect-type-ingress property for evc [" + evcId +"]"); + String dmaapMsg = System.nanoTime() + "_" + evcVtx.id().toString() + "_" + evcVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); +// try { +// final Introspector evcIntrospector = serializer.getLatestVersionView(evcVtx); +// this.notificationHelper.addEvent(evcVtx, evcIntrospector, EventAction.UPDATE, this.serializer +// .getURIForVertex(evcVtx, false)); +// } catch (UnsupportedEncodingException e) { +// logger.info("\t ERROR: Could not send update notification for evc " + evcId, e.getMessage()); +// } catch (AAIException e) { +// logger.info("\t ERROR: Could not send update notification for evc " + evcId, e.getMessage()); +// } + } + } + + + @Override + public Status getStatus() { + if (checkLog) { + return Status.CHECK_LOGS; + } + else if (success) { + return Status.SUCCESS; + } + else { + return Status.FAILURE; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{MigrateINVEvcInventory.EVC_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateINVEvcInventory"; + } +} diff --git a/src/main/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventory.java b/src/main/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventory.java new file mode 100644 index 0000000..0c85481 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/MigrateINVPhysicalInventory.java @@ -0,0 +1,361 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.javatuples.Pair; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + + +@MigrationPriority(25) +@MigrationDangerRating(100) +public class MigrateINVPhysicalInventory extends Migrator { + + private static final String NODE_TYPE_PNF = "pnf"; + private static final String NODE_TYPE_PINTERFACE = "p-interface"; + private static final String NODE_TYPE_PINTERFACES = "p-interfaces"; + private static final String PROPERTY_PNF_NAME = "pnf-name"; + private static final String PROPERTY_INTERFACE_NAME = "interface-name"; + protected final AtomicInteger skippedRowsCount = new AtomicInteger(0); + protected final AtomicInteger processedRowsCount = new AtomicInteger(0); + + private boolean success = true; + private boolean checkLog = false; + private GraphTraversalSource g = null; + protected int headerLength; + + protected final AtomicInteger falloutRowsCount = new AtomicInteger(0); + private static final String homeDir = System.getProperty("AJSC_HOME"); + private static List<String> dmaapMsgList = new ArrayList<String>(); + + public MigrateINVPhysicalInventory(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void run() { + logger.info("---------- Start migration of INV File Physical Inventory ----------"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + + String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/sarea-inventory/"; + String fileName = feedDir+ "inv.csv"; + logger.info(fileName); + logger.info("---------- Processing INV Entries from file ----------"); + + + try { + Map<String, Set<String>> data = loadFile(fileName); + this.processData(data); + + logger.info("\n ******* Summary Report for Inv File Physical Migration *******"); + logger.info("Number of distinct pnfs processed: "+data.keySet().size()); + logger.info("Rows processed: " + processedRowsCount); + logger.info("Rows skipped: "+ skippedRowsCount); + logger.info("Fallout Rows count: " + falloutRowsCount); + + } catch (FileNotFoundException e) { + logger.info("ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + checkLog = true; + } catch (IOException e) { + logger.info("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info("encountered exception", e); + e.printStackTrace(); + success = false; + } + } + + protected void processData(Map<String, Set<String>> data) throws Exception{ + + for (Map.Entry<String, Set<String>> entry : data.entrySet()) { + String pnfName = entry.getKey(); + final Set<String> newPInterfaces = entry.getValue(); + Introspector pnf; + Vertex pnfVertex; + EventAction eventAction = EventAction.UPDATE; + boolean pnfChangesMade = false; + + if (pnfExists(pnfName)) { + pnf = serializer.getLatestVersionView(getPnf(pnfName)); + pnfVertex = getPnf(pnfName); + } else { + pnf = loader.introspectorFromName(NODE_TYPE_PNF); + pnf.setValue(PROPERTY_PNF_NAME, pnfName); + pnfVertex = serializer.createNewVertex(pnf); + eventAction = EventAction.CREATE; + pnfChangesMade = true; + } + + if (pnfChangesMade) { + serializer.serializeSingleVertex(pnfVertex, pnf, getMigrationName()); + logger.info ("\t Pnf [" + pnfName +"] created with vertex id "+pnfVertex); +// pnf = serializer.getLatestVersionView(pnfVertex); +// this.notificationHelper.addEvent(pnfVertex, serializer.getLatestVersionView(pnfVertex), eventAction, this.serializer.getURIForVertex(pnfVertex, false)); +// logger.info("\t Dmaap notification sent for creation of pnf "); + String dmaapMsg = System.nanoTime() + "_" + pnfVertex.id().toString() + "_" + pnfVertex.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + } else { + logger.info("\t Pnf ["+ pnfName +"] already exists "); + } + + if (!newPInterfaces.isEmpty()) { + Introspector pInterfacesIntrospector = pnf.getWrappedValue(NODE_TYPE_PINTERFACES); + if ( pInterfacesIntrospector == null) { + pInterfacesIntrospector = pnf.newIntrospectorInstanceOfProperty(NODE_TYPE_PINTERFACES); + pnf.setValue(NODE_TYPE_PINTERFACES, pInterfacesIntrospector.getUnderlyingObject()); + } + + for (Introspector introspector : pInterfacesIntrospector.getWrappedListValue(NODE_TYPE_PINTERFACE)) { + String interfaceName = introspector.getValue(PROPERTY_INTERFACE_NAME).toString(); + if (newPInterfaces.contains(interfaceName)) { + newPInterfaces.remove(interfaceName); + } + } + + for (String pInterfaceName : newPInterfaces) { + Introspector pInterface = loader.introspectorFromName(NODE_TYPE_PINTERFACE); + pInterface.setValue(PROPERTY_INTERFACE_NAME, pInterfaceName); + Vertex pInterfaceVertex = serializer.createNewVertex(pInterface); + pInterfaceVertex.property(AAIProperties.AAI_URI, pnfVertex.property(AAIProperties.AAI_URI).value() + "/p-interfaces/p-interface/" + pInterfaceName); + edgeSerializer.addTreeEdge(g, pnfVertex, pInterfaceVertex); + eventAction = EventAction.CREATE; + serializer.serializeSingleVertex(pInterfaceVertex, pInterface, getMigrationName()); + logger.info ("\t p-interface [" + pInterfaceName +"] created with vertex id "+ pInterfaceVertex + " on pnf ["+pnfName+"]"); +// pInterface = serializer.getLatestVersionView(pInterfaceVertex); +// this.notificationHelper.addEvent(pInterfaceVertex, pInterface, eventAction, this.serializer.getURIForVertex(pInterfaceVertex, false)); +// logger.info("\t Dmaap notification sent for creation of p-interface "); + String dmaapMsg = System.nanoTime() + "_" + pInterfaceVertex.id().toString() + "_" + pInterfaceVertex.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + } + } + } + } + + protected boolean pnfExists(String pnfName) { + return g.V().has(PROPERTY_PNF_NAME, pnfName).has(AAIProperties.NODE_TYPE, NODE_TYPE_PNF).hasNext(); + } + + protected Vertex getPnf(String pnfName) { + return g.V().has(PROPERTY_PNF_NAME, pnfName).has(AAIProperties.NODE_TYPE, NODE_TYPE_PNF).next(); + } + + /** + * Load file to the map for processing + * @param fileName + * @return + * @throws Exception + */ + protected Map<String,Set<String>> loadFile(String fileName) throws Exception { + List<String> lines = Files.readAllLines(Paths.get(fileName)); + return this.getFileContents(lines); + } + + /** + * Get lines from file. + * @param lines + * @return + * @throws Exception + */ + protected Map<String,Set<String>> getFileContents(List<String> lines) throws Exception { + + final Map<String,Set<String>> fileContents = new ConcurrentHashMap<>(); + + processAndRemoveHeader(lines); + + logger.info("Total rows count excluding header: "+ lines.size()); + + lines.stream() + .filter(line -> !line.isEmpty()) + .map(line -> Arrays.asList(line.split("\\s*,\\s*", -1))) +// .filter(this::verifyLine) + .map(this::processLine) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(p -> { + processedRowsCount.getAndIncrement(); + String pnfName = p.getValue0(); + if (!fileContents.containsKey(pnfName)) { + Set<String> s = new HashSet<>(); + fileContents.put(p.getValue0(), s); + } + if (p.getValue1() != null) { + fileContents.get(p.getValue0()).add(p.getValue1()); + } + }) + ; + + return fileContents; + + + } + + /** + * Verify line has the necessary details. + * @param line + * @return + */ + protected boolean verifyLine(List<String> line) { + if (line.size() != headerLength) { + logger.info("ERROR: INV line should contain " + headerLength + " columns, contains " + line.size() + " instead."); + this.skippedRowsCount.getAndIncrement(); + return false; + } + return true; + } + + /** +* * Get the pnf name and interface name from the line. + * @param line + * @return + */ + protected Optional<Pair<String,String>> processLine(List<String> line) { + logger.info("Processing line... " + line.toString()); + int lineSize = line.size(); + if (lineSize < 11){ + logger.info("Skipping line, does not contain pnf and/or port columns"); + skippedRowsCount.getAndIncrement(); + return Optional.empty(); + } + + String pnfName = line.get(0); + String portAid = line.get(11).replaceAll("^\"|\"$", "").replaceAll("\\s+",""); + + if (pnfName.isEmpty() && portAid.isEmpty()) { + logger.info("Line missing pnf name and port " + line); + falloutRowsCount.getAndIncrement(); + return Optional.empty(); + } else if (pnfName.isEmpty()) { + logger.info("Line missing pnf name" + line); + falloutRowsCount.getAndIncrement(); + return Optional.empty(); + } else if (portAid.isEmpty()) { + logger.info("Line missing port " + line); + return Optional.of(Pair.with(pnfName, null)); + } + return Optional.of(Pair.with(pnfName, portAid)); + } + + /** + * Verify header of the csv and remove it from the list. + * @param lines + * @throws Exception + */ + protected String processAndRemoveHeader(List<String> lines) throws Exception { + String firstLine; + if (lines.isEmpty()) { + String msg = "ERROR: Missing Header in file"; + success = false; + logger.error(msg); + throw new Exception(msg); + } else { + firstLine = lines.get(0); + } + + this.headerLength = firstLine.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength); + if (this.headerLength < 21){ + String msg = "ERROR: Input file should have 21 columns"; + success = false; + logger.error(msg); + throw new Exception(msg); + } + + return lines.remove(0); + } + + + @Override + public Status getStatus() { + if (checkLog) { + return Status.CHECK_LOGS; + } + else if (success) { + return Status.SUCCESS; + } + else { + return Status.FAILURE; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{NODE_TYPE_PNF}); + } + + @Override + public String getMigrationName() { + return "MigrateINVPhysicalInventory"; + } + +} diff --git a/src/main/java/org/onap/aai/migration/v12/MigratePATHEvcInventory.java b/src/main/java/org/onap/aai/migration/v12/MigratePATHEvcInventory.java new file mode 100644 index 0000000..b0bacde --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/MigratePATHEvcInventory.java @@ -0,0 +1,713 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; + + +@MigrationPriority(29) +@MigrationDangerRating(100) +public class MigratePATHEvcInventory extends Migrator { + + private static Map<String, Vertex> portList = new HashMap<String, Vertex>(); + private static Map<String, Vertex> pnfList = new HashMap<String, Vertex>(); + private final String FORWARDER_EVC_NODE_TYPE = "forwarder-evc"; + private final String LAGINTERFACE_NODE_TYPE = "lag-interface"; + private final String CONFIGURATION_NODE_TYPE = "configuration"; + private final String FORWARDING_PATH_NODE_TYPE = "forwarding-path"; + private final String FORWARDING_PATH_ID = "forwarding-path-id"; + private final String PROPERTY_CONFIGURATION_ID = "configuration-id"; + private final String PNF_NODE_TYPE = "pnf"; + private final String PROPERTY_PNF_NAME = "pnf-name"; + private final String PROPERTY_INTERFACE_NAME = "interface-name"; + private final String PINTERFACE_NODE_TYPE = "p-interface"; + private static boolean success = true; + private static boolean checkLog = false; + private static GraphTraversalSource g = null; + private int headerLength; + + //Create a map to store the evcs processed where lag-interfaces were found to track the sequence of ports + //key contains the evcName + //value is a map that contains the mapping for sequence of forwarders and corresponding portAids in the order they are found + + private static Map<String, Map<Vertex, String>> pathFileMap = new HashMap<String, Map<Vertex, String>>(); + + private static int processedEvcsCount = 0; + private static int falloutEvcsCount = 0; + + //Map with lineNumber and the reason for failure for each EVC + private static Map<Integer, String> falloutEvcsList = new HashMap<Integer, String>(); + private static final String homeDir = System.getProperty("AJSC_HOME"); + private static List<String> dmaapMsgList = new ArrayList<String>(); + + + public MigratePATHEvcInventory(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void run() { + logger.info("---------- Start migration of PATH EVC Inventory ----------"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + + String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/sarea-inventory/"; + int fileLineCounter = 0; + String fileName = feedDir+ "path.csv"; + logger.info(fileName); + logger.info("---------- Processing PATH Entries from file ----------"); + try { + List<String> lines = Files.readAllLines(Paths.get(fileName)); + Iterator<String> lineItr = lines.iterator(); + while (lineItr.hasNext()){ + String line = lineItr.next().replace("\n", "").replace("\r", ""); + logger.info("\n"); + if (!line.isEmpty()) { + if (fileLineCounter != 0) { + String[] colList = line.split("\\s*,\\s*", -1); + if (colList.length != headerLength) { + logger.info("ERROR: PATH line should contain " + headerLength + " columns, contains " + colList.length + " instead."); +// success = false; + continue; + } + Map<String, String> pathColValues = new HashMap<String, String>(); + pathColValues.put("evcName", colList[1]); + pathColValues.put("bearerFacingCircuit", colList[4]); + pathColValues.put("bearerCvlan", colList[6]); + pathColValues.put("bearerSvlan", colList[7]); + pathColValues.put("bearerPtniiName", colList[8]); + String bearerPortAid = colList[12].replaceAll("^\"|\"$", "").replaceAll("\\s+",""); + pathColValues.put("bearerPortAid", bearerPortAid); + pathColValues.put("collectorFacingCircuit", colList[14]); + pathColValues.put("collectorCvlan", colList[16]); + pathColValues.put("collectorSvlan", colList[17]); + pathColValues.put("collectorPtniiName", colList[18]); + String collectorPortAid = colList[22].replaceAll("^\"|\"$", "").replaceAll("\\s+",""); + pathColValues.put("collectorPortAid", collectorPortAid); + + + String evcName = pathColValues.get("evcName"); + if (!AAIConfig.isEmpty(evcName)) { + logger.info("---------- Processing Line " + line + "----------"); + logger.info("\t Evc Name = " + evcName ); + + boolean isEntryValid = validatePnfsAndPorts(pathColValues, evcName); + + if (!isEntryValid){ + logger.info("\t ERROR: Skipping processing for line containing evc-name [" +evcName+ "]"); + falloutEvcsCount++; + falloutEvcsList.put(Integer.valueOf(fileLineCounter -1 ), "["+ evcName +"] Ptnii or port does not exist"); + continue; + } + // Get the forwarding path containing forwarders + GraphTraversal<Vertex, Vertex> forwardingPathList = g.V().has(this.FORWARDING_PATH_ID, evcName).has(AAIProperties.NODE_TYPE, this.FORWARDING_PATH_NODE_TYPE) + .where(__.in("org.onap.relationships.inventory.BelongsTo").has("aai-node-type","forwarder")); + + if (!forwardingPathList.hasNext()){ + createNewForwardersFromPATHData(pathColValues, evcName, fileLineCounter); + processedEvcsCount++; + } else { + Vertex forwardingPathVtx = forwardingPathList.next(); + List<Vertex> forwardersList = g.V(forwardingPathVtx.id()).in("org.onap.relationships.inventory.BelongsTo").toList(); + Iterator<Vertex> forwardersItr = forwardersList.iterator(); + List<String> forwarderRoleList = new ArrayList<String>(); + while (forwardersItr.hasNext()){ + Vertex forwarderVtx = forwardersItr.next(); + String role = forwarderVtx.value("forwarder-role"); + if (role!= null ){ + forwarderRoleList.add(role); + } + } + if (forwarderRoleList!= null && !forwarderRoleList.isEmpty()) { + if (forwarderRoleList.contains("ingress") && forwarderRoleList.contains("egress")){ + logger.info("\t Skipping processing for EVC[" + evcName + "] because forwarders related to this EVC already exist."); + falloutEvcsCount++; + falloutEvcsList.put(Integer.valueOf(fileLineCounter -1 ), "["+ evcName +"] Forwarders already exists for EVC"); + } else { + createNewForwardersFromPATHData(pathColValues, evcName, fileLineCounter); + } + } + } + } + } else { + this.headerLength = line.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength); + if (this.headerLength < 24){ + logger.info("ERROR: Input file should have 24 columns"); + this.success = false; + return; + } + } + } + fileLineCounter++; + } + logger.info ("\n \n ******* Final Summary for PATH FILE Migration ********* \n"); + logger.info("Evcs processed: "+processedEvcsCount); + logger.info("Total Rows Count: "+(fileLineCounter + 1)); + logger.info("Fallout Rows Count : "+falloutEvcsCount +"\n"); + if (!falloutEvcsList.isEmpty()) { + logger.info("------ Fallout Details: ------"); + falloutEvcsList.forEach((lineNumber, errorMsg) -> { + logger.info(errorMsg + ": on row "+lineNumber.toString()); + }); + } + } catch (FileNotFoundException e) { + logger.info("ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + checkLog = true; + } catch (IOException e) { + logger.info("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info("encountered exception", e); + e.printStackTrace(); + success = false; + } + } + + + private boolean validatePnfsAndPorts(Map<String, String> pathColValues, String evcName) { + + String collectorPtniiName = pathColValues.get("collectorPtniiName"); + String bearerPtniiName = pathColValues.get("bearerPtniiName"); + String collectorPortAid = pathColValues.get("collectorPortAid"); + String bearerPortAid = pathColValues.get("bearerPortAid"); + boolean isValid = validateCollectorPnf(collectorPtniiName, evcName) && validateBearerPnf(bearerPtniiName, evcName) + && validateCollectorPort(pathColValues, collectorPortAid, collectorPtniiName, evcName) + && validateBearerPort(pathColValues, bearerPortAid, bearerPtniiName, evcName) ; + return isValid; + } + + private boolean validateCollectorPnf(String collectorPtniiName, String evcName) { + + boolean isValid = false; + if (!AAIConfig.isEmpty(collectorPtniiName)) { + if (!pnfList.isEmpty() && pnfList.containsKey(collectorPtniiName)){ + isValid = true; + logger.info("\t Pnf [" + collectorPtniiName + "] found in AAI"); + return isValid; + } + List<Vertex> collectorPnfList = g.V().has(this.PROPERTY_PNF_NAME, collectorPtniiName).has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE).toList(); + if (collectorPnfList != null && collectorPnfList.size() == 1) { + isValid = true; + pnfList.put(collectorPtniiName, collectorPnfList.get(0)); + logger.info("\t Pnf [" + collectorPtniiName + "] found in AAI"); + } else if (collectorPnfList == null || collectorPnfList.size() == 0) { + logger.info("\t ERROR: Failure to find Pnf [" + collectorPtniiName + "] for EVC [" + evcName + "]"); + } + } + return isValid; + } + + private boolean validateBearerPnf(String bearerPtniiName, String evcName) { + boolean isValid = false; + if (!AAIConfig.isEmpty(bearerPtniiName)) { + if (!pnfList.isEmpty() && pnfList.containsKey(bearerPtniiName)){ + isValid = true; + logger.info("\t Pnf [" + bearerPtniiName + "] found in AAI"); + return isValid; + } + List<Vertex> bearerPnfList = g.V().has(this.PROPERTY_PNF_NAME, bearerPtniiName).has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE).toList(); + if (bearerPnfList!= null && bearerPnfList.size() == 1){ + isValid = true; + pnfList.put(bearerPtniiName, bearerPnfList.get(0)); + logger.info("\t Pnf ["+ bearerPtniiName + "] found in AAI"); + } + else if (bearerPnfList == null || bearerPnfList.size() == 0) { + logger.info("\t ERROR: Failure to find Pnf ["+ bearerPtniiName + "] for EVC [" + evcName + "]"); + } + } + return isValid; + } + + private boolean validateCollectorPort(Map<String, String> pathColValues, String collectorPortAid, String collectorPtniiName, String evcName) { + boolean isValid = false; + + if (!AAIConfig.isEmpty(collectorPortAid)) { + + boolean isPortAidALagIntf = false; + GraphTraversal<Vertex, Vertex> collectorPortList; + String lagInterface = null; + + int lagIdentifierIndex = collectorPortAid.indexOf("_"); + + if (lagIdentifierIndex > 0) { + String[] subStringList = collectorPortAid.split("_"); + lagInterface = subStringList[0]; //forwarder will be related to this lagInterface + isPortAidALagIntf = true; + } + + if (isPortAidALagIntf) + { + if (!portList.isEmpty() && portList.containsKey(collectorPtniiName+"_"+lagInterface)){ + isValid = true; + logger.info("\t lag-interface [" + lagInterface + "] found in AAI"); + populatePathFileMapWithForwarderInfo(collectorPtniiName, evcName, lagInterface, portList.get(collectorPtniiName+"_"+lagInterface)); + return isValid; + } + Vertex collectorPnfVtx = pnfList.get(collectorPtniiName); + if (collectorPnfVtx == null ) { + logger.info("\t ERROR: Failure to find lag-interface ["+ lagInterface + "] for EVC [" + evcName + "]"); + return isValid; + } else { + collectorPortList = g.V(collectorPnfVtx).in("tosca.relationships.network.BindsTo").has("interface-name", lagInterface).has("aai-node-type", "lag-interface"); + + if (collectorPortList!= null && collectorPortList.hasNext()) { + Vertex lagInterfaceVtx = collectorPortList.next(); + if (lagInterfaceVtx != null && lagInterfaceVtx.property("interface-name").isPresent()) { + isValid = true; + portList.put(collectorPtniiName+"_"+lagInterface, lagInterfaceVtx); + populatePathFileMapWithForwarderInfo(collectorPtniiName, evcName, lagInterface, lagInterfaceVtx); + logger.info("\t lag-interface [" + lagInterface + + "] found in AAI"); + } + } + else if (collectorPortList == null || !collectorPortList.hasNext()) { + logger.info("\t ERROR: Failure to find lag-interface ["+ lagInterface + "] for EVC [" + evcName + "]"); + } + } + } + else if (!isPortAidALagIntf) + { + if (!portList.isEmpty() && portList.containsKey(collectorPtniiName+"_"+collectorPortAid)){ + isValid = true; + logger.info("\t p-interface [" + collectorPortAid + "] found in AAI"); + populatePathFileMapWithForwarderInfo(collectorPtniiName, evcName, collectorPortAid, portList.get(collectorPtniiName+"_"+collectorPortAid)); + return isValid; + } + + Vertex collectorPnfVtx = pnfList.get(collectorPtniiName); + if (collectorPnfVtx == null ) { + logger.info("\t ERROR: Failure to find p-interface ["+ collectorPortAid + "] for EVC [" + evcName + "]"); + return isValid; + } else { + collectorPortList =g.V(collectorPnfVtx).in("tosca.relationships.network.BindsTo").has("interface-name", collectorPortAid).has("aai-node-type", "p-interface"); + + if (collectorPortList!= null && collectorPortList.hasNext()) { + Vertex pInterfaceVtx = collectorPortList.next(); + if (pInterfaceVtx != null && pInterfaceVtx.property("interface-name").isPresent()) { + isValid = true; + portList.put(collectorPtniiName+"_"+collectorPortAid, pInterfaceVtx ); + populatePathFileMapWithForwarderInfo(collectorPtniiName, evcName, collectorPortAid, pInterfaceVtx); + logger.info("\t p-interface [" + collectorPortAid + + "] found in AAI"); + } + } + else if (collectorPortList == null || !collectorPortList.hasNext()) { + logger.info("\t ERROR: Failure to find p-interface ["+ collectorPortAid + "] for EVC [" + evcName + "]"); + } + } + } + } + return isValid; + } + + private boolean validateBearerPort(Map<String, String> pathColValues, String bearerPortAid, String bearerPtniiName, String evcName) { + boolean isValid = false; + + if (!AAIConfig.isEmpty(bearerPortAid)) { + GraphTraversal<Vertex, Vertex> bearerPortList; + + boolean isPortAidALagIntf = false; + GraphTraversal<Vertex, Vertex> collectorPortList; + String lagInterface = null; + + int lagIdentifierIndex = bearerPortAid.indexOf("_"); + + if (lagIdentifierIndex > 0) { + String[] subStringList = bearerPortAid.split("_"); + lagInterface = subStringList[0]; //forwarder will be related to this lagInterface + isPortAidALagIntf = true; + } + + if (isPortAidALagIntf) + { + if (!portList.isEmpty() && portList.containsKey(bearerPtniiName+"_"+lagInterface)){ + isValid = true; + logger.info("\t lag-interface [" + lagInterface + "] found in AAI"); + populatePathFileMapWithForwarderInfo(bearerPtniiName, evcName, lagInterface, portList.get(bearerPtniiName+"_"+lagInterface)); + return isValid; + } + Vertex bearerPnfVtx = pnfList.get(bearerPtniiName); + if (bearerPnfVtx == null ) { + logger.info("\t ERROR: Failure to find lag-interface ["+ lagInterface + "] for EVC [" + evcName + "]"); + return isValid; + } else { + GraphTraversal<Vertex, Vertex> lagPortList = g.V(bearerPnfVtx).in("tosca.relationships.network.BindsTo").has("interface-name", lagInterface).has("aai-node-type", "lag-interface"); + if (lagPortList!= null && lagPortList.hasNext()) { + Vertex lagInterfaceVtx = lagPortList.next(); + if (lagInterfaceVtx != null && lagInterfaceVtx.property("interface-name").isPresent()) { + isValid = true; + portList.put(bearerPtniiName+"_"+lagInterface, lagInterfaceVtx); + populatePathFileMapWithForwarderInfo(bearerPtniiName, evcName, lagInterface, lagInterfaceVtx); + logger.info("\t lag-interface [" + lagInterface + + "] found in AAI"); + } + } + else if (lagPortList == null || !lagPortList.hasNext()) { + logger.info("\t ERROR: Failure to find lag-interface ["+ lagInterface + "] for EVC [" + evcName + "]"); + } + } + } + else if (!isPortAidALagIntf) { + if (!portList.isEmpty() && portList.containsKey(bearerPtniiName+"_"+bearerPortAid)){ + isValid = true; + logger.info("\t p-interface [" + bearerPortAid + "] found in AAI"); + populatePathFileMapWithForwarderInfo(bearerPtniiName, evcName, bearerPortAid, portList.get(bearerPtniiName+"_"+bearerPortAid)); + return isValid; + } + Vertex bearerPnfVtx = pnfList.get(bearerPtniiName); + if (bearerPnfVtx == null ) { + logger.info("\t ERROR: Failure to find p-interface ["+ bearerPortAid + "] for EVC [" + evcName + "]"); + return isValid; + } else { + bearerPortList = g.V(bearerPnfVtx).in("tosca.relationships.network.BindsTo").has("interface-name", bearerPortAid).has("aai-node-type","p-interface"); + if (bearerPortList!= null && bearerPortList.hasNext()){ + Vertex pInterfaceVtx = bearerPortList.next(); + if (pInterfaceVtx != null && pInterfaceVtx.property("interface-name").isPresent()) { + isValid = true; + portList.put(bearerPtniiName+"_"+bearerPortAid, pInterfaceVtx); + populatePathFileMapWithForwarderInfo(bearerPtniiName, evcName, bearerPortAid, pInterfaceVtx); + logger.info("\t p-interface [" + bearerPortAid + + "] found in AAI"); + } + } + else if (bearerPortList == null || !bearerPortList.hasNext()) { + logger.info("\t ERROR: Failure to find p-interface ["+ bearerPortAid + "] for evc [" + evcName + "]"); + } + } + } + } + return isValid; + } + + private void populatePathFileMapWithForwarderInfo(String ptniiName, String evcName, String lagInterface, Vertex interfaceVtx) { + int size = 0; + Map<Vertex, String> interfaceMap = new HashMap<Vertex, String>(); + interfaceMap = pathFileMap.get(evcName); + if (interfaceMap != null && !interfaceMap.isEmpty()) { + size = interfaceMap.size(); + } + String sequence = Integer.toString(size + 1); + if (interfaceMap != null && size > 0){ + interfaceMap.put(interfaceVtx, sequence +"_"+ ptniiName+"_"+lagInterface); + } else{ + interfaceMap = new HashMap<Vertex, String>(); + interfaceMap.put(interfaceVtx, sequence +"_"+ptniiName+"_"+lagInterface ); + } + pathFileMap.put(evcName, interfaceMap); + } + + private void createNewForwardersFromPATHData(Map<String, String> pathColValues, String evcName, int fileLineCounter) { + Map<Vertex, String> forwarderMap = pathFileMap.get(evcName); + List<Vertex> forwardingPathVtxList = g.V().has(this.FORWARDING_PATH_ID, evcName).has(AAIProperties.NODE_TYPE, FORWARDING_PATH_NODE_TYPE).toList(); + if (forwardingPathVtxList != null && !forwardingPathVtxList.isEmpty()) { + Vertex forwardingPathVtx = forwardingPathVtxList.get(0); + if (forwarderMap != null && !forwarderMap.isEmpty()) { + //for each forwarder, create the new forwarder object + forwarderMap.forEach((portVtx, port) -> { + + Vertex forwarderVtx = createForwarderObject(evcName, portVtx, port, forwardingPathVtx); + if (forwarderVtx != null) { + String forwarderRole = forwarderVtx.value("forwarder-role").toString(); + Vertex configurationVtx = createConfigurationObject(evcName, portVtx, port, forwarderVtx); + createForwarderEvcObject(pathColValues, forwarderRole, portVtx, port, + configurationVtx); + } + }); + } + } else { + falloutEvcsList.put((fileLineCounter + 1), "["+ evcName +"] Forwarding-path does not exist for EVC"); + falloutEvcsCount++; + //Reduce the count of processed evcs since this EVC row cannot be processed + processedEvcsCount--; + logger.info("\t ERROR: Forwarding-path does not exist for EVC [" + evcName + "] skipping processing for this EVC."); + } + } + + private Vertex createForwarderObject(String evcName, Vertex intfVertex, String port, Vertex forwardingPathVtx) { + Vertex forwarderVtx = null; + + try { + //check if the forwarder was already created + List<Vertex> forwardersList = g.V(forwardingPathVtx.id()).in("org.onap.relationships.inventory.BelongsTo").toList(); + Iterator<Vertex> forwardersItr = forwardersList.iterator(); + while (forwardersItr.hasNext()){ + Vertex existingForwarderVtx = forwardersItr.next(); + Vertex existingIntfVtx = g.V(existingForwarderVtx).out("org.onap.relationships.inventory.ForwardsTo").toList().get(0); + if( existingIntfVtx.id().equals(intfVertex.id())) { + //this forwarder has already been created from the forwarderMap + return null; + } + } + Integer sequence = getSequenceFromPathMapPort(port); + String role = getForwarderRole(port); + + Introspector forwarder = loader.introspectorFromName("forwarder"); + forwarderVtx = serializer.createNewVertex(forwarder); + + if (sequence != null && role != null) { + forwarder.setValue("sequence", sequence); + forwarder.setValue("forwarder-role", role ); + + //Create tree edge from forwarding-path + this.createTreeEdge(forwardingPathVtx, forwarderVtx); + //Create cousin edge to p-interface or lag-interface + this.createCousinEdge(intfVertex, forwarderVtx); + + serializer.serializeSingleVertex(forwarderVtx, forwarder, "migrations"); + +// String forwarderVtxProps = this.asString(forwarderVtx); +// logger.info(" forwarderVtxProps:" + forwarderVtxProps); + + String forwarderVtxSequence = forwarderVtx.value("sequence").toString() ; + String forwarderVtxRole = forwarderVtx.value("forwarder-role").toString(); + String forwardingPathId = forwardingPathVtx.value("forwarding-path-id").toString(); + + logger.info("\t Created new forwarder " + forwarderVtx + " with sequence = " + forwarderVtxSequence + " with role [" + forwarderVtxRole + +"] as a child of forwarding-path [" + forwardingPathId + "]" ); + + String dmaapMsg = System.nanoTime() + "_" + forwarderVtx.id().toString() + "_" + forwarderVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + +// Introspector forwarderIntrospector = serializer.getLatestVersionView(forwarderVtx); +// this.notificationHelper.addEvent(forwarderVtx, forwarderIntrospector, EventAction.CREATE, this.serializer +// .getURIForVertex(forwarderVtx, false)); +// logger.info("\t Dmaap event sent for " + forwarderVtx + " for port ["+intfVertex.toString() + "] with sequence = [" + sequence + "] and role [" + role +"]" ); + } + } catch (Exception e) { + logger.info("\t ERROR: Failure to PUT forwarder for EVC [" + evcName + "]" ); + } + return forwarderVtx; + } + + private Integer getSequenceFromPathMapPort(String port) { + String[] subStringList = port.split("_"); + String sequenceStr = subStringList[0]; //forwarder will be have this sequence + if (sequenceStr != null && !sequenceStr.isEmpty()) { + return Integer.parseInt(sequenceStr); + } else { + return null; + } + + } + + private Vertex createConfigurationObject(String evcName, Vertex portVtx, String port, Vertex forwarderVtx) { + Vertex configurationVtx = null; + String configurationId = null; + try { + Introspector configuration = loader.introspectorFromName(CONFIGURATION_NODE_TYPE); + + configurationVtx = serializer.createNewVertex(configuration); + String sequence = forwarderVtx.value("sequence").toString(); + configurationId = evcName + "-" + sequence; + configuration.setValue("configuration-id", configurationId); + configuration.setValue("configuration-type", "forwarder"); + configuration.setValue("configuration-sub-type", "forwarder"); + this.createCousinEdge(forwarderVtx, configurationVtx); + serializer.serializeSingleVertex(configurationVtx, configuration, "migrations"); + + logger.info("\t Created new configuration for forwarder " + configurationVtx + " with configuration-id= " + configurationVtx.value("configuration-id").toString() ); + + String dmaapMsg = System.nanoTime() + "_" + configurationVtx.id().toString() + "_" + configurationVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); +// Introspector introspector = serializer.getLatestVersionView(configurationVtx); +// this.notificationHelper.addEvent(configurationVtx, introspector, EventAction.CREATE, this.serializer.getURIForVertex(configurationVtx, false)); +// logger.info("\t Dmaap event sent for " + configurationVtx + " with configuration-id = " + configurationVtx.value("configuration-id").toString() ); + + return configurationVtx; + } catch (Exception e) { + logger.info("\t ERROR: Failure to PUT Configuration for forwarder [" + configurationId + "]" ); + } + return configurationVtx; + } + + private Vertex createForwarderEvcObject(Map<String, String> pathColValues, String forwarderRole, Vertex portVtx, String port, Vertex configurationVtx) { + Vertex forwarderEvcVtx = null; + String configurationId = null; + try { + Introspector forwarderEvc = loader.introspectorFromName(FORWARDER_EVC_NODE_TYPE); + forwarderEvcVtx = serializer.createNewVertex(forwarderEvc); + configurationId = configurationVtx.value(this.PROPERTY_CONFIGURATION_ID).toString(); + + String collectorFacingCircuit = pathColValues.get("collectorFacingCircuit"); + String bearerFacingCircuit = pathColValues.get("bearerFacingCircuit"); + String collectorCvlan = pathColValues.get("collectorCvlan"); + String bearerCvlan = pathColValues.get("bearerCvlan"); + String collectorSvlan = pathColValues.get("collectorSvlan"); + String bearerSvlan = pathColValues.get("bearerSvlan"); + + forwarderEvc.setValue("forwarder-evc-id", configurationId); + + //Don't set circuitid for forwarder-evc connected to configuration that's connected to intermediate forwarder. + if ("ingress".equalsIgnoreCase(forwarderRole)){ + forwarderEvc.setValue("circuit-id", checkForNull(collectorFacingCircuit)); + if (collectorCvlan != null && !collectorCvlan.isEmpty()) { + forwarderEvc.setValue("cvlan", collectorCvlan); + } + if (collectorSvlan != null && !collectorSvlan.isEmpty()) { + forwarderEvc.setValue("svlan", collectorSvlan); + } + } else if ("egress".equalsIgnoreCase(forwarderRole)){ + forwarderEvc.setValue("circuit-id", bearerFacingCircuit); + if (bearerCvlan != null && !bearerCvlan.isEmpty()) { + forwarderEvc.setValue("cvlan", bearerCvlan); + } + if (bearerSvlan != null && !bearerSvlan.isEmpty()) { + forwarderEvc.setValue("svlan", bearerSvlan); + } + } else { + int lastIndex = configurationId.lastIndexOf("-"); + String sequenceStr = configurationId.substring(lastIndex); + int i = Integer.parseInt(sequenceStr); + if (i%2 == 0){ + forwarderEvc.setValue("cvlan", checkForNull(bearerCvlan)); + forwarderEvc.setValue("svlan", checkForNull(bearerSvlan)); + } else { + forwarderEvc.setValue("cvlan", checkForNull(collectorCvlan)); + forwarderEvc.setValue("svlan", checkForNull(collectorSvlan)); + } + } + this.createTreeEdge(configurationVtx, forwarderEvcVtx); + serializer.serializeSingleVertex(forwarderEvcVtx, forwarderEvc, "migrations"); + + logger.info("\t Created new forwarder-evc as a child of configuration " + forwarderEvcVtx + " with forwarder-evc-id= " + forwarderEvcVtx.value("forwarder-evc-id").toString() ); + String dmaapMsg = System.nanoTime() + "_" + forwarderEvcVtx.id().toString() + "_" + forwarderEvcVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + +// logger.info("\t Forwarder-evc: "+ this.asString(forwarderEvcVtx)); + +// Introspector introspector = serializer.getLatestVersionView(forwarderEvcVtx); +// this.notificationHelper.addEvent(forwarderEvcVtx, introspector, EventAction.CREATE, this.serializer.getURIForVertex(forwarderEvcVtx, false)); +// logger.info("\t Dmaap event sent for " + forwarderEvcVtx + " with forwarder-evc-id = " + forwarderEvcVtx.value("forwarder-evc-id").toString() ); + } catch (Exception e) { + logger.info("\t ERROR: Failure to PUT fowarder-evc for configuration [" + configurationId + "]" ); + } + return forwarderEvcVtx; + + } + + private String getForwarderRole( String port) { + String role = null; + Integer seq = getSequenceFromPathMapPort(port); + if (seq != null ) { + int sequence = seq.intValue(); + if (sequence == 1){ + role = "ingress"; + } else if (sequence > 1 && port.indexOf(".") > 0) { + role = "egress"; + } else { + role = "intermediate"; + } + } + return role; + } + + private String checkForNull(String s){ + if (s!= null && !s.isEmpty()){ + return s; + } + return null; + } + + @Override + public Status getStatus() { + if (checkLog) { + return Status.CHECK_LOGS; + } + else if (success) { + return Status.SUCCESS; + } + else { + return Status.FAILURE; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{this.FORWARDING_PATH_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigratePATHEvcInventory"; + } +} diff --git a/src/main/java/org/onap/aai/migration/v12/MigratePATHPhysicalInventory.java b/src/main/java/org/onap/aai/migration/v12/MigratePATHPhysicalInventory.java new file mode 100644 index 0000000..af3d90a --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/MigratePATHPhysicalInventory.java @@ -0,0 +1,348 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; + + +@MigrationPriority(26) +@MigrationDangerRating(100) +public class MigratePATHPhysicalInventory extends Migrator { + + private static List<String> lagPortList = new ArrayList<String>(); + private static Map<String, Vertex> pnfList = new HashMap<String, Vertex>(); + private final String LAGINTERFACE_NODE_TYPE = "lag-interface"; + private final String PNF_NODE_TYPE = "pnf"; + private final String PROPERTY_PNF_NAME = "pnf-name"; + private final String PROPERTY_INTERFACE_NAME = "interface-name"; + private final String LAG_INTERFACE_NODE_TYPE = "lag-interface"; + private static boolean success = true; + private static boolean checkLog = false; + private static GraphTraversalSource g = null; + private int headerLength; + + private static List<String> dmaapMsgList = new ArrayList<String>(); + private static final String homeDir = System.getProperty("AJSC_HOME"); + + //Create a map to store the evcs processed where lag-interfaces were found to track the sequence of ports + //key contains the evcName + //value is a map that contains the mapping for sequence of forwarders and corresponding portAids in the order they are found + + private static Map<String, Map<Vertex, String>> pathFileMap = new HashMap<String, Map<Vertex, String>>(); + + private static int processedLagInterfacesCount = 0; + private static int skippedRowCount = 0; + //Map with lineNumber and the reason for failure for each interface + private static Map<String, String> lagInterfacesNotProcessedMap = new HashMap<String, String>(); + + + public MigratePATHPhysicalInventory(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void run() { + logger.info("---------- Start migration of PATH file Physical Inventory ----------"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + + String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/sarea-inventory/"; + int fileLineCounter = 0; + String fileName = feedDir+ "path.csv"; + logger.info(fileName); + logger.info("---------- Processing PATH Entries from file ----------"); + try { + List<String> lines = Files.readAllLines(Paths.get(fileName)); + Iterator<String> lineItr = lines.iterator(); + while (lineItr.hasNext()){ + String line = lineItr.next().replace("\n", "").replace("\r", ""); + logger.info("\n"); + if (!line.isEmpty()) { + if (fileLineCounter != 0) { + String[] colList = line.split("\\s*,\\s*", -1); + Map<String, String> pathColValues = new HashMap<String, String>(); + pathColValues.put("evcName", colList[1]); + pathColValues.put("bearerFacingCircuit", colList[4]); + pathColValues.put("bearerCvlan", colList[6]); + pathColValues.put("bearerSvlan", colList[7]); + pathColValues.put("bearerPtniiName", colList[8]); + pathColValues.put("bearerPortAid", colList[12]); + pathColValues.put("collectorFacingCircuit", colList[14]); + pathColValues.put("collectorCvlan", colList[16]); + pathColValues.put("collectorSvlan", colList[17]); + pathColValues.put("collectorPtniiName", colList[18]); + pathColValues.put("collectorPortAid", colList[22]); + + // For each row, check if the collector and bearerPnfs exist and create lag interfaces + + validateCollectorPnfAndCreateLagInterface(pathColValues, (fileLineCounter+1)); + validateBearerPnfAndCreateLagInterface(pathColValues, (fileLineCounter+1)); + + } else { + this.headerLength = line.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength); + if (this.headerLength < 21){ + logger.info("ERROR: Input file should have 21 columns"); + this.success = false; + return; + } + } + } + fileLineCounter++; + } + logger.info ("\n \n ******* Final Summary for PATH FILE Physical Inventory Migration ********* \n"); + logger.info("Lag Interfaces processed: "+processedLagInterfacesCount); + logger.info("Total Rows Count: "+(fileLineCounter + 1)); + logger.info("Fallout Lag Interfaces Count : "+lagInterfacesNotProcessedMap.size() +"\n"); + + if (!lagInterfacesNotProcessedMap.isEmpty()) { + logger.info("------ Fallout Details: ------"); + lagInterfacesNotProcessedMap.forEach((lineEntry, errorMsg) -> { + int lineNumberIndex = lineEntry.indexOf("-"); + String lineNumber = lineEntry.substring(0, lineNumberIndex); + String portDetail = lineEntry.substring(lineNumberIndex+1); + logger.info(errorMsg + ": on row "+ lineNumber +" for PortAid ["+ portDetail+"]"); + }); + } + } catch (FileNotFoundException e) { + logger.info("ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + checkLog = true; + } catch (IOException e) { + logger.info("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info("encountered exception", e); + e.printStackTrace(); + success = false; + } + } + + + private void validateBearerPnfAndCreateLagInterface(Map<String, String> pathColValues, int lineNumber) { + String bearerPtniiName = pathColValues.get("bearerPtniiName"); + String bearerPortAid = pathColValues.get("bearerPortAid"); + Vertex pnfVtx = getPnf(bearerPtniiName); + if (pnfVtx != null){ + //create lag-interface + createLagInterfaceObject(pnfVtx, bearerPortAid, lineNumber); + } else { + int lagIdentifierIndex = bearerPortAid.indexOf("_"); + if (lagIdentifierIndex > 0) { + lagInterfacesNotProcessedMap.put(""+ lineNumber+ "-"+bearerPtniiName+"-"+bearerPortAid+"", "Pnf ["+bearerPtniiName+"] not found" ); + } + } + + } + + private void validateCollectorPnfAndCreateLagInterface(Map<String, String> pathColValues, int lineNumber) { + String collectorPtniiName = pathColValues.get("collectorPtniiName"); + String collectorPortAid = pathColValues.get("collectorPortAid"); + Vertex pnfVtx = getPnf(collectorPtniiName); + if (pnfVtx != null){ + //create lag-interface + createLagInterfaceObject(pnfVtx, collectorPortAid, lineNumber); + }else { + int lagIdentifierIndex = collectorPortAid.indexOf("_"); + if (lagIdentifierIndex > 0) { + lagInterfacesNotProcessedMap.put(""+ lineNumber+ "-"+collectorPtniiName+"-"+collectorPortAid+"", "Pnf ["+collectorPtniiName+"] not found" ); + } + } + } + + private void createLagInterfaceObject(Vertex pnfVtx, String portAid, int lineNumber) { + String pnfName = pnfVtx.value(PROPERTY_PNF_NAME); + + if (pnfName != null && !pnfName.isEmpty()) { + + if(portAid == null || portAid.isEmpty()){ + logger.info("\t Invalid Port entry [" +portAid + "] - Invalid record - skipping..." ); + } else{ + if (!AAIConfig.isEmpty(portAid)) { + GraphTraversal<Vertex, Vertex> portList; + + boolean isPortAidALagIntf = false; + String interfaceName = null; + + int lagIdentifierIndex = portAid.indexOf("_"); + + if (lagIdentifierIndex > 0) { + String[] subStringList = portAid.split("_"); + interfaceName = subStringList[0]; + isPortAidALagIntf = true; + } + + if (isPortAidALagIntf) + { + try { + + if (lagPortList != null && lagPortList.contains(pnfName+"_"+interfaceName)){ + logger.info("\t lag-interface [" + interfaceName + "] already exists in AAI - skipping"); + return; + } + + + portList = g.V(pnfVtx).in("tosca.relationships.network.BindsTo").has("interface-name", interfaceName).has("aai-node-type", "lag-interface"); + if (portList!= null && portList.hasNext()) { + Vertex lagInterfaceVtx = portList.next(); + if (lagInterfaceVtx != null && lagInterfaceVtx.property("interface-name").isPresent()) { + logger.info("\t lag-interface [" + interfaceName + "] already exists in AAI - skipping"); +// lagInterfacesNotProcessedMap.put(""+lineNumber+"-"+pnfName+"-"+portAid+"", "lag-interface already exists for ["+interfaceName+"]" ); + } + } + else if (portList == null || !portList.hasNext()) { + //Create lag-interface in pnf + Introspector lagInterface = loader.introspectorFromName(LAG_INTERFACE_NODE_TYPE); + + Vertex lagIntVtx = serializer.createNewVertex(lagInterface); + lagInterface.setValue("interface-name", interfaceName); + this.createTreeEdge(pnfVtx, lagIntVtx); + serializer.serializeSingleVertex(lagIntVtx, lagInterface, "migrations"); + + logger.info("\t Created new lag-interface " + lagIntVtx + " with interface-name= " + lagIntVtx.value("interface-name")); + + processedLagInterfacesCount++; + lagPortList.add(pnfName+"_"+interfaceName); + + String dmaapMsg = System.nanoTime() + "_" + lagIntVtx.id().toString() + "_" + lagIntVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); +// Introspector introspector = serializer.getLatestVersionView(lagIntVtx); +// this.notificationHelper.addEvent(lagIntVtx, introspector, EventAction.CREATE, this.serializer.getURIForVertex(lagIntVtx, false)); +// logger.info("\t Dmaap event sent for " + lagIntVtx + " with interface-name= " + lagIntVtx.value("interface-name").toString() ); + } + } catch (Exception e) { + logger.info("\t ERROR: Failure to create lag-interface ["+ interfaceName + "]"); + lagInterfacesNotProcessedMap.put(""+lineNumber+"-"+pnfName+"-"+portAid+"", "Failed to create lag-interface ["+interfaceName+"]" ); + } + } + else + { + logger.info("\t Port-Aid[" +portAid +"] on PNF["+pnfName+"] not a lag-interface, skipping...."); + } + } + + } + } + } + + + private Vertex getPnf(String ptniiName) { + Vertex pnfVtx = null; + if (!AAIConfig.isEmpty(ptniiName)) { + if (!pnfList.isEmpty() && pnfList.containsKey(ptniiName)){ + return pnfList.get(ptniiName); + } + List<Vertex> collectorPnfList = g.V().has(this.PROPERTY_PNF_NAME, ptniiName).has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE).toList(); + if (collectorPnfList != null && collectorPnfList.size() == 1) { + pnfVtx = collectorPnfList.get(0); + pnfList.put(ptniiName, pnfVtx); + logger.info("\t Pnf [" + ptniiName + "] found in AAI"); + } else if (collectorPnfList == null || collectorPnfList.size() == 0) { + logger.info("\t ERROR: Failure to find Pnf [" + ptniiName + "]" ); + } + } else { + logger.info("\t ERROR: Failure to find Pnf [" + ptniiName + "]" ); + } + return pnfVtx; + } + + @Override + public Status getStatus() { + if (checkLog) { + return Status.CHECK_LOGS; + } + else if (success) { + return Status.SUCCESS; + } + else { + return Status.FAILURE; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{this.LAG_INTERFACE_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigratePATHPhysicalInventory"; + } +} diff --git a/src/main/java/org/onap/aai/migration/v12/MigrateSAREvcInventory.java b/src/main/java/org/onap/aai/migration/v12/MigrateSAREvcInventory.java new file mode 100644 index 0000000..e05999d --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v12/MigrateSAREvcInventory.java @@ -0,0 +1,551 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v12; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.List; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; + + +@MigrationPriority(27) +@MigrationDangerRating(100) +public class MigrateSAREvcInventory extends Migrator { + + private static Map<String, Vertex> pnfList = new HashMap<String, Vertex>(); + private static List<String> portList = new ArrayList<String>();; + private final String SAREA_GLOBAL_CUSTOMER_ID = "8a00890a-e6ae-446b-9dbe-b828dbeb38bd"; + private final String CONFIGURATION_NODE_TYPE = "configuration"; + private final String SERVICE_INSTANCE_NODE_TYPE = "service-instance"; + private final String SERVICE_SUBSCRIPTON_NODE_TYPE = "service-subscription"; + private final String PROPERTY_SERVICE_TYPE = "service-type"; + private final String SERVICE_INSTANCE_ID = "service-instance-id"; + private final String FORWARDING_PATH_NODE_TYPE = "forwarding-path"; + private final String FOWARDING_PATH_ID = "forwarding-path-id"; + private final String EVC_NODE_TYPE = "evc"; + private final String PROPERTY_CONFIGURATION_ID = "configuration-id"; + private final String PNF_NODE_TYPE = "pnf"; + private final String PROPERTY_PNF_NAME = "pnf-name"; + private final String PROPERTY_INTERFACE_NAME = "interface-name"; + private final String PINTERFACE_NODE_TYPE = "p-interface"; + private static boolean success = true; + private static boolean checkLog = false; + private static GraphTraversalSource g = null; + private int headerLength; + + private static int processedEvcsCount = 0; + private static int falloutEvcsCount = 0; + private static Map<String, String> falloutEvcsMap = new HashMap<String, String>(); + + private static final String homeDir = System.getProperty("AJSC_HOME"); + private static List<String> dmaapMsgList = new ArrayList<String>(); + + public MigrateSAREvcInventory(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void run() { + logger.info("---------- Start migration of SAR EVC Inventory ----------"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + + String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/sarea-inventory/"; + int fileLineCounter = 0; + String fileName = feedDir+ "sar.csv"; + logger.info(fileName); + logger.info("---------- Processing SAR Entries from file ----------"); + + try { + String line; + + List<String> lines = Files.readAllLines(Paths.get(fileName)); + Iterator<String> lineItr = lines.iterator(); + while (lineItr.hasNext()){ + line = lineItr.next().replace("\n", "").replace("\r", ""); + logger.info("\n"); + if (!line.isEmpty()) { + if (fileLineCounter != 0) { + String[] colList = line.split("\\s*,\\s*", -1); +// if (colList.length != headerLength) { +// logger.info("ERROR: SAR line should contain " + headerLength + " columns, contains " + colList.length + " instead."); +// success = false; +// continue; +// } + Map<String, String> sarColValues = new HashMap<String, String>(); + sarColValues.put("evcName", colList[0]); + sarColValues.put("subscriberName", colList[1]); + sarColValues.put("espName", colList[2]); + sarColValues.put("bearerCircuitId", colList[3]); + sarColValues.put("bearerTagMode", colList[4]); + sarColValues.put("bearerCvlan", colList[5]); + sarColValues.put("bearerSvlan", colList[6]); + sarColValues.put("bearerPtniiName", colList[7]); + sarColValues.put("bearerSlotName", colList[8]); + String bearerPortAid = colList[9].replaceAll("^\"|\"$", "").replaceAll("\\s+",""); + sarColValues.put("bearerPortAid", bearerPortAid); + sarColValues.put("bearerPortType", colList[10]); + sarColValues.put("collectorCircuitId", colList[11]); + sarColValues.put("collectorTagMode", colList[12]); + sarColValues.put("collectorCvlan", colList[13]); + sarColValues.put("collectorSvlan", colList[14]); + sarColValues.put("collectorPtniiName", colList[15]); + sarColValues.put("collectorSlotName", colList[16]); + String collectorPortAid = colList[17].replaceAll("^\"|\"$", "").replaceAll("\\s+",""); + sarColValues.put("collectorPortAid", collectorPortAid); + sarColValues.put("collectorPortType", colList[18]); + sarColValues.put("espEvcCircuitId", colList[19]); + sarColValues.put("evcAccessCIR", colList[20]); + + String evcName = sarColValues.get("evcName"); + if (!AAIConfig.isEmpty(evcName)) { + logger.info("---------- Processing Line " + line + "----------"); + logger.info("\t Evc Name = " + evcName ); + + boolean isEntryValid = validatePnfsAndPorts(sarColValues, evcName); + + if (!isEntryValid){ + logger.info("\t ERROR: Skipping processing for line containing evc-name [" +evcName+ "]"); + falloutEvcsCount++; + falloutEvcsMap.put((fileLineCounter+1)+"", "["+evcName+"] - PortAid/Pnf does not exist" ); + fileLineCounter++; + continue; + } + + createNewObjectsFromSARFile(sarColValues, evcName, fileLineCounter); + + } + } else { + this.headerLength = line.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength); + if (this.headerLength < 21){ + logger.info("ERROR: Input file should have 21 columns"); + this.success = false; + return; + } + } + } + fileLineCounter++; + } + + logger.info ("\n \n ******* Final Summary for SAR FILE Migration ********* \n"); + logger.info("Evcs processed: "+processedEvcsCount); + logger.info("Fallout Evcs count: "+falloutEvcsCount); + if (!falloutEvcsMap.isEmpty()) { + logger.info("------ Fallout Details: ------"); + falloutEvcsMap.forEach((lineNumber, errorMsg) -> { + logger.info(errorMsg + ": on row "+lineNumber.toString()); + }); + } + } catch (FileNotFoundException e) { + logger.info("ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + checkLog = true; + } catch (IOException e) { + logger.info("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info("encountered exception", e); + e.printStackTrace(); + success = false; + } + } + + + private boolean validatePnfsAndPorts(Map<String, String> sarColValues, String evcName) { + + String collectorPtniiName = sarColValues.get("collectorPtniiName"); + String bearerPtniiName = sarColValues.get("bearerPtniiName"); + String collectorPortAid = sarColValues.get("collectorPortAid"); + String bearerPortAid = sarColValues.get("bearerPortAid"); + boolean isValid = validateCollectorPnf(collectorPtniiName, evcName) && validateBearerPnf(bearerPtniiName, evcName) + && validateCollectorPort(collectorPortAid, collectorPtniiName, evcName) + && validateBearerPort(bearerPortAid, bearerPtniiName, evcName) ; + return isValid; + } + + private boolean validateCollectorPnf(String collectorPtniiName, String evcName) { + + boolean isValid = false; + if (!AAIConfig.isEmpty(collectorPtniiName)) { + if (!pnfList.isEmpty() && pnfList.containsKey(collectorPtniiName)){ + isValid = true; + logger.info("\t Pnf [" + collectorPtniiName + "] found in AAI"); + return isValid; + } + List<Vertex> collectorPnfList = g.V().has(this.PROPERTY_PNF_NAME, collectorPtniiName).has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE).toList(); + if (collectorPnfList != null && collectorPnfList.size() == 1) { + isValid = true; + pnfList.put(collectorPtniiName, collectorPnfList.get(0)); + logger.info("\t Pnf [" + collectorPtniiName + "] found in AAI"); + } else if (collectorPnfList == null || collectorPnfList.size() == 0) { + logger.info("\t ERROR: Failure to find Pnf [" + collectorPtniiName + "] for EVC [" + evcName + "]"); + } + } + return isValid; + } + + private boolean validateBearerPnf(String bearerPtniiName, String evcName) { + boolean isValid = false; + if (!AAIConfig.isEmpty(bearerPtniiName)) { + if (!pnfList.isEmpty() && pnfList.containsKey(bearerPtniiName)){ + isValid = true; + logger.info("\t Pnf [" + bearerPtniiName + "] found in AAI"); + return isValid; + } + List<Vertex> bearerPnfList = g.V().has(this.PROPERTY_PNF_NAME, bearerPtniiName).has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE).toList(); + if (bearerPnfList!= null && bearerPnfList.size() == 1){ + isValid = true; + pnfList.put(bearerPtniiName, bearerPnfList.get(0)); + logger.info("\t Pnf ["+ bearerPtniiName + "] found in AAI"); + } + else if (bearerPnfList == null || bearerPnfList.size() == 0) { + logger.info("\t ERROR: Failure to find Pnf ["+ bearerPtniiName + "] for EVC [" + evcName + "]"); + } + } + return isValid; + } + + private boolean validateCollectorPort(String collectorPortAid, String collectorPtniiName, String evcName) { + boolean isValid = false; + if (!AAIConfig.isEmpty(collectorPortAid)) { + if (!portList.isEmpty() && portList.contains(collectorPtniiName+"_"+collectorPortAid)){ + isValid = true; + logger.info("\t Port ["+ collectorPortAid + "] found in AAI"); + return isValid; + } + GraphTraversal<Vertex, Vertex> collectorPortList; + Vertex collectorPnfVtx = pnfList.get(collectorPtniiName); + if (collectorPnfVtx == null ) { + logger.info("\t ERROR: Failure to find p-interface ["+ collectorPortAid + "] for EVC [" + evcName + "]"); + return isValid; + } else { + collectorPortList =g.V(collectorPnfVtx).in("tosca.relationships.network.BindsTo").has("interface-name", collectorPortAid).has("aai-node-type", "p-interface"); + if (collectorPortList!= null && collectorPortList.hasNext()) { + isValid = true; + portList.add(collectorPtniiName+"_"+collectorPortAid); + logger.info("\t Port ["+ collectorPortAid + "] found in AAI"); + } + else if (collectorPortList == null || !collectorPortList.hasNext()) { + logger.info("\t ERROR: Failure to find p-interface ["+ collectorPortAid + "] for EVC [" + evcName + "]"); + } + } + } + return isValid; + } + + private boolean validateBearerPort(String bearerPortAid, String bearerPtniiName, String evcName) { + boolean isValid = false; + + if (!AAIConfig.isEmpty(bearerPortAid)) { + if (!portList.isEmpty() && portList.contains(bearerPtniiName+"_"+bearerPortAid)){ + isValid = true; + logger.info("\t Port ["+ bearerPortAid + "] found in AAI"); + return isValid; + } + GraphTraversal<Vertex, Vertex> bearerPortList; + Vertex bearerPnfVtx = pnfList.get(bearerPtniiName); + if (bearerPnfVtx == null ) { + logger.info("\t ERROR: Failure to find p-interface ["+ bearerPortAid + "] for EVC [" + evcName + "]"); + return isValid; + } else { + bearerPortList =g.V(bearerPnfVtx).in("tosca.relationships.network.BindsTo").has("interface-name", bearerPortAid).has("aai-node-type", "p-interface"); + if (bearerPortList!= null && bearerPortList.hasNext()){ + isValid = true; + portList.add(bearerPtniiName+"_"+bearerPortAid); + logger.info("\t Port ["+ bearerPortAid + "] found in AAI"); + } + else if (bearerPortList == null || !bearerPortList.hasNext()) { + logger.info("\t ERROR: Failure to find p-interface ["+ bearerPortAid + "] for evc [" + evcName + "]"); + } + } + } + return isValid; + } + + private void createNewObjectsFromSARFile(Map<String, String> sarColValues, String evcName, int lineNumber) { + Vertex serviceInstanceVtx = createNewServiceInstanceFromSARData(sarColValues, evcName, lineNumber); + if (serviceInstanceVtx != null && serviceInstanceVtx.property("service-instance-id").isPresent()) { + Vertex forwardingPathVtx = createNewForwardingPathFromSARData(sarColValues, serviceInstanceVtx, lineNumber); + Vertex configurationVtx = createNewConfigurationFromSARData(sarColValues, forwardingPathVtx, lineNumber); + Vertex evcVtx = createNewEvcFromSARData(sarColValues, configurationVtx, lineNumber); + } + } + + private Vertex createNewServiceInstanceFromSARData(Map<String, String> sarColValues, String evcName, int lineNumber) { + + String serviceType = "SAREA"; + Vertex serviceInstanceVtx = null; + + try { + + GraphTraversal<Vertex, Vertex> servSubVtxList = g.V().has("global-customer-id", SAREA_GLOBAL_CUSTOMER_ID) + .in("org.onap.relationships.inventory.BelongsTo").has("service-type", "SAREA"); + + if (servSubVtxList!= null && servSubVtxList.hasNext()){ + Vertex serviceSubscriptionVtx = servSubVtxList.next(); + if (serviceSubscriptionVtx != null ) { + + List<Vertex> existingServInstVtxList = g.V(serviceSubscriptionVtx).in("org.onap.relationships.inventory.BelongsTo").has("aai-node-type", "service-instance") + .has("service-instance-id",evcName).toList(); + + if (existingServInstVtxList!= null && existingServInstVtxList.size() >0){ + logger.info("\t service-instance already exists for evc " + evcName + " - skipping"); + + return existingServInstVtxList.iterator().next(); + } + else if (existingServInstVtxList!= null && existingServInstVtxList.size() == 0) { + Introspector servInstance = loader.introspectorFromName("service-instance"); + serviceInstanceVtx = serializer.createNewVertex(servInstance); + String serviceInstanceId = (String) sarColValues.get("evcName"); + servInstance.setValue("service-instance-id", serviceInstanceId); + servInstance.setValue("service-type", serviceType); + this.createTreeEdge(serviceSubscriptionVtx, serviceInstanceVtx); + serializer.serializeSingleVertex(serviceInstanceVtx, servInstance, "migrations"); + + logger.info("\t Created new service-instance " + serviceInstanceVtx + " with service-instance-id = " + serviceInstanceId ); + + String dmaapMsg = System.nanoTime() + "_" + serviceInstanceVtx.id().toString() + "_" + serviceInstanceVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + processedEvcsCount++; + } + else { + logger.info("\t ERROR: More than one service-instance found for evc-name: " + evcName); + } + } + } else { + logger.info("\t ERROR: SAREA Subscription not found for Customer ["+SAREA_GLOBAL_CUSTOMER_ID+"]"); + falloutEvcsCount++; + falloutEvcsMap.put((lineNumber+1)+"", "["+evcName+"] - SAREA Subscription not found for Customer ["+SAREA_GLOBAL_CUSTOMER_ID+"]" ); + } + } catch (Exception e) { + logger.info("\t ERROR: Failure to PUT service-instance for EVC [" + evcName + "]" ); + falloutEvcsCount++; + falloutEvcsMap.put((lineNumber+1)+"", "["+evcName+"] - Failure to PUT service-instance for EVC" ); + } + return serviceInstanceVtx; + + } + + private Vertex createNewForwardingPathFromSARData(Map<String, String> sarColValues, Vertex serviceInstanceVtx, int lineNumber) { + Vertex fpVertex = null; + String serviceInstanceId = serviceInstanceVtx.value(this.SERVICE_INSTANCE_ID); + + try { + + List<Vertex> fpList = g.V(serviceInstanceVtx).in("org.onap.relationships.inventory.AppliesTo").has("aai-node-type","forwarding-path") + .has("forwarding-path-id", serviceInstanceId).toList(); + if (fpList != null && !fpList.isEmpty()){ + logger.info("\t forwarding-path already exists for evc " + serviceInstanceId + " - skipping"); + return fpList.iterator().next(); + } + + //If forwarding-path does not exist, create it + Introspector fpIntrospector = loader.introspectorFromName(FORWARDING_PATH_NODE_TYPE); + fpVertex = serializer.createNewVertex(fpIntrospector); + + fpIntrospector.setValue("forwarding-path-id", serviceInstanceId); + fpIntrospector.setValue("forwarding-path-name", serviceInstanceId); + this.createCousinEdge(fpVertex, serviceInstanceVtx); + serializer.serializeSingleVertex(fpVertex, fpIntrospector, "migrations"); + + logger.info("\t Created new forwarding-path " + fpVertex + " with forwarding-path-id = " + fpVertex.value("forwarding-path-id").toString() ); + String dmaapMsg = System.nanoTime() + "_" + fpVertex.id().toString() + "_" + fpVertex.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + + } catch (Exception e) { + logger.info("\t ERROR: Failure to PUT forwarding-path for EVC [" + serviceInstanceId + "]" ); + processedEvcsCount--; + falloutEvcsCount++; + falloutEvcsMap.put((lineNumber+1)+"", "["+serviceInstanceId+"] - Failure to PUT forwarding-path for EVC" ); + } + return fpVertex; + } + + private Vertex createNewConfigurationFromSARData(Map<String, String> sarColValues, Vertex forwardingPathVtx, int lineNumber) { + + Vertex configurationVtx = null; + String forwardingPathId = forwardingPathVtx.value(this.FOWARDING_PATH_ID); + try { + + List<Vertex> configList = g.V(forwardingPathVtx).out("org.onap.relationships.inventory.Uses").has("aai-node-type","configuration") + .has("configuration-id", forwardingPathId).toList(); + if (configList != null && !configList.isEmpty()){ + logger.info("\t configuration already exists for evc " + forwardingPathId + " - skipping"); + return configList.iterator().next(); + } + + //If configuration does not exist, create it + Introspector configuration = loader.introspectorFromName(CONFIGURATION_NODE_TYPE); + configurationVtx = serializer.createNewVertex(configuration); + + configuration.setValue("configuration-id", forwardingPathId); + configuration.setValue("configuration-type", "forwarding-path"); + configuration.setValue("configuration-sub-type", "evc"); + this.createCousinEdge(forwardingPathVtx, configurationVtx); + serializer.serializeSingleVertex(configurationVtx, configuration, "migrations"); + + logger.info("\t Created new configuration for forwarding-path " + configurationVtx + " with configuration-id= " + configurationVtx.value("configuration-id").toString() ); + + String dmaapMsg = System.nanoTime() + "_" + configurationVtx.id().toString() + "_" + configurationVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + + }catch (Exception e) { + logger.info("\t ERROR: Failure to PUT configuration for EVC [" + forwardingPathId + "]" ); + processedEvcsCount--; + falloutEvcsCount++; + falloutEvcsMap.put((lineNumber+1)+"", "["+forwardingPathId+"] - Failure to PUT configuration for EVC" ); + } + return configurationVtx; + } + + private Vertex createNewEvcFromSARData(Map<String, String> sarColValues, Vertex configurationVtx, int lineNumber) { + String evcId = null; + Vertex evcVtx = null; + try { + Introspector evc = loader.introspectorFromName(EVC_NODE_TYPE); + evcVtx = serializer.createNewVertex(evc); + evcId = configurationVtx.value(this.PROPERTY_CONFIGURATION_ID); + + String cir = sarColValues.get("evcAccessCIR"); + int length = cir.length(); + String cirValue = cir.substring(0,(length-4)); + String cirUnits = cir.substring((length-4), (length)); + + String espEvcCircuitId = sarColValues.get("espEvcCircuitId"); + String espName = sarColValues.get("espName"); + String collectorTagMode = sarColValues.get("collectorTagMode"); + String bearerTagMode = sarColValues.get("bearerTagMode"); + + evc.setValue("evc-id", evcId); + evc.setValue("forwarding-path-topology", "PointToPoint"); + evc.setValue("cir-value", checkForNull(cirValue)); + evc.setValue("cir-units", checkForNull(cirUnits)); + evc.setValue("esp-evc-circuit-id", checkForNull(espEvcCircuitId)); + evc.setValue("esp-evc-cir-value", checkForNull(cirValue)); + evc.setValue("esp-evc-cir-units", checkForNull(cirUnits)); + evc.setValue("esp-itu-code", checkForNull(espName)); + evc.setValue("tagmode-access-ingress", checkForNull(collectorTagMode)); + evc.setValue("tagmode-access-egress", checkForNull(bearerTagMode)); + this.createTreeEdge(configurationVtx, evcVtx); + serializer.serializeSingleVertex(evcVtx, evc, "migrations"); + + logger.info("\t Created new evc as a child of configuration " + evcVtx + " with evc-id= " + evcVtx.value("evc-id").toString() ); + String dmaapMsg = System.nanoTime() + "_" + evcVtx.id().toString() + "_" + evcVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + +// Introspector introspector = serializer.getLatestVersionView(evcVtx); +// this.notificationHelper.addEvent(evcVtx, introspector, EventAction.CREATE, this.serializer.getURIForVertex(evcVtx, false)); +// logger.info("\t Dmaap event sent for " + evcVtx + " with evc-id = " + evcId); + }catch (Exception e) { + logger.info("\t ERROR: Failure to PUT EVC for evc-name [" + evcId + "]" ); + processedEvcsCount--; + falloutEvcsCount++; + falloutEvcsMap.put((lineNumber+1)+"", "["+evcId+"] - Failure to PUT EVC" ); + } + return evcVtx; + + } + + private String checkForNull(String s){ + if (s!= null && !s.isEmpty()){ + return s; + } + return null; + } + + + @Override + public Status getStatus() { + if (checkLog) { + return Status.CHECK_LOGS; + } + else if (success) { + return Status.SUCCESS; + } + else { + return Status.FAILURE; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{this.SERVICE_INSTANCE_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateSAREvcInventory"; + } +} diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartOne.java b/src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartOne.java new file mode 100644 index 0000000..5f74835 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartOne.java @@ -0,0 +1,343 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.javatuples.Pair; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +@Enabled +@MigrationPriority(20) +@MigrationDangerRating(100) +public class MigrateBadWidgetModelsPartOne extends EdgeSwingMigrator { + private boolean success = true; + private final GraphTraversalSource g; + private int candidateCount = 0; + private int nqEdgeCount = 0; + + // migration restrictions that we will use for this migration + private final String NODE_TYPE_RESTRICTION = "named-query-element"; + private final String EDGE_LABEL_RESTRICTION = "org.onap.relationships.inventory.IsA"; + private final String EDGE_DIR_RESTRICTION = "IN"; + + GraphTraversal<Vertex, Vertex> widgetModelTraversal; + GraphTraversal<Vertex, Vertex> widgetModelVersionTraversal; + GraphTraversal<Vertex, Vertex> validModVerTraversal; + GraphTraversal<Vertex, Vertex> widgetModelNqEdgeTraversal; + + + + public MigrateBadWidgetModelsPartOne(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{"model", "named-query-element"}); + } + + @Override + public String getMigrationName() { + return "MigrateBadWidgetModelsPartOne"; + } + + + /** + * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate. + * @return + */ + @Override + public List<Pair<Vertex, Vertex>> getAffectedNodePairs() { + + List<Pair<Vertex, Vertex>> fromToVertPairList = new ArrayList<Pair<Vertex, Vertex>>(); + ArrayList <Vertex> badModVtxList = new <Vertex> ArrayList (); + + logAndPrintInfo("--------- GET AFFECTED NODE PAIRS -------------"); + // Read the json file to populate the validWidgetModelVesionIdHash and also + // validWidgetModelInvIdHash which will be used to figure out which data is in the db with + // an invalid id. + ArrayList <String> fileLines = readInValidWidgetInfoFile(); + + // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType + // Note - we currently only have one valid version per model for widget models. + HashMap <String,String> validModelInvariantIdHash = getModelInvariantIdHash( fileLines ); + + // See what (widget) models are being used in the DB + widgetModelTraversal = this.engine.asAdmin().getTraversalSource().V() + .has("aai-node-type", "model") + .has("model-type", "widget"); + + if(!(widgetModelTraversal.hasNext())){ + logAndPrintInfo("unable to find widget models in database. "); + } + + while (widgetModelTraversal.hasNext()) { + Vertex widgetModVertexInDb = widgetModelTraversal.next(); + String invId = widgetModVertexInDb.property("model-invariant-id").value().toString(); + if( validModelInvariantIdHash.containsValue(invId) ){ + // This is a valid model, we don't need to do anything with it. + continue; + } + // For this bad widget model, need to look at the model-version node to + // find out what type of widget it is supposed to be so we can look up the correct invId. + // Note - We expect just one per model, but there could be more. + logAndPrintInfo(" Found invalid widget model-invariant-id = [" + invId + "]."); + + // We're using badModIdList to help us figure out how many bad edges go with the + // bad model nodes - which is really just for logging purposes. + badModVtxList.add(widgetModVertexInDb); + + widgetModelVersionTraversal = this.engine.asAdmin().getTraversalSource() + .V(widgetModVertexInDb) + .in("org.onap.relationships.inventory.BelongsTo") + .has("aai-node-type", "model-ver"); + + if(!(widgetModelVersionTraversal.hasNext())){ + logAndPrintInfo("unable to find widget model version in database for model-invariant-id = [" + invId + "]."); + } + + while (widgetModelVersionTraversal.hasNext()) { + Vertex widgetModVersionVertex = widgetModelVersionTraversal.next(); + String nodeType = widgetModVersionVertex.property("model-name").value().toString(); + logAndPrintInfo(" nodeType that goes with invalid widget model-invariant-id = [" + invId + "] is: [" + nodeType + "]."); + + // Now we can use the nodeType to find the correct/valid model-invariant-id to use + if( validModelInvariantIdHash.containsKey(nodeType) ){ + // We know what the model-invariant-id SHOULD be, so swing edges from the invalid node to this valid one. + String validModInvId = validModelInvariantIdHash.get(nodeType); + Iterator<Vertex> toVtxItr= + this.g.V().has("model-invariant-id",validModInvId).has(AAIProperties.NODE_TYPE, "model"); + int ct = 0; + while(toVtxItr.hasNext()) { + Vertex toValidVert = toVtxItr.next(); + ct++; + if( ct == 1 ){ + fromToVertPairList.add(new Pair<>(widgetModVertexInDb, toValidVert)); + } + else { + logAndPrintInfo("ERROR - More than one model node found for model-invariant-id = [" + validModInvId + "]."); + } + } + if( ct == 0 ){ + logAndPrintInfo("unable to find model node in database for valid model-invariant-id = [" + validModInvId + "]."); + } + } + else { + logAndPrintInfo("unable to find a valid widget model in database for model-name = [" + nodeType + "]."); + } + } + } + candidateCount = fromToVertPairList.size(); + + // For each of the bad model nodes, see how many actually have an IN edge from a named-query-element + for( int i = 0; i < badModVtxList.size(); i++ ){ + widgetModelNqEdgeTraversal = this.engine.asAdmin().getTraversalSource() + .V(badModVtxList.get(i)) + .in("org.onap.relationships.inventory.IsA") + .has("aai-node-type", "named-query-element"); + + if(widgetModelNqEdgeTraversal.hasNext()) { + nqEdgeCount++; + } + } + + return fromToVertPairList; + } + + + public String getNodeTypeRestriction(){ + return NODE_TYPE_RESTRICTION; + } + + public String getEdgeLabelRestriction(){ + return EDGE_LABEL_RESTRICTION; + } + + public String getEdgeDirRestriction(){ + return EDGE_DIR_RESTRICTION; + } + + /** + * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate. + * @return + */ + public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) { + + // Cleanup of model nodes will be done by the other migration script after the + // model-ver records have edges swung off of them. + + // We're just going to give count of how many of these edges were found. + logAndPrintInfo(" >>>> SUMMARY for Migration of named-query-element to model edges: "); + logAndPrintInfo(" >>>> Count of bad widget model nodes found: " + candidateCount ); + logAndPrintInfo(" >>>> Count of bad widget model nodes that have named-query-element edges: " + nqEdgeCount ); + + } + + private ArrayList <String> readInValidWidgetInfoFile(){ + + ArrayList <String> fileLines = new ArrayList <String> (); + String homeDir = System.getProperty("AJSC_HOME"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logAndPrintInfo("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return fileLines; + } + if (configDir == null) { + logAndPrintInfo("ERROR: Could not find sys prop BUNDLECONFIG_DIR"); + success = false; + return fileLines; + } + String fileName = homeDir + "/" + configDir + "/" + "migration-input-files/widget-model-migration-data/widget-model-migration-input.csv"; + try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { + String modelInfoLine; + while ((modelInfoLine = br.readLine()) != null) { + modelInfoLine = modelInfoLine.replace("\n", "").replace("\r", ""); + if (!modelInfoLine.isEmpty()) { + fileLines.add(modelInfoLine); + } + } + + } + catch (FileNotFoundException e) { + logger.error("ERROR: Could not find file " + fileName, e); + success = false; + } catch (IOException e) { + logger.error("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.error("encountered exception", e); + e.printStackTrace(); + success = false; + } + return fileLines; + } + + + HashMap <String,String> getModelVersionIdHash( ArrayList <String> fileLines ){ + + HashMap <String, String> versionIdHash = new HashMap <String,String> (); + + if( fileLines == null ){ + logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash"); + success = false; + return versionIdHash; + } + + for(int i = 0; i < fileLines.size(); i++ ){ + String mLine = fileLines.get(i); + String[] fields = mLine.split("\\,"); + if (fields.length != 3) { + logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line."); + success = false; + } + else { + versionIdHash.put(fields[0],fields[1]); + } + } + + // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is + // the correct model info for "virtual-data-center". Problem is that there is no vdc nodeType, but + // there are named-queries pointing at a bad widget-model for "vdc". + String virtDataCenterVerId = versionIdHash.get("virtual-data-center"); + if( virtDataCenterVerId != null ){ + versionIdHash.put("vdc",virtDataCenterVerId ); + } + + return versionIdHash; + } + + + HashMap <String,String> getModelInvariantIdHash( ArrayList <String> fileLines ){ + HashMap <String, String> invIdHash = new HashMap <String,String> (); + + if( fileLines == null ){ + logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash"); + success = false; + return invIdHash; + } + + for(int i = 0; i < fileLines.size(); i++ ){ + String mLine = fileLines.get(i); + String[] fields = mLine.split("\\,"); + if (fields.length != 3) { + logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line."); + success = false; + } + else { + invIdHash.put(fields[0],fields[2]); + } + } + + // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is + // the correct model info for "virtual-data-center". Problem is that there is no vdc nodeType, but + // there are named-queries pointing at a bad widget-model for "vdc". + String virtDataCenterInvId = invIdHash.get("virtual-data-center"); + if( invIdHash != null ){ + invIdHash.put("vdc",virtDataCenterInvId ); + } + + return invIdHash; + } + + /** + * Log and print. + * + * @param msg + * the msg + */ + protected void logAndPrintInfo(String msg) { + System.out.println(msg); + logger.info(msg); + } + + + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartTwo.java b/src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartTwo.java new file mode 100644 index 0000000..bb525c3 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartTwo.java @@ -0,0 +1,508 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.javatuples.Pair; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +@Enabled +@MigrationPriority(21) +@MigrationDangerRating(100) +public class MigrateBadWidgetModelsPartTwo extends EdgeSwingMigrator { + private boolean success = true; + private final GraphTraversalSource g; + + // NOTE -- this migration is for "model-ver" nodes only. It needs to be run AFTER + // the MigrateWidgetModelsPartOne. + // + + // migration restrictions that we will use for this migration + private final String NODE_TYPE_RESTRICTION = "model-element"; + private final String EDGE_LABEL_RESTRICTION = "org.onap.relationships.inventory.IsA"; + private final String EDGE_DIR_RESTRICTION = "IN"; + + GraphTraversal<Vertex, Vertex> widgetModelTraversal; + GraphTraversal<Vertex, Vertex> widgetModelVersionTraversal; + GraphTraversal<Vertex, Vertex> validModVerTraversal; + + + + public MigrateBadWidgetModelsPartTwo(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{"model", "model-element", "model-ver"}); + } + + @Override + public String getMigrationName() { + return "MigrateBadWidgetModelsPartTwo"; + } + + + /** + * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate. + * @return + */ + @Override + public List<Pair<Vertex, Vertex>> getAffectedNodePairs() { + logAndPrintInfo("--------- GET AFFECTED NODE PAIRS -------------"); + // Read the json file to populate the validWidgetModelVesionIdHash and also + // validWidgetModelInvIdHash which will be used to figure out which data is in the db with + // an invalid id. + ArrayList <String> fileLines = readInValidWidgetInfoFile(); + + // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType + // Note - we currently only have one valid version per model for widget models. + HashMap <String,String> validModelVersionIdHash = getModelVersionIdHash( fileLines ); + + // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType + // Note - we currently only have one valid version per model for widget models. + HashMap <String,String> validModelInvariantIdHash = getModelInvariantIdHash( fileLines ); + + // Now we will see what is actually in the DB + List<Pair<Vertex, Vertex>> fromToVertPairList = new ArrayList<Pair<Vertex, Vertex>>(); + widgetModelTraversal = this.engine.asAdmin().getTraversalSource().V() + .has("aai-node-type", "model") + .has("model-type", "widget"); + + if(!(widgetModelTraversal.hasNext())){ + logAndPrintInfo("unable to find widget models in database. "); + } + + int validModelVerCount = 0; + while (widgetModelTraversal.hasNext()) { + Vertex widgetModVertex = widgetModelTraversal.next(); + String invId = widgetModVertex.property("model-invariant-id").value().toString(); + + // Find the model-version nodes that belong to this model. + // We expect just one per model, but there could be more. + widgetModelVersionTraversal = this.engine.asAdmin().getTraversalSource() + .V(widgetModVertex) + .in("org.onap.relationships.inventory.BelongsTo") + .has("aai-node-type", "model-ver"); + + if(!(widgetModelVersionTraversal.hasNext())){ + logAndPrintInfo("unable to find widget model version in database for model-invariant-id = [" + invId + "]."); + } + + while (widgetModelVersionTraversal.hasNext()) { + Vertex widgetModVersionVertex = widgetModelVersionTraversal.next(); + String modVersionIdInDb = widgetModVersionVertex.property("model-version-id").value().toString(); + String nodeType = widgetModVersionVertex.property("model-name").value().toString(); + + if( validModelVersionIdHash.containsKey(nodeType) ){ + // We know what the model-version-id SHOULD be, so make sure we're using it. + String validModVerId = validModelVersionIdHash.get(nodeType); + if( !modVersionIdInDb.equals(validModVerId) ){ + logAndPrintInfo(" Bad model-version-id found in DB for model-name = " + nodeType + ", verId = " + modVersionIdInDb ); + validModVerTraversal = this.engine.asAdmin().getTraversalSource() + .V() + .has("model-version-id",validModVerId) + .has("aai-node-type","model-ver"); + if(!(validModVerTraversal.hasNext())){ + logAndPrintInfo("unable to find widget model version in database for valid model-version-id = [" + validModVerId + "]."); + } + int ct = 0; + while (validModVerTraversal.hasNext()) { + ct++; + if( ct > 1 ){ + logAndPrintInfo("ERROR - More than one model-ver found for model-version-id = [" + validModVerId + "]."); + break; + } + Vertex toVert = validModVerTraversal.next(); + fromToVertPairList.add(new Pair<>(widgetModVersionVertex, toVert)); + } + } + else { + validModelVerCount++; + logAndPrintInfo("Valid model-version-id used in DB for model-name = [" + nodeType + "]."); + } + } + else { + logAndPrintInfo("unable to find a valid widget model-ver in database for model-name = [" + nodeType + "]."); + } + } + } + + return fromToVertPairList; + } + + + public String getNodeTypeRestriction(){ + return NODE_TYPE_RESTRICTION; + } + + public String getEdgeLabelRestriction(){ + return EDGE_LABEL_RESTRICTION; + } + + public String getEdgeDirRestriction(){ + return EDGE_DIR_RESTRICTION; + } + + /** + * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate. + * @return + */ + public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) { + + // The first node in each pair is the model-ver that we were migrating edges AWAY FROM because + // it is an invalid model-ver node. + // Delete those as well as their parent model node (if the parent model node has no other users + // and is not on the validModelInvIdList). + + int badModelVerCount = 0; + int modelVerDelCount = 0; + int modelDelCount = 0; + int parentPreventValidDelCount = 0; + + HashMap <String,String> parentPreventInEdgeIdHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it. + HashMap <String,String> parentPreventOutEdgeIdHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it. + HashMap <String,String> parentPreventIsaEdgeDelHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it. + + ArrayList <String> fileLines = readInValidWidgetInfoFile(); + // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType + // Note - we currently only have one valid version per model for widget models. + HashMap <String,String> validModelInvariantIdHash = getModelInvariantIdHash( fileLines ); + + try { + for (Pair<Vertex, Vertex> nodePair : nodePairL) { + // The "fromNode" is the "bad/old" model-ver node that we moved off of + badModelVerCount++; + Vertex oldNode = nodePair.getValue0(); + String oldModVerId = oldNode.property("model-version-id").value().toString(); + Vertex parentModelNode = null; + + //DOUBLE CHECK THAT THIS IS NOT a valid model-version-id + + + boolean okToDelete = true; + //---- delete the oldNode if the only edge it has is its "belongsTo/OUT" edge to its parent model. + // AND if its parent node does not have any named-query edges ("IsA" edges) pointing to it. + Iterator <Edge> edgeInIter = oldNode.edges(Direction.IN); + while( edgeInIter.hasNext() ){ + Edge inE = edgeInIter.next(); + Vertex otherSideNode4ThisEdge = inE.inVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + // If there are any IN edges, we won't delete this thing. + okToDelete = false; + logAndPrintInfo("We will not delete old model-ver node because it still has IN edges. This model-version-id = [" + + oldModVerId + "], has IN edge from a [" + otherSideNodeType + "] node. "); + } + if( okToDelete ){ + // there were no OUT edges, make sure the only OUT edge is to it's parent + Iterator <Edge> edgeOutIter = oldNode.edges(Direction.OUT); + int edgeCount = 0; + while( edgeOutIter.hasNext() ){ + Edge badModVerE = edgeOutIter.next(); + edgeCount++; + if( edgeCount > 1 ){ + // If there are more than one OUT edges, we won't delete this thing. + okToDelete = false; + parentModelNode = null; + logAndPrintInfo("We will not delete old model-ver node because it still has > 1 OUT-edges. model-version-id = [" + oldModVerId + "]."); + } + else { + String eLabel = badModVerE.label().toString(); + Vertex otherSideNode4ThisEdge = badModVerE.inVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + if( ! eLabel.equals("org.onap.relationships.inventory.BelongsTo") ){ + logAndPrintInfo("We will not delete old model-ver node because it still has a non 'belongsTo' OUT-edge. model-version-id = [" + + oldModVerId + "], edgeLabel = [" + eLabel + "] edge goes to a [" + otherSideNodeType + "]. "); + okToDelete = false; + } + else { + if( ! otherSideNodeType.equals("model") ){ + logAndPrintInfo("We will not delete old model-ver node (model-version-id = [" + oldModVerId + "]) " + + " because it still has an OUT edge to a [" + otherSideNodeType + "] node. "); + okToDelete = false; + parentModelNode = null; + } + else { + parentModelNode = otherSideNode4ThisEdge; + String parentInvId = parentModelNode.property("model-invariant-id").value().toString(); + Iterator <Edge> pInIter = parentModelNode.edges(Direction.IN); + while( pInIter.hasNext() ){ + Edge inE = pInIter.next(); + String inELabel = inE.label().toString(); + if( ! inELabel.equals("org.onap.relationships.inventory.BelongsTo") ){ + Vertex otherSideNode = inE.outVertex(); + String otherSideNT = otherSideNode.value(AAIProperties.NODE_TYPE); + // If there are any IN edges still on the parent, + // we won't delete this model-ver since once the model-ver + // is gone, its hard to know what nodeType the model was + // for - so it would be hard to know what valid model-invariant-id + // to migrate its edges to. + okToDelete = false; + parentPreventIsaEdgeDelHash.put(parentInvId,""); + logAndPrintInfo("We will not delete old model-ver node because its" + + " parent model still has IN edges. The model with model-invariant-id = [" + + parentInvId + "], has an non-belongsTo IN edge, label = [" + + inELabel + "] from a [" + otherSideNT + "] node. "); + } + } + } + } + } + } + } + + if( okToDelete ){ + logAndPrintInfo(" >>> DELETEING model-ver node with model-version-id = [" + oldModVerId + "]" ); + modelVerDelCount++; + oldNode.remove(); + } + + if( parentModelNode != null && okToDelete ){ + // Delete the corresponding parent model IF it now has no + // edges anymore (and is not in our known valid model list) + // and we were deleting the model-ver also. + boolean okToDelParent = true; + String parentModInvId = parentModelNode.property("model-invariant-id").value().toString(); + + if( validModelInvariantIdHash.containsValue(parentModInvId) ){ + okToDelParent = false; + logAndPrintInfo("We will not delete old model node because it is on our valid widget list. " + + " model-invariant-id = [" + parentModInvId + "] "); + parentPreventValidDelCount++; + } + else { + Iterator <Edge> pInIter = parentModelNode.edges(Direction.IN); + while( pInIter.hasNext() ){ + Edge inE = pInIter.next(); + String inELabel = inE.label().toString(); + Vertex otherSideNode4ThisEdge = inE.outVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + // If there are any IN edges, we won't delete this thing. + okToDelParent = false; + parentPreventInEdgeIdHash.put(parentModInvId, ""); + logAndPrintInfo("We will not delete old model node (yet) because it still has IN edges. This model-invariant-id = [" + + parentModInvId + "], has IN edge, label = [" + + inELabel + "] from a [" + otherSideNodeType + "] node. "); + } + Iterator <Edge> pOutIter = parentModelNode.edges(Direction.OUT); + while( pOutIter.hasNext() ){ + Edge outE = pOutIter.next(); + String outELabel = outE.label().toString(); + Vertex otherSideNode4ThisEdge = outE.inVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + // If there are any OUT edges, we won't delete this thing. + okToDelParent = false; + parentPreventOutEdgeIdHash.put(parentModInvId, ""); + logAndPrintInfo("We will not delete old model node because it still has OUT edges. This model-invariant-id = [" + + parentModInvId + "], has OUT edge, label = [" + + outELabel + "] to a [" + otherSideNodeType + "] node. "); + } + } + + if( okToDelParent ){ + if( parentPreventInEdgeIdHash.containsKey(parentModInvId) ){ + // This parent had been prevented from being deleted until all its + // child model-ver's were deleted (it must have had more than one). + // So we can now remove it from the list of parent guys that + // could not be deleted. + parentPreventInEdgeIdHash.remove(parentModInvId); + } + logAndPrintInfo(" >>> DELETEING model node which was the parent of model-ver with model-version-id = [" + + oldModVerId + "]. This model-invariant-id = [" + parentModInvId + "]" ); + modelDelCount++; + parentModelNode.remove(); + } + } + } + + logAndPrintInfo(" >>> SUMMARY: total number of bad model-ver nodes found = " + badModelVerCount ); + logAndPrintInfo(" >>> SUMMARY: number of model-ver nodes deleted = " + modelVerDelCount ); + logAndPrintInfo(" >>> SUMMARY: number of model nodes deleted = " + modelDelCount ); + logAndPrintInfo(" >>> SUMMARY: number of model-ver nodes not deleted because their PARENT still had IsA edges = " + + parentPreventIsaEdgeDelHash.size() ); + logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they were valid = " + + parentPreventValidDelCount); + logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they had IN edges = " + + parentPreventInEdgeIdHash.size() ); + logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they had OUT edges = " + + parentPreventOutEdgeIdHash.size() ); + + + } catch (Exception e) { + logger.error("error encountered", e ); + success = false; + } + + } + + private ArrayList <String> readInValidWidgetInfoFile(){ + + ArrayList <String> fileLines = new ArrayList <String> (); + String homeDir = System.getProperty("AJSC_HOME"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logAndPrintInfo("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return fileLines; + } + if (configDir == null) { + logAndPrintInfo("ERROR: Could not find sys prop BUNDLECONFIG_DIR"); + success = false; + return fileLines; + } + String fileName = homeDir + "/" + configDir + "/" + "migration-input-files/widget-model-migration-data/widget-model-migration-input.csv"; + try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { + String modelInfoLine; + while ((modelInfoLine = br.readLine()) != null) { + modelInfoLine = modelInfoLine.replace("\n", "").replace("\r", ""); + if (!modelInfoLine.isEmpty()) { + fileLines.add(modelInfoLine); + } + } + } + catch (FileNotFoundException e) { + logger.error("ERROR: Could not find file " + fileName, e); + success = false; + } catch (IOException e) { + logger.error("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.error("encountered exception", e); + e.printStackTrace(); + success = false; + } + return fileLines; + } + + + HashMap <String,String> getModelVersionIdHash( ArrayList <String> fileLines ){ + + HashMap <String, String> versionIdHash = new HashMap <String,String> (); + + if( fileLines == null ){ + logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash"); + success = false; + return versionIdHash; + } + + for(int i = 0; i < fileLines.size(); i++ ){ + String mLine = fileLines.get(i); + String[] fields = mLine.split("\\,"); + if (fields.length != 3) { + logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line."); + success = false; + } + else { + versionIdHash.put(fields[0],fields[1]); + } + } + + // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is + // the correct model info for "virtual-data-center". Problem is that there is no vdc nodeType, but + // there are named-queries pointing at a bad widget-model for "vdc". + String virtDataCenterVerId = versionIdHash.get("virtual-data-center"); + if( virtDataCenterVerId != null ){ + versionIdHash.put("vdc",virtDataCenterVerId ); + } + + return versionIdHash; + } + + + HashMap <String,String> getModelInvariantIdHash( ArrayList <String> fileLines ){ + HashMap <String, String> invIdHash = new HashMap <String,String> (); + + if( fileLines == null ){ + logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash"); + success = false; + return invIdHash; + } + + for(int i = 0; i < fileLines.size(); i++ ){ + String mLine = fileLines.get(i); + String[] fields = mLine.split("\\,"); + if (fields.length != 3) { + logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line."); + success = false; + } + else { + invIdHash.put(fields[0],fields[2]); + } + } + + // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is + // the correct model info for "virtual-data-center". Problem is that there is no vdc nodeType, but + // there are named-queries pointing at a bad widget-model for "vdc". + String virtDataCenterInvId = invIdHash.get("virtual-data-center"); + if( invIdHash != null ){ + invIdHash.put("vdc",virtDataCenterInvId ); + } + return invIdHash; + } + + /** + * Log and print. + * + * @param msg + * the msg + */ + protected void logAndPrintInfo(String msg) { + System.out.println(msg); + logger.info(msg); + } + + + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateEdgesBetweenVnfcAndVfModule.java b/src/main/java/org/onap/aai/migration/v13/MigrateEdgesBetweenVnfcAndVfModule.java new file mode 100644 index 0000000..3e09c51 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateEdgesBetweenVnfcAndVfModule.java @@ -0,0 +1,83 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +/*- +* ============LICENSE_START======================================================= +* org.openecomp.aai +* ================================================================================ +* Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +* ================================================================================ +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +* */ + +package org.onap.aai.migration.v13; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import org.javatuples.Pair; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.EdgeMigrator; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +@MigrationPriority(10) +@MigrationDangerRating(100) +@Enabled +public class MigrateEdgesBetweenVnfcAndVfModule extends EdgeMigrator { + + public MigrateEdgesBetweenVnfcAndVfModule(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.empty(); + } + + @Override + public List<Pair<String, String>> getAffectedNodePairTypes() { + logger.info("Starting migration to update edge properties between vf-module and vnfc...."); + List<Pair<String, String>> nodePairList = new ArrayList<Pair<String, String>>(); + nodePairList.add(new Pair<>("vf-module", "vnfc")); + return nodePairList; + } + + @Override + public String getMigrationName() { + return "migrate-edge-vnfc-and-vf-module"; + } +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateForwarderEvcCircuitId.java b/src/main/java/org/onap/aai/migration/v13/MigrateForwarderEvcCircuitId.java new file mode 100644 index 0000000..3f90934 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateForwarderEvcCircuitId.java @@ -0,0 +1,297 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(26) +@MigrationDangerRating(100) +@Enabled +public class MigrateForwarderEvcCircuitId extends Migrator { + + private final String PNF_NODE_TYPE = "pnf"; + private final String PROPERTY_PNF_NAME = "pnf-name"; + private final String PROPERTY_INTERFACE_NAME = "interface-name"; + private final String PROPERTY_FORWARDER_ROLE = "forwarder-role"; + private final String VALUE_INGRESS = "ingress"; + private final String PROPERTY_SEQUENCE = "sequence"; + private final int VALUE_EXPECTED_SEQUENCE = 1; + private final String FORWARDER_EVC_NODE_TYPE = "forwarder-evc"; + private final String PROPERTY_CIRCUIT_ID = "circuit-id"; + + private static boolean success = true; + private static boolean checkLog = false; + private static GraphTraversalSource g = null; + private int headerLength; + private int migrationSuccess = 0; + private int migrationFailure = 0; + + private static List<String> dmaapMsgList = new ArrayList<String>(); + private static final String homeDir = System.getProperty("AJSC_HOME"); + + protected class CircuitIdFileData { + String pnfName; + String interfaceName; + + String oldCircuitId; + String newCircuitId; + + public String getPnfName() { + return pnfName; + } + public void setPnfName(String pnfName) { + this.pnfName = pnfName; + } + public String getInterfaceName() { + return interfaceName; + } + public void setInterfaceName(String interfaceName) { + this.interfaceName = interfaceName; + } + + public String getOldCircuitId() { + return oldCircuitId; + } + public void setOldCircuitId(String oldCircuitId) { + this.oldCircuitId = oldCircuitId; + } + public String getNewCircuitId() { + return newCircuitId; + } + public void setNewCircuitId(String newCircutId) { + this.newCircuitId = newCircutId; + } + } + + private static ArrayList<CircuitIdFileData> circuitIdList = new ArrayList<CircuitIdFileData>(); + + public MigrateForwarderEvcCircuitId(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void run() { + logger.info("---------- Start migration ----------"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info(this.MIGRATION_ERROR + "ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + + String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/sarea-inventory/"; + int fileLineCounter = 0; + String fileName = feedDir+ "circuitIds.csv"; + logger.info(fileName); + logger.info("---------- Processing Entries from file ----------"); + try { + List<String> lines = Files.readAllLines(Paths.get(fileName)); + Iterator<String> lineItr = lines.iterator(); + while (lineItr.hasNext()){ + String line = lineItr.next().replace("\n", "").replace("\r", ""); + if (!line.isEmpty()) { + if (fileLineCounter != 0) { + String[] colList = line.split("\\s*,\\s*", -1); + CircuitIdFileData lineData = new CircuitIdFileData(); + lineData.setPnfName(colList[0].replaceAll("^\"|\"$", "") + .replaceAll("[\t\n\r]+", "").trim()); + lineData.setInterfaceName(colList[1].replaceAll("^\"|\"$", "") + .replaceAll("[\t\n\r]+", "").trim()); + lineData.setOldCircuitId(colList[2].replaceAll("^\"|\"$", "") + .replaceAll("[\t\n\r]+", "").trim()); + lineData.setNewCircuitId(colList[4].replaceAll("^\"|\"$", "") + .replaceAll("[\t\n\r]+", "").trim()); + circuitIdList.add(lineData); + + } else { + this.headerLength = line.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength + "\n"); + if (this.headerLength != 6){ + logger.info(this.MIGRATION_ERROR + "ERROR: Input file should have 6 columns"); + this.success = false; + return; + } + } + } + fileLineCounter++; + } + updateCircuitIdCount(); + logger.info ("\n \n ******* Final Summary for Circuit Id Migration ********* \n"); + logger.info(this.MIGRATION_SUMMARY_COUNT + "CircuitIds processed: "+ migrationSuccess); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Total Rows Count: "+(fileLineCounter + 1)); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Unprocessed CircuitIds : "+ migrationFailure +"\n"); + + } catch (FileNotFoundException e) { + logger.info(this.MIGRATION_ERROR + "ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + checkLog = true; + } catch (IOException e) { + logger.info(this.MIGRATION_ERROR + "ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info(this.MIGRATION_ERROR + "encountered exception", e); + e.printStackTrace(); + success = false; + } + } + + private void updateCircuitIdCount() { + int numberOfLines = circuitIdList.size(); + for(int i = 0; i < numberOfLines; i ++) { + GraphTraversal<Vertex, Vertex> nodeList = g.V().has(this.PROPERTY_PNF_NAME, circuitIdList.get(i).getPnfName()) + .has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE).in("tosca.relationships.network.BindsTo") + .has(this.PROPERTY_INTERFACE_NAME, circuitIdList.get(i).getInterfaceName()).in("org.onap.relationships.inventory.ForwardsTo") + .has(this.PROPERTY_FORWARDER_ROLE, this.VALUE_INGRESS).has(this.PROPERTY_SEQUENCE, this.VALUE_EXPECTED_SEQUENCE) + .out("org.onap.relationships.inventory.Uses").in("org.onap.relationships.inventory.BelongsTo"); + if(!nodeList.hasNext()) { + logger.info(this.MIGRATION_ERROR + "ERROR: Failure to update Circuit Id " + circuitIdList.get(i).getOldCircuitId() + + " to " + circuitIdList.get(i).getNewCircuitId() + " Graph Traversal failed \n"); + migrationFailure++; + } + while (nodeList.hasNext()) { + Vertex forwarderEvcVtx = nodeList.next(); + boolean updateSuccess = false; + if (forwarderEvcVtx != null) { + logger.info("forwarder-evc-id is " + forwarderEvcVtx.value("forwarder-evc-id")); + if(forwarderEvcVtx.property(PROPERTY_CIRCUIT_ID).isPresent() && + forwarderEvcVtx.value(PROPERTY_CIRCUIT_ID).equals(circuitIdList.get(i).getNewCircuitId())) { + logger.info("Skipping Record: Old Collector CircuitId " + forwarderEvcVtx.value(PROPERTY_CIRCUIT_ID) + + " is the same as New Collector CircuitId " + circuitIdList.get(i).getNewCircuitId() + "\n"); + migrationFailure++; + } + else if(!circuitIdList.get(i).getNewCircuitId().isEmpty() && + forwarderEvcVtx.property(PROPERTY_CIRCUIT_ID).isPresent() && + circuitIdList.get(i).getOldCircuitId().equals(forwarderEvcVtx.value(PROPERTY_CIRCUIT_ID))) + { + try { + forwarderEvcVtx.property(PROPERTY_CIRCUIT_ID, circuitIdList.get(i).getNewCircuitId()); + this.touchVertexProperties(forwarderEvcVtx, false); + updateSuccess = true; + + } catch (Exception e) { + logger.info(e.toString()); + logger.info(this.MIGRATION_ERROR + "ERROR: Failure to update Circuit Id " + circuitIdList.get(i).getOldCircuitId() + + " to " + circuitIdList.get(i).getNewCircuitId() + "\n"); + migrationFailure++; + + } + if(updateSuccess) { + String dmaapMsg = System.nanoTime() + "_" + forwarderEvcVtx.id().toString() + "_" + + forwarderEvcVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + logger.info("Update of Circuit Id " + circuitIdList.get(i).getOldCircuitId() + " to " + + circuitIdList.get(i).getNewCircuitId() + " successful \n"); + migrationSuccess++; + } + } + else if(!forwarderEvcVtx.property(PROPERTY_CIRCUIT_ID).isPresent()) + { + logger.info(this.MIGRATION_ERROR + "ERROR: Old Collector Circuit Id not found " + circuitIdList.get(i).getOldCircuitId() + + " was not updated to " + circuitIdList.get(i).getNewCircuitId() + "\n"); + migrationFailure++; + } + else { + logger.info(this.MIGRATION_ERROR + "ERROR: Failure to update Circuit Id " + circuitIdList.get(i).getOldCircuitId() + + " to " + circuitIdList.get(i).getNewCircuitId() + "\n"); + migrationFailure++; + } + } + } + } + + } + + @Override + public Status getStatus() { + if (checkLog) { + return Status.CHECK_LOGS; + } + else if (success) { + return Status.SUCCESS; + } + else { + return Status.FAILURE; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{this.FORWARDER_EVC_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateForwarderEvcCircuitId"; + } +} diff --git a/src/main/java/org/onap/aai/migration/v14/MigrateGenericVnfMgmtOptions.java b/src/main/java/org/onap/aai/migration/v14/MigrateGenericVnfMgmtOptions.java new file mode 100644 index 0000000..d32ce81 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v14/MigrateGenericVnfMgmtOptions.java @@ -0,0 +1,103 @@ +/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v14;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Status;
+import org.onap.aai.migration.ValueMigrator;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(1)
+@MigrationDangerRating(1)
+@Enabled
+public class MigrateGenericVnfMgmtOptions extends ValueMigrator {
+
+ protected static final String VNF_NODE_TYPE = "generic-vnf";
+
+
+ private static Map<String, Map> map;
+ private static Map<String, String> pair1;
+ private static Map<String, List<String>> conditionsMap;
+
+ public MigrateGenericVnfMgmtOptions(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setMgmtOptions(), setConditionsMap(), false);
+
+ }
+
+ private static Map<String, Map> setMgmtOptions(){
+ map = new HashMap<>();
+ pair1 = new HashMap<>();
+
+ pair1.put("management-option", "AT&T Managed-Basic");
+ map.put("generic-vnf", pair1);
+
+ return map;
+ }
+
+
+
+ public static Map<String, List<String>> setConditionsMap() {
+ List<String> conditionsList = new ArrayList<String>();
+ conditionsMap = new HashMap<>();
+
+ conditionsList.add("HN");
+ conditionsList.add("HP");
+ conditionsList.add("HG");
+
+ conditionsMap.put("vnf-type", conditionsList);
+
+ return conditionsMap;
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{VNF_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateGenericVnfMgmtOptions";
+ }
+
+ @Override
+ public boolean isUpdateDmaap(){
+ return true;
+ }
+
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v14/MigrateMissingFqdnOnPservers.java b/src/main/java/org/onap/aai/migration/v14/MigrateMissingFqdnOnPservers.java new file mode 100644 index 0000000..77ecc7e --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v14/MigrateMissingFqdnOnPservers.java @@ -0,0 +1,142 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v14; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(20) +@MigrationDangerRating(2) +@Enabled +public class MigrateMissingFqdnOnPservers extends Migrator { + + protected static final String PSERVER_NODE_TYPE = "pserver"; + protected static final String PSERVER_FQDN = "fqdn"; + protected static final String PSERVER_HOSTNAME = "hostname"; + protected static final String PSERVER_SOURCEOFTRUTH = "source-of-truth"; + + private boolean success = true; + private GraphTraversalSource g = null; + + protected final AtomicInteger falloutRowsCount = new AtomicInteger(0); + + public MigrateMissingFqdnOnPservers(TransactionalGraphEngine engine, LoaderFactory loaderFactory, + EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + public void run() { + logger.info("---------- Start Updating fqdn for pserver ----------"); + + try { + int pserverCount = 0; + int pserverUpdatedCount = 0; + int pserverSkippedCount = 0; + int pserverErrorCount = 0; + int pserverWithMissingSOTCount = 0; + + GraphTraversal<Vertex, Vertex> pserverList = this.engine.asAdmin().getTraversalSource().V() + .has(AAIProperties.NODE_TYPE, PSERVER_NODE_TYPE).union(__.hasNot(PSERVER_FQDN),__.has(PSERVER_FQDN,""));//gets list of pservers with missing and empty fqdn + + while (pserverList.hasNext()) { + pserverCount++; + Vertex vertex = pserverList.next(); + String hostname = null; + String sourceOfTruth = null; + hostname = vertex.property(PSERVER_HOSTNAME).value().toString(); + + if(vertex.property(PSERVER_SOURCEOFTRUTH).isPresent()) { + sourceOfTruth = vertex.property(PSERVER_SOURCEOFTRUTH).value().toString(); + }else { + logger.info("Missing source of truth for hostname : " + hostname); + pserverWithMissingSOTCount++; + } + + if (!hostname.contains(".")) { + logger.info("Invalid format hostname :" + hostname + " and its source of truth is : " + sourceOfTruth); + pserverSkippedCount++; + continue; + } + + try { + vertex.property(PSERVER_FQDN, hostname); + this.touchVertexProperties(vertex, false); + logger.info("Updated fqdn from hostname : " + hostname + " and its source of truth is : " + sourceOfTruth); + pserverUpdatedCount++; + } catch (Exception e) { + success = false; + pserverErrorCount++; + logger.error(MIGRATION_ERROR + "encountered exception for fqdn update for pserver with hostname :" + hostname + + " and source of truth : " + sourceOfTruth, e); + } + } + + logger.info("\n \n ******* Final Summary of Updated fqdn for pserver Migration ********* \n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Total Number of pservers with missing or empty fqdn : "+pserverCount + "\n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Number of pservers updated: " + pserverUpdatedCount + "\n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Number of pservers invalid: " + pserverSkippedCount + "\n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Number of pservers failed to update due to error : " + pserverErrorCount + "\n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Number of pservers with missing source of truth: " + pserverWithMissingSOTCount + "\n"); + + } catch (Exception e) { + logger.info("encountered exception", e); + success = false; + } + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[] { PSERVER_NODE_TYPE }); + } + + @Override + public String getMigrationName() { + return "MigrateMissingFqdnOnPserver"; + } + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v14/MigrateNetworkTechToCloudRegion.java b/src/main/java/org/onap/aai/migration/v14/MigrateNetworkTechToCloudRegion.java new file mode 100644 index 0000000..afdea57 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v14/MigrateNetworkTechToCloudRegion.java @@ -0,0 +1,172 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v14; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + +@MigrationPriority(20) +@MigrationDangerRating(2) +@Enabled +public class MigrateNetworkTechToCloudRegion extends Migrator{ + + protected static final String CLOUDREGION_NODETYPE = "cloud-region"; + protected static final String CLOUD_OWNER = "cloud-owner"; + protected static final String NETWORK_TECHNOLOGY_NODETYPE = "network-technology"; + protected static final String NETWORK_TECHNOLOGY_ID = "network-technology-id"; + protected static final String NETWORK_TECHNOLOGY_NAME = "network-technology-name"; + + + private boolean success = true; + + private static List<String> dmaapMsgList = new ArrayList<String>(); + + + public MigrateNetworkTechToCloudRegion(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + public void run() { + + List<Vertex> cloudRegionVertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, CLOUDREGION_NODETYPE).has(CLOUD_OWNER,"att-aic").toList(); + logger.info("Number of cloud-region with cloud-owner att-aic : " + cloudRegionVertextList.size()); + createEdges(cloudRegionVertextList, "CONTRAIL"); + createEdges(cloudRegionVertextList, "AIC_SR_IOV"); + + cloudRegionVertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, CLOUDREGION_NODETYPE).has(CLOUD_OWNER,"att-nc").toList(); + logger.info("Number of cloud-region with cloud-owner att-nc : " + cloudRegionVertextList.size()); + createEdges(cloudRegionVertextList, "OVS"); + createEdges(cloudRegionVertextList, "STANDARD-SR-IOV"); + + } + + private void createEdges(List<Vertex> sourceVertexList, String networkTechName) + { + int networkTechEdgeCount = 0; + int networkTechEdgeErrorCount = 0; + + List<Vertex> networkTechVertexList = this.engine.asAdmin().getTraversalSource().V() + .has(AAIProperties.NODE_TYPE, NETWORK_TECHNOLOGY_NODETYPE).has(NETWORK_TECHNOLOGY_NAME, networkTechName) + .toList(); + + logger.info("---------- Start Creating an Edge from cloud-region to network-technology nodes with network-technology-name " + networkTechName + " ----------"); + + for (Vertex cloudRegionVertex : sourceVertexList) { + + try { + + for (Vertex networkVertex : networkTechVertexList) { + if (networkVertex != null) { + boolean edgePresent = false; + // Check if edge already exists for each of the source vertex + List<Vertex> outVertexList = this.engine.asAdmin().getTraversalSource().V(cloudRegionVertex) + .out().has(AAIProperties.NODE_TYPE, NETWORK_TECHNOLOGY_NODETYPE) + .has(NETWORK_TECHNOLOGY_NAME, networkTechName).has(NETWORK_TECHNOLOGY_ID, + networkVertex.property(NETWORK_TECHNOLOGY_ID).value().toString()) + .toList(); + Iterator<Vertex> vertexItr = outVertexList.iterator(); + if (outVertexList != null && !outVertexList.isEmpty() && vertexItr.hasNext()) { + logger.info("\t Edge already exists from " + CLOUDREGION_NODETYPE + " with " + CLOUD_OWNER + + " and cloud-region-id " + + cloudRegionVertex.property("cloud-region-id").value().toString() + " to " + + NETWORK_TECHNOLOGY_NODETYPE + " nodes with " + NETWORK_TECHNOLOGY_NAME + " " + + networkTechName); + edgePresent = true; + continue; + } + // Build edge from vertex to modelVerVertex + if (!edgePresent) { + this.createCousinEdge(cloudRegionVertex, networkVertex); + updateDmaapList(cloudRegionVertex); + networkTechEdgeCount++; + } + } else { + networkTechEdgeErrorCount++; + logger.info("\t" + MIGRATION_ERROR + "Unable to create edge from " + CLOUDREGION_NODETYPE + + " with " + CLOUD_OWNER + " to " + NETWORK_TECHNOLOGY_NODETYPE + " nodes with " + + NETWORK_TECHNOLOGY_NAME + " " + networkTechName); + + } + } + } catch (Exception e) { + success = false; + networkTechEdgeErrorCount++; + logger.error("\t" + MIGRATION_ERROR + "encountered exception from " + NETWORK_TECHNOLOGY_NODETYPE + + " node when trying to create edge to " + CLOUDREGION_NODETYPE, e); + } + } + + logger.info("\n \n ******* Summary " + NETWORK_TECHNOLOGY_NODETYPE + " Nodes: Finished creating an Edge from " + + CLOUDREGION_NODETYPE + " with " + CLOUD_OWNER + " to " + NETWORK_TECHNOLOGY_NODETYPE + " nodes with " + + NETWORK_TECHNOLOGY_NAME + " " + networkTechName + " ********* \n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Number of edges created from cloud-region to "+networkTechName +" network-technology : " + networkTechEdgeCount + "\n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Number of edges failed from cloud-region to "+networkTechName +" network-technology : " + networkTechEdgeErrorCount + "\n"); + + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{NETWORK_TECHNOLOGY_NODETYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateNetworkTech"; + } + + private void updateDmaapList(Vertex v){ + String dmaapMsg = System.nanoTime() + "_" + v.id().toString() + "_" + v.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + logger.info("\tAdding Updated "+ CLOUDREGION_NODETYPE +" Vertex " + v.id().toString() + " to dmaapMsgList...."); + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + } + +} diff --git a/src/main/java/org/onap/aai/migration/v14/MigrateSameSourcedRCTROPserverData.java b/src/main/java/org/onap/aai/migration/v14/MigrateSameSourcedRCTROPserverData.java new file mode 100644 index 0000000..d0c1e15 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v14/MigrateSameSourcedRCTROPserverData.java @@ -0,0 +1,576 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v14; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.*; +import org.javatuples.Pair; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.enums.AAIDirection; +import org.onap.aai.edges.enums.EdgeProperty; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.onap.aai.setup.SchemaVersions; +import org.springframework.web.util.UriUtils; + +import javax.ws.rs.core.UriBuilder; + +import java.io.UnsupportedEncodingException; +import java.util.*; +import java.util.stream.Collectors; + +@Enabled +@MigrationPriority(5) +@MigrationDangerRating(100) +public class MigrateSameSourcedRCTROPserverData extends EdgeSwingMigrator { + /** + * Instantiates a new migrator. + * + * @param engine + */ + private final String PARENT_NODE_TYPE = "pserver"; + private boolean success = true; + protected Set<Object> seen = new HashSet<>(); + private Map<String, UriBuilder> nodeTypeToUri; + private Map<String, Set<String>> nodeTypeToKeys; + private static List<String> dmaapMsgList = new ArrayList<String>(); + private static List<Introspector> dmaapDeleteList = new ArrayList<Introspector>(); + Vertex complexFromOld; + private static int dupROCount = 0; + private static int roPserversUpdatedCount = 0; + private static int roPserversDeletedCount = 0; + private static int dupRctCount = 0; + private static int rctPserversUpdatedCount = 0; + private static int rctPserversDeletedCount = 0; + + public MigrateSameSourcedRCTROPserverData(TransactionalGraphEngine engine , LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + createDmaapFilesForDelete(dmaapDeleteList); + } + + @Override + public List<Pair<Vertex, Vertex>> getAffectedNodePairs() { + return null; + } + + @Override + public String getNodeTypeRestriction() { + return null; + } + + @Override + public String getEdgeLabelRestriction() { + return null; + } + + @Override + public String getEdgeDirRestriction() { + return null; + } + + @Override + public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) { + + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{"lag-interface", "l-interface", "l3-interface-ipv4-address", "l3-interface-ipv6-address", "sriov-vf", "vlan", "p-interface", "sriov-pf"}); + } + + @Override + public String getMigrationName() { + return "MigrateCorrectRCTSourcedPserverData"; + } + + @Override + public void run() { + + + nodeTypeToUri = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect( + Collectors.toMap( + e -> e.getKey(), + e -> UriBuilder.fromPath(e.getValue().getFullGenericURI().replaceAll("\\{"+ e.getKey() + "-", "{")) + )); + + nodeTypeToKeys = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect( + Collectors.toMap( + e -> e.getKey(), + e -> e.getValue().getKeys() + )); + + List<Vertex> pserverTraversalRCT = graphTraversalSource().V().has("aai-node-type", "pserver").has("source-of-truth", P.within("RCT", "AAIRctFeed")).toList(); + int rctCount = pserverTraversalRCT.size(); + + try { + logger.info("RCT pserver count: "+rctCount); + updateToLatestRCT(pserverTraversalRCT); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } catch (AAIException e) { + e.printStackTrace(); + } + + List<Vertex> pserverTraversalRO = graphTraversalSource().V().has("aai-node-type", "pserver").has("source-of-truth", P.within("RO", "AAI-EXTENSIONS")).toList(); + int roCount = pserverTraversalRO.size(); + try { + logger.info("RO pserver count: "+roCount); + updateToLatestRO(pserverTraversalRO); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } catch (AAIException e) { + e.printStackTrace(); + } + + logger.info ("\n \n ******* Migration Summary Counts for RCT and RO sourced pservers in A&AI ********* \n"); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Total number of RCT pservers: " +rctCount); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Duplicate RCT pserver count: "+ dupRctCount); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Number of RCT updated: "+ rctPserversUpdatedCount); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Number of RCT deleted: "+ rctPserversDeletedCount +"\n"); + + logger.info(this.MIGRATION_SUMMARY_COUNT + "Total number of RO pservers: " +roCount); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Duplicate RO pserver count: "+ dupROCount); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Number of RO updated: "+ roPserversUpdatedCount); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Number of RO deleted: "+ roPserversDeletedCount +"\n"); + } + + public void updateToLatestRO(List<Vertex> list) throws UnsupportedEncodingException, AAIException { + List<Vertex> removeROList = new ArrayList<>(); + + Vertex latestV = null; + + for(int i=0;i<list.size();i++){ + Vertex currV = list.get(i); + + if (removeROList.contains(currV)){ + logger.info("RO Pserver: "+currV.property("hostname").value().toString() + "was already added to delete list. No further processing needed for this."); + continue; + } + logger.info("RO Pserver: "+currV.property("hostname").value().toString()); + + for(int j=i+1; j<list.size();j++) { + + Vertex temp = list.get(j); + + String[] currentVHostname = currV.property("hostname").value().toString().split("\\."); + String[] tempHostname = temp.property("hostname").value().toString().split("\\."); + + if (currentVHostname.length >0 && tempHostname.length > 0){ + if (!currentVHostname[0].isEmpty() && !tempHostname[0].isEmpty() && currentVHostname[0].equals(tempHostname[0])) { + dupROCount++; + logger.info("\tTemp RO Pserver: "+temp.property("hostname").value().toString()); + if (temp.property("hostname").value().toString().length() > currV.property("hostname").value().toString().length()) { + //temp is the latest vertex swing everything from currV to temp + latestV = temp; + movePlink(currV, latestV); + moveLagInterfaces(currV, latestV); + swingEdges(currV, latestV, null, null, "BOTH"); + modifyChildrenUri(latestV); + String dmaapMsg = System.nanoTime() + "_" + temp.id().toString() + "_" + temp.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + roPserversUpdatedCount++; + logger.info("\tAdding pserver "+latestV.property("hostname").value().toString() + " to updated list"); + if (!removeROList.contains(list.get(i))) { + removeROList.add(list.get(i)); + Introspector obj = serializer.getLatestVersionView(currV);//currV + logger.info("\tAdding pserver "+currV.property("hostname").value().toString() + " to delete list"); + dmaapDeleteList.add(obj); + roPserversDeletedCount++; + } + currV = latestV; + } else { + //currV is the latest temp is the old vertex swing everything from temp to currV + latestV = currV; + movePlink(temp, latestV); + moveLagInterfaces(temp, latestV); + swingEdges(temp, latestV, null, null, "BOTH"); + modifyChildrenUri(latestV); + String dmaapMsg = System.nanoTime() + "_" + currV.id().toString() + "_" + currV.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + logger.info("\tAdding pserver "+latestV.property("hostname").value().toString() + " to updated list"); + roPserversUpdatedCount++; + + if (!removeROList.contains(list.get(j))) { + removeROList.add(list.get(j)); + Introspector obj = serializer.getLatestVersionView(temp);//temp + logger.info("\tAdding pserver "+temp.property("hostname").value().toString() + " to delete list"); + dmaapDeleteList.add(obj); + roPserversDeletedCount++; + } + } + } + } + } + } + logger.info("\tCount of RO Pservers removed = "+removeROList.size()+"\n"); + removeROList.forEach(v ->v.remove()); + + } + +// public void addComplexEdge(Vertex Latest) throws AAIException { +// +// if(!(graphTraversalSource().V(Latest).has("aai-node-type", "pserver").out("org.onap.relationships.inventory.LocatedIn").has("aai-node-type","complex").hasNext())){ +// if (complexFromOld != null) +// createCousinEdge(Latest,complexFromOld); +// +// } +// } + + +// public void dropComplexEdge(Vertex old){ +// List<Vertex> locatedInEdgeVertexList = graphTraversalSource().V(old).has("aai-node-type", "pserver").out("org.onap.relationships.inventory.LocatedIn").has("aai-node-type","complex").toList(); +// if (locatedInEdgeVertexList != null && !locatedInEdgeVertexList.isEmpty()){ +// Iterator<Vertex> locatedInEdgeVertexListItr = locatedInEdgeVertexList.iterator(); +// while (locatedInEdgeVertexListItr.hasNext()){ +// complexFromOld = locatedInEdgeVertexListItr.next(); +// if ("complex".equalsIgnoreCase(complexFromOld.property("aai-node-type").value().toString())){ +// Edge pserverToComplexEdge = complexFromOld.edges(Direction.IN, "org.onap.relationships.inventory.LocatedIn").next(); +// pserverToComplexEdge.remove(); +// } +// } +// } +// } + + + private GraphTraversalSource graphTraversalSource() { + return this.engine.asAdmin().getTraversalSource(); + } + + public void updateToLatestRCT(List<Vertex> list) throws UnsupportedEncodingException, AAIException { + List<Vertex>removeRCTList = new ArrayList<>(); + + Vertex latestV = null; + for(int i=0;i<list.size();i++) { + Vertex currV = list.get(i); + if (!currV.property("fqdn").isPresent()){ + continue; + } + + if (removeRCTList.contains(currV)){ + logger.info("RCT Pserver: "+currV.property("hostname").value().toString() + "was already added to delete list. No further processing needed for this."); + continue; + } + logger.info("RCT Pserver: "+currV.property("hostname").value().toString()); + for(int j=i+1;j<list.size();j++) { + + Vertex temp = list.get(j); + if (temp.property("fqdn").isPresent()) { + String[] currentVFqdn = currV.property("fqdn").value().toString().split("\\."); + String[] tempFqdn = temp.property("fqdn").value().toString().split("\\."); + if (currentVFqdn.length >0 && tempFqdn.length > 0){ + String currentFqdnFirstToken = currentVFqdn[0]; + String tempFqdnFirstToken = tempFqdn[0]; + if (!currentFqdnFirstToken.isEmpty() && !tempFqdnFirstToken.isEmpty() && currentFqdnFirstToken.equals(tempFqdnFirstToken)) { + dupRctCount++; + logger.info("\tMatching Temp RCT Pserver: "+temp.property("hostname").value().toString()); + long tempRV = Long.parseLong(temp.value("resource-version")); + long currRV = Long.parseLong(currV.value("resource-version")); + logger.info("\tcurrRV: "+currRV+ ", tempRV: "+tempRV); + if (Long.parseLong(temp.value("resource-version")) > Long.parseLong(currV.value("resource-version"))) { + //currv is old, temp vertex found in traversal is the latest + latestV = temp; + movePlink(currV, latestV); + moveLagInterfaces(currV, latestV); + swingEdges(currV, latestV, null, null, "BOTH"); + modifyChildrenUri(latestV); + String dmaapMsg = System.nanoTime() + "_" + temp.id().toString() + "_" + temp.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + rctPserversUpdatedCount++; + logger.info("\tAdding pserver "+latestV.property("hostname").value().toString() + " to updated list"); + if (!removeRCTList.contains(list.get(i))) { + removeRCTList.add(list.get(i)); + Introspector obj = serializer.getLatestVersionView(currV); + logger.info("\tAdding pserver "+currV.property("hostname").value().toString() + " to delete list"); + dmaapDeleteList.add(obj); + rctPserversDeletedCount++; + } + currV = latestV; + } else { + //currv Is the latest, temp vertex found is an older version + latestV = currV; + movePlink(temp, latestV); + moveLagInterfaces(temp, latestV); + swingEdges(temp, latestV, null, null, "BOTH"); + modifyChildrenUri(latestV); + String dmaapMsg = System.nanoTime() + "_" + currV.id().toString() + "_" + currV.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + rctPserversUpdatedCount++; + logger.info("\tAdding pserver "+latestV.property("hostname").value().toString() + " to updated list"); + if (!removeRCTList.contains(list.get(j))) { + removeRCTList.add(list.get(j)); + Introspector obj = serializer.getLatestVersionView(temp); + logger.info("\tAdding pserver "+temp.property("hostname").value().toString() + " to delete list"); + dmaapDeleteList.add(obj); + rctPserversDeletedCount++; + } + } + + } + } + } + } + } + logger.info("\tCount of RCT Pservers removed = "+removeRCTList.size() +"\n"); + removeRCTList.forEach((r)-> r.remove()); + + } + + + public void movePlink(Vertex old, Vertex latest) throws AAIException { + + List<Vertex> pInterfacesOnOldPserver = graphTraversalSource().V(old).has("aai-node-type","pserver").in("tosca.relationships.network.BindsTo").has("aai-node-type","p-interface").toList(); + List<Vertex> pInterfacesOnLatestPserver = graphTraversalSource().V(latest).has("aai-node-type","pserver").in("tosca.relationships.network.BindsTo").has("aai-node-type","p-interface").toList(); + // SCENARIO 1 = no match found move everything from pserver old to new in swing edges call outside this fcn + + if(pInterfacesOnLatestPserver.size() == 0){ + logger.info("\tNo P-interfaces found on "+latest.property("hostname").value().toString()+ "..."); + if(pInterfacesOnOldPserver.size() != 0) { + logger.info("\tP-interfaces found on "+old.property("hostname").value().toString()+ ". Update plink name and move the p-interfaces to latest pserver."); + for (int i = 0; i < pInterfacesOnOldPserver.size(); i++) { + if (graphTraversalSource().V(pInterfacesOnOldPserver.get(i)).has("aai-node-type", "p-interface").out("tosca.relationships.network.LinksTo").hasNext()) { + Vertex oldPlink = graphTraversalSource().V(pInterfacesOnOldPserver.get(i)).has("aai-node-type", "p-interface").out("tosca.relationships.network.LinksTo").next(); + String linkName = oldPlink.property("link-name").value().toString(); + logger.info("\tPhysical-link "+linkName+ " found on "+graphTraversalSource().V(pInterfacesOnOldPserver.get(i).property("interface-name").value().toString())); + linkName = linkName.replaceAll(old.property("hostname").value().toString(), latest.property("hostname").value().toString()); + String[] PlinkBarSplit = linkName.split("\\|"); + if (PlinkBarSplit.length > 1) { + modifyPlinkName(oldPlink, linkName, old); + } + } + } + } + + return; + } + + for(int i=0; i<pInterfacesOnOldPserver.size();i++){ + for(int j=0; j<pInterfacesOnLatestPserver.size(); j++){ + Vertex oldPinterface = graphTraversalSource().V(pInterfacesOnOldPserver.get(i)).has("aai-node-type","p-interface").next(); + //pinterfaces are the same + if(pInterfacesOnOldPserver.get(i).property("interface-name").value().toString().equals(pInterfacesOnLatestPserver.get(j).property("interface-name").value().toString())){ + Vertex newPinterface = graphTraversalSource().V(pInterfacesOnLatestPserver.get(j)).has("aai-node-type","p-interface").next(); + logger.info("\tMatching P-interface "+newPinterface.property("interface-name").value().toString()+ " found on pservers"); +// SCENARIO 3 there already exists a plink in the new pinterface need to move all other pinterfaces and nodes in swing edges after the fcn no need for plink name change + List<Vertex> oldPlinkList = graphTraversalSource().V(pInterfacesOnOldPserver.get(i)).has("aai-node-type","p-interface").out("tosca.relationships.network.LinksTo").toList(); + if(graphTraversalSource().V(pInterfacesOnLatestPserver.get(j)).has("aai-node-type","p-interface").out("tosca.relationships.network.LinksTo").hasNext()){ + logger.info("\tPhysical-link exists on new pserver's p-interface also... So, don't move this p-interface to new pserver..."); + if (!oldPlinkList.isEmpty()) { + //drop edge b/w oldPInterface and oldPlink + String oldPlinkName = ""; + Edge oldPIntToPlinkEdge = oldPinterface.edges(Direction.OUT, "tosca.relationships.network.LinksTo").next(); + oldPIntToPlinkEdge.remove(); + + //remove physical link vertex also + Vertex oldPlink = null; + + oldPlink = oldPlinkList.get(0); + oldPlinkName = oldPlink.property("link-name").value().toString(); + oldPlink.remove(); + logger.info("\tDropped edge b/w old P-interface and Physical-link, and deleted old physical-link "+oldPlinkName); + } + moveChildrenOfMatchingPInterfaceToNewPserver(pInterfacesOnOldPserver, i, oldPinterface, newPinterface); + } +// SCENARIO 2 = there is no plink in new pinterface and move old plink to new + else{ + logger.info("\tNo Physical-link exists on new pserver's p-interface... Move old plink to new pserver's p-interface"); + Vertex oldPlink = null; + if (!oldPlinkList.isEmpty()) { + oldPlink = oldPlinkList.get(0); + String linkName = oldPlink.property("link-name").value().toString(); + createCousinEdge(newPinterface,oldPlink); + logger.info("\tCreated edge b/w new P-interface and old physical-link "+linkName); + //drop edge b/w oldPInterface and oldPlink + Edge oldPIntToPlinkEdge = oldPinterface.edges(Direction.OUT, "tosca.relationships.network.LinksTo").next(); + oldPIntToPlinkEdge.remove(); + logger.info("\tDropped edge b/w old P-interface and Physical-link "+linkName); + linkName = linkName.replaceAll(old.property("hostname").value().toString(),latest.property("hostname").value().toString()); + + String[] PlinkBarSplit = linkName.split("\\|"); + if(PlinkBarSplit.length>1) { + modifyPlinkName(oldPlink,linkName,old); + } + else{ + logger.info("\t" +oldPlink.property("link-name").value().toString()+ " does not comply with naming conventions related to pserver hostname:" + old.property("hostname").value().toString()); + } + moveChildrenOfMatchingPInterfaceToNewPserver(pInterfacesOnOldPserver, i, oldPinterface, newPinterface); + } else { + moveChildrenOfMatchingPInterfaceToNewPserver(pInterfacesOnOldPserver, i, oldPinterface, newPinterface); + } + } + //delete the oldPInterface + oldPinterface.remove(); + break; + } + } + } + } + + private void moveChildrenOfMatchingPInterfaceToNewPserver(List<Vertex> pInterfacesOnOldPserver, int i, Vertex oldPinterface, Vertex newPinterface) { + // Check if there are children under old pserver's p-int and move them to new pserver's matching p-int + List<Vertex> oldPIntChildren = graphTraversalSource().V(pInterfacesOnOldPserver.get(i)).has("aai-node-type","p-interface").in().has("aai-node-type", P.within("l-interface","sriov-pf")).toList(); + if (oldPIntChildren != null && !oldPIntChildren.isEmpty()){ + oldPIntChildren.forEach((c)-> { swingEdges(oldPinterface, newPinterface, null, null, "IN"); +// c.remove(); + }); + logger.info("\t"+"Child vertices of p-interface on old pserver have been moved to p-interface on new pserver"); + + } + } + + public void modifyPlinkName(Vertex oldPlink,String linkName,Vertex old ){ + + String[] PlinkBarSplit = linkName.split("\\|"); + if(PlinkBarSplit.length>1) { + String[] pserv1Connection = PlinkBarSplit[0].split(":"); + String[] pserv2Connection = PlinkBarSplit[1].split(":"); + + HashMap<String, String> map = new HashMap<>(); + map.put(pserv1Connection[0], pserv1Connection[1]); + map.put(pserv2Connection[0], pserv2Connection[1]); + + String[] temp = new String[2]; + temp[0] = pserv1Connection[0]; + temp[1] = pserv2Connection[0]; + Arrays.sort(temp); + String linkNameNew = temp[0] + ":" + map.get(temp[0]).toString() + "|" + temp[1] + ":" + map.get(temp[1]).toString(); + oldPlink.property("link-name", linkNameNew); + logger.info("\tUpdate physical-link name from "+linkName+ " to "+linkNameNew); + } + else{ + logger.info("\t" +oldPlink.property("link-name").value().toString()+ "Does not comply with naming conventions related to pserver hostname:" + old.property("hostname").value().toString()); + + } + } + + public void moveLagInterfaces(Vertex old, Vertex latest) throws AAIException { + + List<Vertex> lagInterfacesOnOldPserver = graphTraversalSource().V(old).has("aai-node-type","pserver").in("tosca.relationships.network.BindsTo").has("aai-node-type","lag-interface").toList(); + List<Vertex> lagInterfacesOnLatestPserver = graphTraversalSource().V(latest).has("aai-node-type","pserver").in("tosca.relationships.network.BindsTo").has("aai-node-type","lag-interface").toList(); + // SCENARIO 1 = no match found move everything from pserver old to new in swing edges call outside this fcn + + if(lagInterfacesOnLatestPserver.size() == 0){ + return; + } + + for(int i=0; i<lagInterfacesOnOldPserver.size();i++){ + + for(int j=0; j<lagInterfacesOnLatestPserver.size(); j++){ + //lag interface-name matches on both + if(lagInterfacesOnOldPserver.get(i).property("interface-name").value().toString().equals(lagInterfacesOnLatestPserver.get(j).property("interface-name").value().toString())){ + Vertex oldLaginterface = graphTraversalSource().V(lagInterfacesOnOldPserver.get(i)).has("aai-node-type","lag-interface").next(); + Vertex newLaginterface = graphTraversalSource().V(lagInterfacesOnLatestPserver.get(j)).has("aai-node-type","lag-interface").next(); + //Check if there are any children on the old lag-interface and move them to new + // Check if there are children under old pserver's p-int and move them to new pserver's matching p-int + List<Vertex> oldPIntChildren = graphTraversalSource().V(lagInterfacesOnOldPserver.get(i)).has("aai-node-type","lag-interface").in().has("aai-node-type", P.within("l-interface")).toList(); + if (oldPIntChildren != null && !oldPIntChildren.isEmpty()){ + oldPIntChildren.forEach((c)-> swingEdges(oldLaginterface, newLaginterface, null, null, "BOTH")); + } + logger.info("\t"+"Child vertices of lag-interface on old pserver have been moved to lag-interface on new pserver"); + //delete the oldLagInterface + oldLaginterface.remove(); + break; + } + } + } + } + + + private void modifyChildrenUri(Vertex v) throws UnsupportedEncodingException, AAIException { + logger.info("\tModifying children uri for all levels....."); + Set<Vertex> parentSet = new HashSet<>(); + parentSet.add(v); + verifyOrAddUri("", parentSet); + } + + + protected void verifyOrAddUri(String parentUri, Set<Vertex> vertexSet) throws UnsupportedEncodingException, AAIException { + + + String correctUri; + for (Vertex v : vertexSet) { + seen.add(v.id()); + //if there is an issue generating the uri catch, log and move on; + try { + correctUri = parentUri + this.getUriForVertex(v); + } catch (Exception e) { + logger.error("\tVertex has issue generating uri " + e.getMessage() + "\n\t" + this.asString(v)); + continue; + } + try { + v.property(AAIProperties.AAI_URI, correctUri); + } catch (Exception e) { + logger.info(e.getMessage() + "\n\t" + this.asString(v)); + } + if (!v.property(AAIProperties.AAI_UUID).isPresent()) { + v.property(AAIProperties.AAI_UUID, UUID.randomUUID().toString()); + } + this.verifyOrAddUri(correctUri, getChildren(v)); + } + } + + protected Set<Vertex> getChildren(Vertex v) { + + Set<Vertex> children = graphTraversalSource().V(v).bothE().not(__.has(EdgeProperty.CONTAINS.toString(), AAIDirection.NONE.toString())).otherV().toSet(); + + return children.stream().filter(child -> !seen.contains(child.id())).collect(Collectors.toSet()); + } + + protected String getUriForVertex(Vertex v) { + String aaiNodeType = v.property(AAIProperties.NODE_TYPE).value().toString(); + + + Map<String, String> parameters = this.nodeTypeToKeys.get(aaiNodeType).stream().collect(Collectors.toMap( + key -> key, + key -> encodeProp(v.property(key).value().toString()) + )); + + return this.nodeTypeToUri.get(aaiNodeType).buildFromEncodedMap(parameters).toString(); + } + private static String encodeProp(String s) { + try { + return UriUtils.encode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + return ""; + } + } + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v14/MigrateSdnaIvlanData.java b/src/main/java/org/onap/aai/migration/v14/MigrateSdnaIvlanData.java new file mode 100644 index 0000000..6f759fb --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v14/MigrateSdnaIvlanData.java @@ -0,0 +1,443 @@ +/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v14;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Migrator;
+import org.onap.aai.migration.Status;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+@MigrationPriority(100)
+@MigrationDangerRating(1)
+@Enabled
+public class MigrateSdnaIvlanData extends Migrator {
+
+ private final String CONFIGURATION_NODE_TYPE = "configuration";
+ private final String EVC_NODE_TYPE = "evc";
+ private final String FORWARDER_NODE_TYPE = "forwarder";
+ private final String FORWRDER_EVC_NODE_TYPE = "forwarder-evc";
+ private final String FORWARDING_PATH_NODE_TYPE = "forwarding-path";
+ private final String PNF_NODE_TYPE = "pnf";
+ private final String P_INTERFACE_NODE_TYPE = "p-interface";
+ private final String LAG_INTERFACE_NODE_TYPE = "lag-interface";
+ private final String SAREA_GLOBAL_CUSTOMER_ID = "8a00890a-e6ae-446b-9dbe-b828dbeb38bd";
+
+ GraphTraversal<Vertex, Vertex> serviceSubscriptionGt;
+
+ private static GraphTraversalSource g = null;
+ private static boolean success = true;
+ private static boolean checkLog = false;
+ private int headerLength;
+ private int migrationSuccess = 0;
+ private int migrationFailure = 0;
+ private int invalidPInterfaceCount = 0;
+ private int invalidLagInterfaceCount = 0;
+
+
+ private static List<String> dmaapMsgList = new ArrayList<String>();
+ private static final String homeDir = System.getProperty("AJSC_HOME");
+
+ private static List<String> validPnfList = new ArrayList<String>();
+ private static List<String> invalidPnfList = new ArrayList<String>();
+
+ private static Map<String, List<String>> validInterfaceMap = new HashMap<String, List<String>>();
+ private static Map<String, List<String>> invalidInterfaceMap = new HashMap<String, List<String>>();
+
+ protected class SdnaIvlanFileData{
+ String evcName;
+ String pnfName;
+ String interfaceAID;
+ int ivlanValue;
+
+ public String getEvcName() {
+ return evcName;
+ }
+ public void setEvcName(String evcName) {
+ this.evcName = evcName;
+ }
+
+ public String getPnfName() {
+ return pnfName;
+ }
+ public void setPnfName(String pnfName) {
+ this.pnfName = pnfName;
+ }
+ public String getInterfaceAID() {
+ return interfaceAID;
+ }
+ public void setInterfaceAID(String interfaceAID) {
+ this.interfaceAID = interfaceAID;
+ }
+
+ public int getIvlanValue() {
+ return ivlanValue;
+ }
+ public void setIvlanValue(int ivlanValue) {
+ this.ivlanValue = ivlanValue;
+ }
+
+ }
+
+ private static ArrayList<SdnaIvlanFileData> ivlanList = new ArrayList<SdnaIvlanFileData>();
+
+ public MigrateSdnaIvlanData(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+
+ this.g = this.engine.asAdmin().getTraversalSource();
+ this.serviceSubscriptionGt = g.V().has("global-customer-id", SAREA_GLOBAL_CUSTOMER_ID).in("org.onap.relationships.inventory.BelongsTo").has("service-type", "SAREA");
+ }
+
+ @Override
+ public void run() {
+ logger.info("---------- Start migration ----------");
+ String configDir = System.getProperty("BUNDLECONFIG_DIR");
+ if (homeDir == null) {
+ logger.info(this.MIGRATION_ERROR + "ERROR: Could not find sys prop AJSC_HOME");
+ success = false;
+ return;
+ }
+ if (configDir == null) {
+ success = false;
+ return;
+ }
+
+ String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/sarea-inventory/";
+
+ int fileLineCounter = 0;
+
+ String fileName = feedDir+ "ivlanData.csv";
+ logger.info(fileName);
+ logger.info("---------- Processing Entries from file ----------");
+
+
+ try {
+ List<String> lines = Files.readAllLines(Paths.get(fileName));
+ Iterator<String> lineItr = lines.iterator();
+ while (lineItr.hasNext()){
+ String line = lineItr.next().trim();
+ if (!line.isEmpty()) {
+ if (fileLineCounter != 0) {
+
+ try{
+ String[] colList = line.split(",", -1);
+ SdnaIvlanFileData lineData = new SdnaIvlanFileData();
+ lineData.setEvcName(colList[0].trim());
+ lineData.setPnfName(colList[1].trim());
+ lineData.setInterfaceAID(colList[2].trim());
+ lineData.setIvlanValue(Integer.valueOf(colList[3].trim()));
+ ivlanList.add(lineData);
+
+ } catch (Exception e){
+ logger.info(this.MIGRATION_ERROR + " ERROR: Record Format is invalid. Expecting Numeric value for Forwarder_Id and Ivlan_Value. Skipping Record: " + line);
+ this.migrationFailure++;
+ }
+
+ } else {
+ this.headerLength = line.split(",", -1).length;
+ if (this.headerLength < 4){
+ logger.info(this.MIGRATION_ERROR + "ERROR: Input file should have atleast 4 columns");
+ this.success = false;
+ return;
+ }
+ }
+ }
+ fileLineCounter++;
+ }
+
+ processSdnaIvlan();
+
+ int invalidInterfacesCount = getInvalidInterfaceCount();
+
+ logger.info ("\n \n ******* Final Summary for SDN-A IVLAN Migration ********* \n");
+ logger.info(this.MIGRATION_SUMMARY_COUNT + "SDN-A forward-evcs: IVLANs updated: "+ migrationSuccess);
+ logger.info(this.MIGRATION_SUMMARY_COUNT + "Total File Record Count: "+(fileLineCounter - 1));
+ logger.info(this.MIGRATION_SUMMARY_COUNT + "Unprocessed SDNA File Records : "+ migrationFailure);
+ logger.info(this.MIGRATION_SUMMARY_COUNT + "PNFs from Input File not found : "+ Integer.toString(invalidPnfList.size()) + "\n");
+
+
+ logger.info(this.MIGRATION_SUMMARY_COUNT + "Total PNF + P-INTERFACEs from Input File not found : " + Integer.toString(invalidPInterfaceCount));
+ logger.info(this.MIGRATION_SUMMARY_COUNT + "Total PNF + LAG-INTERFACEs from Input File not found : " + Integer.toString(invalidLagInterfaceCount));
+ logger.info(this.MIGRATION_SUMMARY_COUNT + "Total PNF/INTERFACEs from Input File not found : " + Integer.toString(invalidInterfacesCount));
+
+ } catch (FileNotFoundException e) {
+ logger.info(this.MIGRATION_ERROR + "ERROR: Could not find file " + fileName, e.getMessage());
+ success = false;
+ checkLog = true;
+ } catch (NoSuchFileException e) {
+ logger.info(this.MIGRATION_ERROR + "ERROR: Could not find file " + fileName, e.getMessage());
+ success = false;
+ checkLog = true;
+ } catch (IOException e) {
+ logger.info(this.MIGRATION_ERROR + "ERROR: Issue reading file " + fileName, e);
+ success = false;
+ } catch (Exception e) {
+ logger.info(this.MIGRATION_ERROR + "encountered exception", e);
+ e.printStackTrace();
+ success = false;
+ }
+
+ }
+ private void processSdnaIvlan() {
+
+ for(int i = 0; i < ivlanList.size(); i ++) {
+ String evc = ivlanList.get(i).getEvcName();
+ String pnf = ivlanList.get(i).getPnfName();
+ String interfaceId = ivlanList.get(i).getInterfaceAID();
+ String ivlanValue = Integer.toString(ivlanList.get(i).getIvlanValue());
+
+ Boolean pnfExists = pnfExists(pnf);
+ GraphTraversal<Vertex, Vertex> forwarderEvcGT;
+ Vertex forwarderEvcVtx = null;
+ String interfaceNodeType;
+ String forwarderEvcId = null;
+
+ if (!pnfExists){
+ migrationFailure++;
+ }else{
+
+ if (interfaceId.contains(".")){
+ interfaceNodeType = P_INTERFACE_NODE_TYPE;
+ }else{
+ interfaceNodeType = LAG_INTERFACE_NODE_TYPE;
+ }
+
+ validateInterface(pnf, interfaceNodeType, interfaceId);
+
+ forwarderEvcGT = g.V()
+ .has("pnf-name", pnf).has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE)
+ .in("tosca.relationships.network.BindsTo")
+ .has(AAIProperties.NODE_TYPE, interfaceNodeType).has("interface-name", interfaceId)
+ .in("org.onap.relationships.inventory.ForwardsTo")
+ .where(__.out("org.onap.relationships.inventory.BelongsTo").has("forwarding-path-id", evc))
+ .out("org.onap.relationships.inventory.Uses")
+ .in("org.onap.relationships.inventory.BelongsTo");
+
+ // fwd-evc not found for pnf + interface
+ if(!forwarderEvcGT.hasNext()){
+ forwarderEvcId = pnf + " " + evc;
+ migrationError(PNF_NODE_TYPE + "/" + EVC_NODE_TYPE, forwarderEvcId, "ivlan", ivlanValue);
+
+ }
+
+ while(forwarderEvcGT.hasNext()){
+ forwarderEvcVtx = forwarderEvcGT.next();
+
+ // fwd-evc vertex is null
+ if(forwarderEvcVtx == null){
+ forwarderEvcId = pnf + " " + evc;
+ migrationError(PNF_NODE_TYPE + "/" + EVC_NODE_TYPE, forwarderEvcId, "ivlan", ivlanValue);
+ }
+ // update fwd-evc with ivlan value
+ else{
+
+ forwarderEvcId = forwarderEvcVtx.property("forwarder-evc-id").value().toString();
+ try{
+ forwarderEvcVtx.property("ivlan", ivlanValue);
+ logger.info(String.format("Updating Node Type forwarder-evc Property ivlan value %s", ivlanValue.toString()));
+ this.touchVertexProperties(forwarderEvcVtx, false);
+ updateDmaapList(forwarderEvcVtx);
+ migrationSuccess++;
+
+ }catch (Exception e){
+ logger.info(e.toString());
+ migrationError(FORWRDER_EVC_NODE_TYPE, forwarderEvcId, "ivlan", ivlanValue);
+ }
+ }
+ }
+ }
+
+ }
+ }
+
+ /**
+ * Description: Validate if pnf node exists in Graph
+ * @param pnf
+ * @return boolean
+ */
+ private boolean pnfExists(String pnf){
+ if (invalidPnfList.contains(pnf)){
+ logger.info(this.MIGRATION_ERROR + "ERROR: PNF value " + pnf + " does not exist.");
+ return false;
+ }
+ if (validPnfList.contains(pnf)){
+ return true;
+ }
+
+ GraphTraversal<Vertex, Vertex> pnfGT = g.V()
+ .has("pnf-name", pnf).has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE);
+
+ if(pnfGT.hasNext()){
+ validPnfList.add(pnf);
+ return true;
+ }
+ else{
+ logger.info(this.MIGRATION_ERROR + "ERROR: PNF value " + pnf + " does not exist.");
+ invalidPnfList.add(pnf);
+ return false;
+ }
+
+ }
+
+ /**
+ * Description: Validate if p-interface or lag-interface node exists in Graph
+ * @param pnf
+ * @param interfaceNodeType
+ * @param interfaceName
+ */
+ private void validateInterface(String pnf, String interfaceNodeType, String interfaceName){
+
+ List <String> validInterfaceList;
+ List <String> invalidInterfaceList;
+
+ if(!validInterfaceMap.containsKey(pnf) ){
+ validInterfaceList = new ArrayList<String>();
+ }else{
+ validInterfaceList = validInterfaceMap.get(pnf);
+ }
+
+ if(!invalidInterfaceMap.containsKey(pnf)){
+ invalidInterfaceList = new ArrayList<String>();
+ }else{
+ invalidInterfaceList = invalidInterfaceMap.get(pnf);
+ }
+
+ if(invalidInterfaceList.contains(interfaceName)){
+ logger.info(this.MIGRATION_ERROR + "ERROR PNF " + pnf + " with a " + interfaceNodeType + " of " + interfaceName + " does not exist.");
+ return;
+ }
+ if(validInterfaceList.contains(interfaceName)){
+ return;
+ }
+
+ GraphTraversal<Vertex, Vertex> interfaceGT = g.V()
+ .has("pnf-name", pnf).has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE)
+ .in("tosca.relationships.network.BindsTo")
+ .has("interface-name", interfaceName).has(AAIProperties.NODE_TYPE, interfaceNodeType);
+
+ if(interfaceGT.hasNext()){
+ validInterfaceList.add(interfaceName);
+ validInterfaceMap.put(pnf, validInterfaceList);
+ }
+ else{
+ logger.info(this.MIGRATION_ERROR + "ERROR PNF " + pnf + " with a " + interfaceNodeType + " of " + interfaceName + " does not exist.");
+ invalidInterfaceList.add(interfaceName);
+ invalidInterfaceMap.put(pnf, invalidInterfaceList);
+ }
+ }
+
+
+ /**
+ * Description: Error Routine if graph is not updated by input file record
+ * @param nodeType
+ * @param nodeId
+ * @param property
+ * @param propertyValue
+ */
+ private void migrationError(String nodeType, String nodeId, String property, String propertyValue){
+ logger.info(this.MIGRATION_ERROR + "ERROR: Failure to update "
+ + nodeType + " ID " + nodeId + ", " + property + " to value " + propertyValue
+ + ". Node Not Found \n");
+ migrationFailure++;
+ }
+
+ private int getInvalidInterfaceCount(){
+ int interfaceCount = 0;
+
+ for (Map.Entry<String, List<String>> entry: invalidInterfaceMap.entrySet()){
+ String key = entry.getKey();
+ List <String> invalidList = invalidInterfaceMap.get(key);
+
+ for (int i = 0; i < invalidList.size(); i++){
+ if(invalidList.get(i).contains(".")){
+ invalidPInterfaceCount++;
+ }else{
+ invalidLagInterfaceCount++;
+ }
+
+ }
+ interfaceCount = interfaceCount + invalidInterfaceMap.get(key).size();
+ }
+ return interfaceCount;
+ }
+
+ /**
+ * Description: Dmaap Routine
+ * @param v
+ */
+ private void updateDmaapList(Vertex v){
+ String dmaapMsg = System.nanoTime() + "_" + v.id().toString() + "_" + v.value("resource-version").toString();
+ dmaapMsgList.add(dmaapMsg);
+ logger.info("\tAdding Updated Vertex " + v.id().toString() + " to dmaapMsgList....");
+ }
+
+ @Override
+ public Status getStatus() {
+ if (checkLog) {
+ return Status.CHECK_LOGS;
+ }
+ else if (success) {
+ return Status.SUCCESS;
+ }
+ else {
+ return Status.FAILURE;
+ }
+ }
+
+ @Override
+ public void commit() {
+ engine.commit();
+ createDmaapFiles(dmaapMsgList);
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{this.FORWRDER_EVC_NODE_TYPE});
+ }
+
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateSdnaIvlanData";
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v14/PserverDedupWithDifferentSourcesOfTruth.java b/src/main/java/org/onap/aai/migration/v14/PserverDedupWithDifferentSourcesOfTruth.java new file mode 100644 index 0000000..80944ff --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v14/PserverDedupWithDifferentSourcesOfTruth.java @@ -0,0 +1,358 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v14; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.structure.*; +import org.janusgraph.core.attribute.Text; +import org.javatuples.Pair; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.enums.AAIDirection; +import org.onap.aai.edges.enums.EdgeProperty; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.springframework.web.util.UriUtils; + +import javax.ws.rs.core.UriBuilder; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.*; +import java.util.stream.Collectors; + +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.introspection.Introspector; + +@Enabled +@MigrationPriority(10) +@MigrationDangerRating(100) +public class PserverDedupWithDifferentSourcesOfTruth extends EdgeSwingMigrator { + /** + * Instantiates a new migrator. + * + * @param engine + */ + private final String PARENT_NODE_TYPE = "pserver"; + private boolean success = true; + protected Set<Object> seen = new HashSet<>(); + private Map<String, UriBuilder> nodeTypeToUri; + private Map<String, Set<String>> nodeTypeToKeys; + private static List<String> dmaapMsgList = new ArrayList<String>(); + private static List<Introspector> dmaapDeleteList = new ArrayList<Introspector>(); + private static int pserversUpdatedCount = 0; + private static int pserversDeletedCount = 0; + + + private static String[] rctSourceOfTruth = new String[]{"AAIRctFeed", "RCT"}; + private static String[] roSourceOfTruth = new String[]{"AAI-EXTENSIONS", "RO"}; + + List<Vertex> RemoveROList = new ArrayList<>(); + + public PserverDedupWithDifferentSourcesOfTruth(TransactionalGraphEngine engine , LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + createDmaapFilesForDelete(dmaapDeleteList); + + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } + else { + return Status.FAILURE; + } + } + + @Override + public List<Pair<Vertex, Vertex>> getAffectedNodePairs() { + return null; + } + + @Override + public String getNodeTypeRestriction() { + return null; + } + + @Override + public String getEdgeLabelRestriction() { + return null; + } + + @Override + public String getEdgeDirRestriction() { + return null; + } + + @Override + public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) { + + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return null; + } + + @Override + public String getMigrationName() { + return "PserverDedupWithDifferentSourcesOfTruth"; + } + + @Override + public void run() { + + int dupCount = 0; + nodeTypeToUri = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect( + Collectors.toMap( + e -> e.getKey(), + e -> UriBuilder.fromPath(e.getValue().getFullGenericURI().replaceAll("\\{"+ e.getKey() + "-", "{")) + )); + + nodeTypeToKeys = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect( + Collectors.toMap( + e -> e.getKey(), + e -> e.getValue().getKeys() + )); + + List<Vertex> rctList = graphTraversalSource().V().has("aai-node-type", "pserver").has("source-of-truth", P.within(rctSourceOfTruth)).toList(); + List<Vertex> roList = graphTraversalSource().V().has("aai-node-type", "pserver").has("source-of-truth", P.within(roSourceOfTruth)).toList(); + + logger.info("Total number of RCT sourced pservers in A&AI :" +rctList.size()); + logger.info("Total number of RO sourced pservers in A&AI :" +roList.size()); + + for(int i=0;i<rctList.size();i++){ + Vertex currRct = rctList.get(i); + Object currRctFqdn = null; + if (currRct.property("fqdn").isPresent() && (currRct.property("fqdn").value() != null)){ + currRctFqdn = currRct.property("fqdn").value(); + logger.info("\n"); + logger.info("Current RCT Pserver hostname: " + currRct.property("hostname").value().toString() + " fqdn: " +currRct.property("fqdn").value().toString()); + for(int j=0;j<roList.size();j++){ + Vertex currRo = roList.get(j); + Object currRoHostname = null; + if (currRo.property("hostname").isPresent()){ + currRoHostname = currRo.property("hostname").value(); + } + if (currRoHostname != null){ + String[] rctFqdnSplit = currRctFqdn.toString().split("\\."); + String[] roHostnameSplit = currRoHostname.toString().split("\\."); + if (rctFqdnSplit.length >0 && roHostnameSplit.length > 0){ + if(!rctFqdnSplit[0].isEmpty() && !roHostnameSplit[0].isEmpty() && rctFqdnSplit[0].equals(roHostnameSplit[0])){ + logger.info("\tPserver match found - RO Pserver with hostname: "+currRo.property("hostname").value().toString()); + dupCount++; + try { + mergePservers(currRct,currRo); + break; + } catch (UnsupportedEncodingException e) { + success = false; + } catch (AAIException e) { + success = false; + } + } + } + } + } + } + } + RemoveROList.forEach(v ->v.remove()); + logger.info ("\n \n ******* Migration Summary Counts for Dedup of RCT and RO sourced pservers ********* \n"); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Total number of RCT: " +rctList.size()); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Total number of RO: " +roList.size()); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Duplicate pserver count: "+ dupCount); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Number of RCT updated: "+pserversUpdatedCount); + logger.info(this.MIGRATION_SUMMARY_COUNT + "Number of RO deleted: "+ pserversDeletedCount +"\n"); + } + private GraphTraversalSource graphTraversalSource() { + return this.engine.asAdmin().getTraversalSource(); + } + + + public void mergePservers(Vertex rct, Vertex ro) throws UnsupportedEncodingException, AAIException { + Introspector obj = serializer.getLatestVersionView(ro); + dmaapDeleteList.add(obj); + rct.property("fqdn",ro.property("hostname").value().toString()); + dropComplexEdge(ro); + dropMatchingROPInterfaces(ro, rct); + dropMatchingROLagInterfaces(ro, rct); + swingEdges(ro, rct, null, null, "BOTH"); + modifyChildrenUri(rct); + if(!(rct.property("pserver-id").isPresent())){ + rct.property("pserver-id",UUID.randomUUID().toString()); + } + String dmaapMsg = System.nanoTime() + "_" + rct.id().toString() + "_" + rct.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + pserversUpdatedCount++; + logger.info("\tAdding RO pserver to the delete list...."); + RemoveROList.add(ro); + pserversDeletedCount++; + } + + private void dropMatchingROPInterfaces(Vertex ro, Vertex rct) { + Map<String, Vertex> removeROPIntMap = new HashMap<String, Vertex>(); + List<Vertex> pIntList = graphTraversalSource().V(ro).in("tosca.relationships.network.BindsTo").has("aai-node-type","p-interface").toList(); + if (pIntList != null && !pIntList.isEmpty()) { + Iterator<Vertex> pIntListItr = pIntList.iterator(); + while(pIntListItr.hasNext()){ + Vertex pInt = pIntListItr.next(); + + removeROPIntMap.put(pInt.property("interface-name").value().toString(), pInt); + } + Set<String> interfaceNameSet = removeROPIntMap.keySet(); + List<Vertex> rctPIntList = graphTraversalSource().V(rct).in("tosca.relationships.network.BindsTo").has("aai-node-type","p-interface").toList(); + if (rctPIntList != null && !rctPIntList.isEmpty()){ + Iterator<Vertex> rctPIntListItr = rctPIntList.iterator(); + while(rctPIntListItr.hasNext()){ + Vertex rctPInt = rctPIntListItr.next(); + String rctIntfName = rctPInt.property("interface-name").value().toString(); + if (interfaceNameSet.contains(rctIntfName)){ + Vertex pIntToRemoveFromROPserver = removeROPIntMap.get(rctIntfName); + String roPIntUri = "roPIntUri"; + if (pIntToRemoveFromROPserver.property("aai-uri").isPresent()){ + roPIntUri = pIntToRemoveFromROPserver.property("aai-uri").value().toString(); + } + Edge roPIntToPserverEdge = pIntToRemoveFromROPserver.edges(Direction.OUT, "tosca.relationships.network.BindsTo").next(); + roPIntToPserverEdge.remove(); + pIntToRemoveFromROPserver.remove(); + logger.info("\tRemoved p-interface "+roPIntUri + " and its edge to RO pserver, not swinging the p-interface to RCT pserver"); + } + } + } + } + } + + private void dropMatchingROLagInterfaces(Vertex ro, Vertex rct) { + Map<String, Vertex> removeROLagIntMap = new HashMap<String, Vertex>(); + List<Vertex> lagIntList = graphTraversalSource().V(ro).in("tosca.relationships.network.BindsTo").has("aai-node-type","lag-interface").toList(); + if (lagIntList != null && !lagIntList.isEmpty()) { + Iterator<Vertex> lagIntListItr = lagIntList.iterator(); + while(lagIntListItr.hasNext()){ + Vertex lagInt = lagIntListItr.next(); + + removeROLagIntMap.put(lagInt.property("interface-name").value().toString(), lagInt); + } + Set<String> interfaceNameSet = removeROLagIntMap.keySet(); + List<Vertex> rctLagIntList = graphTraversalSource().V(rct).in("tosca.relationships.network.BindsTo").has("aai-node-type","lag-interface").toList(); + if (rctLagIntList != null && !rctLagIntList.isEmpty()){ + Iterator<Vertex> rctLagIntListItr = rctLagIntList.iterator(); + while(rctLagIntListItr.hasNext()){ + Vertex rctPInt = rctLagIntListItr.next(); + String rctIntfName = rctPInt.property("interface-name").value().toString(); + if (interfaceNameSet.contains(rctIntfName)){ + Vertex lagIntToRemoveFromROPserver = removeROLagIntMap.get(rctIntfName); + String roLagIntUri = "roPIntUri"; + if (lagIntToRemoveFromROPserver.property("aai-uri").isPresent()){ + roLagIntUri = lagIntToRemoveFromROPserver.property("aai-uri").value().toString(); + } + Edge roLagIntToPserverEdge = lagIntToRemoveFromROPserver.edges(Direction.OUT, "tosca.relationships.network.BindsTo").next(); + roLagIntToPserverEdge.remove(); + lagIntToRemoveFromROPserver.remove(); + logger.info("\tRemoved lag-interface "+roLagIntUri + " and its edge to RO pserver, not swinging the lag-interface to RCT pserver"); + } + } + } + } + } + + public void dropComplexEdge(Vertex ro){ + List<Vertex> locatedInEdgeVertexList = graphTraversalSource().V(ro).has("aai-node-type", "pserver").out("org.onap.relationships.inventory.LocatedIn").has("aai-node-type","complex").toList(); + if (locatedInEdgeVertexList != null && !locatedInEdgeVertexList.isEmpty()){ + Iterator<Vertex> locatedInEdgeVertexListItr = locatedInEdgeVertexList.iterator(); + while (locatedInEdgeVertexListItr.hasNext()){ + Vertex v = locatedInEdgeVertexListItr.next(); + if ("complex".equalsIgnoreCase(v.property("aai-node-type").value().toString())){ + Edge pserverToComplexEdge = v.edges(Direction.IN, "org.onap.relationships.inventory.LocatedIn").next(); + pserverToComplexEdge.remove(); + } + } + } + } + + + private void modifyChildrenUri(Vertex v) throws UnsupportedEncodingException, AAIException { + Set<Vertex> parentSet = new HashSet<>(); + parentSet.add(v); + verifyOrAddUri("", parentSet); + } + + + protected void verifyOrAddUri(String parentUri, Set<Vertex> vertexSet) throws UnsupportedEncodingException, AAIException { + + + String correctUri; + for (Vertex v : vertexSet) { + seen.add(v.id()); + //if there is an issue generating the uri catch, log and move on; + try { + correctUri = parentUri + this.getUriForVertex(v); + } catch (Exception e) { + logger.error("Vertex has issue generating uri " + e.getMessage() + "\n\t" + this.asString(v)); + continue; + } + try { + v.property(AAIProperties.AAI_URI, correctUri); + } catch (Exception e) { + logger.info("\t" + e.getMessage() + "\n\t" + this.asString(v)); + } + if (!v.property(AAIProperties.AAI_UUID).isPresent()) { + v.property(AAIProperties.AAI_UUID, UUID.randomUUID().toString()); + } + this.verifyOrAddUri(correctUri, getChildren(v)); + } + } + + protected Set<Vertex> getChildren(Vertex v) { + + Set<Vertex> children = graphTraversalSource().V(v).bothE().not(__.has(EdgeProperty.CONTAINS.toString(), AAIDirection.NONE.toString())).otherV().toSet(); + + return children.stream().filter(child -> !seen.contains(child.id())).collect(Collectors.toSet()); + } + + protected String getUriForVertex(Vertex v) { + String aaiNodeType = v.property(AAIProperties.NODE_TYPE).value().toString(); + + + Map<String, String> parameters = this.nodeTypeToKeys.get(aaiNodeType).stream().collect(Collectors.toMap( + key -> key, + key -> encodeProp(v.property(key).value().toString()) + )); + + return this.nodeTypeToUri.get(aaiNodeType).buildFromEncodedMap(parameters).toString(); + } + private static String encodeProp(String s) { + try { + return UriUtils.encode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + return ""; + } + } + +} diff --git a/src/main/java/org/onap/aai/migration/v15/MigrateBooleanDefaultsToFalse.java b/src/main/java/org/onap/aai/migration/v15/MigrateBooleanDefaultsToFalse.java new file mode 100644 index 0000000..3152436 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v15/MigrateBooleanDefaultsToFalse.java @@ -0,0 +1,115 @@ +/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v15;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Status;
+import org.onap.aai.migration.ValueMigrator;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+@Enabled
+public class MigrateBooleanDefaultsToFalse extends ValueMigrator {
+ protected static final String VNF_NODE_TYPE = "generic-vnf";
+ protected static final String VSERVER_NODE_TYPE = "vserver";
+ protected static final String VNFC_NODE_TYPE = "vnfc";
+ protected static final String L3NETWORK_NODE_TYPE = "l3-network";
+ protected static final String SUBNET_NODE_TYPE = "subnet";
+ protected static final String LINTERFACE_NODE_TYPE = "l-interface";
+ protected static final String VFMODULE_NODE_TYPE = "vf-module";
+
+ private static Map<String, Map> map;
+ private static Map<String, Boolean> pair1;
+ private static Map<String, Boolean> pair2;
+ private static Map<String, Boolean> pair3;
+ private static Map<String, Boolean> pair4;
+ private static Map<String, Boolean> pair5;
+ private static Map<String, Boolean> pair6;
+
+ public MigrateBooleanDefaultsToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setBooleanDefaultsToFalse(), false);
+
+ }
+
+ private static Map<String, Map> setBooleanDefaultsToFalse(){
+ map = new HashMap<>();
+ pair1 = new HashMap<>();
+ pair2 = new HashMap<>();
+ pair3 = new HashMap<>();
+ pair4 = new HashMap<>();
+ pair5 = new HashMap<>();
+ pair6 = new HashMap<>();
+
+
+ pair1.put("is-closed-loop-disabled", false);
+ map.put("generic-vnf", pair1);
+ map.put("vnfc", pair1);
+ map.put("vserver", pair1);
+
+ pair2.put("is-bound-to-vpn", false);
+ pair2.put("is-provider-network", false);
+ pair2.put("is-shared-network", false);
+ pair2.put("is-external-network", false);
+ map.put("l3-network", pair2);
+
+ pair3.put("dhcp-enabled", false);
+ map.put("subnet", pair3);
+
+ pair4.put("is-port-mirrored", false);
+ pair4.put("is-ip-unnumbered", false);
+ map.put("l-interface", pair4);
+
+ pair5.put("is-base-vf-module", false);
+ map.put("vf-module", pair5);
+
+ pair6.put("is-ip-unnumbered", false);
+ map.put("vlan", pair6);
+
+ return map;
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{VNF_NODE_TYPE,VSERVER_NODE_TYPE,VNFC_NODE_TYPE,L3NETWORK_NODE_TYPE,SUBNET_NODE_TYPE,LINTERFACE_NODE_TYPE,VFMODULE_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateBooleanDefaultsToFalse";
+ }
+
+}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v15/MigrateCloudRegionUpgradeCycle.java b/src/main/java/org/onap/aai/migration/v15/MigrateCloudRegionUpgradeCycle.java new file mode 100644 index 0000000..6104d9d --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v15/MigrateCloudRegionUpgradeCycle.java @@ -0,0 +1,361 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v15; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.javatuples.Pair; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Migrator; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(20) +@MigrationDangerRating(2) +@Enabled +public class MigrateCloudRegionUpgradeCycle extends Migrator { + + protected static final String CLOUD_REGION_NODE_TYPE = "cloud-region"; + protected static final String CLOUD_OWNER = "cloud-owner"; + protected static final String CLOUD_REGION_ID = "cloud-region-id"; + protected static final String UPGRADE_CYCLE = "upgrade-cycle"; + private static final String homeDir = System.getProperty("AJSC_HOME"); + + protected final AtomicInteger skippedRowsCount = new AtomicInteger(0); + protected final AtomicInteger processedRowsCount = new AtomicInteger(0); + + private boolean success = true; + private GraphTraversalSource g = null; + protected int headerLength; + + protected final AtomicInteger falloutRowsCount = new AtomicInteger(0); + + public MigrateCloudRegionUpgradeCycle(TransactionalGraphEngine engine, LoaderFactory loaderFactory, + EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + public void run() { + logger.info("---------- Start Updating upgrade-cycle for cloud-region ----------"); + + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + + String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/CloudRegion-ART-migration-data/"; + String fileName = feedDir + "CloudRegion-ART-migration-data.csv"; + logger.info(fileName); + logger.info("---------- Processing Region Entries from file ----------"); + + Map cloudRegionVertexMap = new HashMap(); + + try { + int cloudRegionCount = 0; + int cloudRegionErrorCount = 0; + ArrayList data = loadFile(fileName); + + Map<String, String> cloudRegionMapFromART = (Map) data.get(0); + Map<String, String> cloudAliasMapFromART = (Map) data.get(1); + + List<Vertex> cloudRegionList = this.engine.asAdmin().getTraversalSource().V() + .has(AAIProperties.NODE_TYPE, CLOUD_REGION_NODE_TYPE).has(CLOUD_OWNER, "att-aic").toList(); + + for (Vertex vertex : cloudRegionList) { + String cloudRegionId = null; + cloudRegionId = getCloudRegionIdNodeValue(vertex); + cloudRegionVertexMap.put(cloudRegionId, vertex); + } + + for (Map.Entry<String, String> entry : cloudRegionMapFromART.entrySet()) { + boolean regionFound = false; + String regionFromART = ""; + String aliasFromART = ""; + String vertexKey = ""; + + regionFromART = (String) entry.getKey(); + + if (cloudRegionVertexMap.containsKey(regionFromART)) { + regionFound = true; + vertexKey = regionFromART; + } else { + aliasFromART = cloudAliasMapFromART.get(regionFromART).toString(); + if (aliasFromART != null && !"".equals(aliasFromART) + && cloudRegionVertexMap.containsKey(aliasFromART)) { + regionFound = true; + vertexKey = aliasFromART; + } + } + + if (regionFound) { + String upgradeCycle = ""; + try { + upgradeCycle = (String) entry.getValue(); + + if (upgradeCycle != null && !"".equals(upgradeCycle)) { + Vertex vertex = (Vertex) cloudRegionVertexMap.get(vertexKey); + vertex.property(UPGRADE_CYCLE, upgradeCycle); + this.touchVertexProperties(vertex, false); + logger.info("Updated cloud-region, upgrade-cycle to " + upgradeCycle + + " having cloud-region-id : " + vertexKey); + cloudRegionCount++; + } else { + logger.info("upgrade-cycle value from ART is null or empty for the cloud-region-id : " + + vertexKey); + } + } catch (Exception e) { + success = false; + cloudRegionErrorCount++; + logger.error(MIGRATION_ERROR + + "encountered exception for upgrade-cycle update having cloud-region-id :" + vertexKey, + e); + } + } else { + logger.info("Region "+regionFromART+" from ART is not found in A&AI"); + } + + } + + logger.info("\n \n ******* Final Summary of Updated upgrade-cycle for cloud-region Migration ********* \n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Number of cloud-region updated: " + cloudRegionCount + "\n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Number of cloud-region failed to update due to error : " + + cloudRegionErrorCount + "\n"); + + } catch (FileNotFoundException e) { + logger.info("ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + } catch (IOException e) { + logger.info("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info("encountered exception", e); + e.printStackTrace(); + success = false; + } + } + + /** + * Load file to the map for processing + * + * @param fileName + * @return + * @throws Exception + */ + protected ArrayList loadFile(String fileName) throws Exception { + List<String> lines = Files.readAllLines(Paths.get(fileName)); + return this.getFileContents(lines); + } + + /** + * Get lines from file. + * + * @param lines + * @return + * @throws Exception + */ + protected ArrayList getFileContents(List<String> lines) throws Exception { + + final Map<String, String> regionMap = new ConcurrentHashMap<>(); + final Map<String, String> aliasMap = new ConcurrentHashMap<>(); + final ArrayList fileContent = new ArrayList(); + + processAndRemoveHeader(lines); + + logger.info("Total rows count excluding header: " + lines.size()); + + lines.stream().filter(line -> !line.isEmpty()).map(line -> Arrays.asList(line.split("\\s*,\\s*", -1))) + .map(this::processRegionUpgradeCycle).filter(Optional::isPresent).map(Optional::get).forEach(p -> { + processedRowsCount.getAndIncrement(); + String pnfName = p.getValue0(); + if (!regionMap.containsKey(pnfName)) { + regionMap.put(p.getValue0(), p.getValue1()); + } + }); + + fileContent.add(regionMap); + + lines.stream().filter(line -> !line.isEmpty()).map(line -> Arrays.asList(line.split("\\s*,\\s*", -1))) + .map(this::processRegionAlias).filter(Optional::isPresent).map(Optional::get).forEach(p -> { + processedRowsCount.getAndIncrement(); + String pnfName = p.getValue0(); + if (!aliasMap.containsKey(pnfName)) { + aliasMap.put(p.getValue0(), p.getValue1()); + } + }); + fileContent.add(aliasMap); + return fileContent; + + } + + /** + * Verify line has the necessary details. + * + * @param line + * @return + */ + protected boolean verifyLine(List<String> line) { + if (line.size() != headerLength) { + logger.info("ERROR: INV line should contain " + headerLength + " columns, contains " + line.size() + + " instead."); + this.skippedRowsCount.getAndIncrement(); + return false; + } + return true; + } + + /** + * * Get the pnf name and interface name from the line. + * + * @param line + * @return + */ + protected Optional<Pair<String, String>> processRegionAlias(List<String> line) { + //logger.info("Processing line... " + line.toString()); + int lineSize = line.size(); + if (lineSize < 4) { + logger.info("Skipping line, does not contain region and/or upgrade-cycle columns"); + skippedRowsCount.getAndIncrement(); + return Optional.empty(); + } + + String cloudRegion = line.get(0); + String upgradeCycle = line.get(1).replaceAll("^\"|\"$", "").replaceAll("\\s+", ""); + + if (cloudRegion.isEmpty()) { + logger.info("Line missing cloudRegion name" + line); + falloutRowsCount.getAndIncrement(); + return Optional.empty(); + } + + return Optional.of(Pair.with(cloudRegion, upgradeCycle)); + } + + /** + * * Get the pnf name and interface name from the line. + * + * @param line + * @return + */ + protected Optional<Pair<String, String>> processRegionUpgradeCycle(List<String> line) { + //logger.info("Processing line... " + line.toString()); + int lineSize = line.size(); + if (lineSize < 4) { + logger.info("Skipping line, does not contain region and/or upgrade-cycle columns"); + skippedRowsCount.getAndIncrement(); + return Optional.empty(); + } + + String cloudRegion = line.get(0); + String upgradeCycle = line.get(3).replaceAll("^\"|\"$", "").replaceAll("\\s+", ""); + + if (cloudRegion.isEmpty()) { + logger.info("Line missing cloudRegion name" + line); + falloutRowsCount.getAndIncrement(); + return Optional.empty(); + } + + return Optional.of(Pair.with(cloudRegion, upgradeCycle)); + } + + /** + * Verify header of the csv and remove it from the list. + * + * @param lines + * @throws Exception + */ + protected String processAndRemoveHeader(List<String> lines) throws Exception { + String firstLine; + if (lines.isEmpty()) { + String msg = "ERROR: Missing Header in file"; + success = false; + logger.error(msg); + throw new Exception(msg); + } else { + firstLine = lines.get(0); + } + + this.headerLength = firstLine.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength); + if (this.headerLength < 4) { + String msg = "ERROR: Input file should have 4 columns"; + success = false; + logger.error(msg); + throw new Exception(msg); + } + + return lines.remove(0); + } + + private String getCloudRegionIdNodeValue(Vertex vertex) { + String propertyValue = ""; + if (vertex != null && vertex.property(CLOUD_REGION_ID).isPresent()) { + propertyValue = vertex.property(CLOUD_REGION_ID).value().toString(); + } + return propertyValue; + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[] { CLOUD_REGION_NODE_TYPE }); + } + + @Override + public String getMigrationName() { + return "MigrateCloudRegionUpgradeCycle"; + } + +} diff --git a/src/main/java/org/onap/aai/migration/v15/MigrateInMaintDefaultToFalse.java b/src/main/java/org/onap/aai/migration/v15/MigrateInMaintDefaultToFalse.java new file mode 100644 index 0000000..d00d4f1 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v15/MigrateInMaintDefaultToFalse.java @@ -0,0 +1,99 @@ +/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v15;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Status;
+import org.onap.aai.migration.ValueMigrator;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+@Enabled
+public class MigrateInMaintDefaultToFalse extends ValueMigrator {
+
+ protected static final String VNF_NODE_TYPE = "generic-vnf";
+ protected static final String LINTERFACE_NODE_TYPE = "l-interface";
+ protected static final String LAG_INTERFACE_NODE_TYPE = "lag-interface";
+ protected static final String LOGICAL_LINK_NODE_TYPE = "logical-link";
+ protected static final String PINTERFACE_NODE_TYPE = "p-interface";
+ protected static final String VLAN_NODE_TYPE = "vlan";
+ protected static final String VNFC_NODE_TYPE = "vnfc";
+ protected static final String VSERVER_NODE_TYPE = "vserver";
+ protected static final String PSERVER_NODE_TYPE = "pserver";
+ protected static final String PNF_NODE_TYPE = "pnf";
+ protected static final String NOS_SERVER_NODE_TYPE = "nos-server";
+
+ private static Map<String, Map> map;
+ private static Map<String, Boolean> pair;
+
+ public MigrateInMaintDefaultToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setInMaintToFalse(), false);
+ }
+
+ private static Map<String, Map> setInMaintToFalse(){
+ map = new HashMap<>();
+ pair = new HashMap<>();
+
+ pair.put("in-maint", false);
+
+ map.put("generic-vnf", pair);
+ map.put("l-interface", pair);
+ map.put("lag-interface", pair);
+ map.put("logical-link", pair);
+ map.put("p-interface", pair);
+ map.put("vlan", pair);
+ map.put("vnfc", pair);
+ map.put("vserver", pair);
+ map.put("pserver", pair);
+ map.put("pnf", pair);
+ map.put("nos-server", pair);
+
+ return map;
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{VNF_NODE_TYPE,LINTERFACE_NODE_TYPE,LAG_INTERFACE_NODE_TYPE,LOGICAL_LINK_NODE_TYPE,PINTERFACE_NODE_TYPE,VLAN_NODE_TYPE,VNFC_NODE_TYPE,VSERVER_NODE_TYPE,PSERVER_NODE_TYPE,PNF_NODE_TYPE,NOS_SERVER_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateInMaintDefaultToFalse";
+ }
+
+}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v15/MigrateRadcomChanges.java b/src/main/java/org/onap/aai/migration/v15/MigrateRadcomChanges.java new file mode 100644 index 0000000..3ae17bd --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v15/MigrateRadcomChanges.java @@ -0,0 +1,733 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v15; +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.javatuples.Pair; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.EdgeSwingMigrator; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(26) +@MigrationDangerRating(100) +@Enabled +public class MigrateRadcomChanges extends EdgeSwingMigrator { + + private final String SERVICE_MODEL_TYPE = "Service"; + private final String RESOURCE_MODEL_TYPE = "VNF-Resource"; + private final String MODEL_INVARIANT_ID = "model-invariant-id"; + private final String MODEL_INVARIANT_ID_LOCAL = "model-invariant-id-local"; + private final String MODEL_VERSION_ID = "model-version-id"; + private final String MODEL_VERSION_ID_LOCAL = "model-version-id-local"; + private final String MODEL_CUSTOMIZATION_ID = "model-customization-id"; + private final String PERSONA_MODEL_VERSION = "persona-model-version"; + private final String GENERIC_VNF = "generic-vnf"; + private final String VNF_NAME = "vnf-name"; + private final String VNF_TYPE = "vnf-type"; + private final String SERVICE_INSTANCE = "service-instance"; + private final String SERVICE_INSTANCE_ID = "service-instance-id"; + private final String VF_MODULE = "vf-module"; + private final String VF_MODULE_ID = "vf-module-id"; + private final String MODEL = "model"; + private final String MODEL_VER = "model-ver"; + private final String MODEL_NAME = "model-name"; + private final String MODEL_VERSION = "model-version"; + private final String MODEL_ELEMENT = "model-element"; + private final String VSERVER = "vserver"; + private final String VSERVER_ID = "vserver-id"; + private final String IMAGE = "image"; + private final String IMAGE_NAME = "image-name"; + private final String TENANT = "tenant"; + private final String CLOUD_REGION = "cloud-region"; + + private static boolean success = true; + private static boolean checkLog = false; + private static GraphTraversalSource g = null; + private int headerLength; + private int genericVnfMigrationSuccess = 0; + private int genericVnfMigrationFailure = 0; + private int serviceInstanceMigrationSuccess = 0; + private int serviceInstanceMigrationFailure = 0; + private int vfModuleMigrationSuccess = 0; + private int vfModuleMigrationFailure = 0; + private int imageMigrationSuccess = 0; + private int imageMigrationFailure = 0; + + private static List<String> dmaapMsgList = new ArrayList<String>(); + private static final String homeDir = System.getProperty("AJSC_HOME"); + + protected class VfModuleFileData { + String vfModuleId; + String vfModuleModelName; + String imageName; + + public VfModuleFileData(String vfModuleId, String vfModuleModelName, String imageName) { + this.vfModuleId = vfModuleId; + this.vfModuleModelName = vfModuleModelName; + this.imageName = imageName; + } + + public String getVfModuleId() { + return vfModuleId; + } + public void setVfModuleId(String vfModuleId) { + this.vfModuleId = vfModuleId; + } + public String getVfModuleModelName() { + return vfModuleModelName; + } + public void setVfModuleModelName(String vfModuleModelName) { + this.vfModuleModelName = vfModuleModelName; + } + public String getImageName() { + return imageName; + } + public void setImageName(String imageName) { + this.imageName = imageName; + } + } + + + + public MigrateRadcomChanges(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + g = this.engine.asAdmin().getTraversalSource(); + } + + @Override + public void executeModifyOperation() { + logger.info("---------- Start migration ----------"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logger.info(MIGRATION_ERROR + "ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return; + } + if (configDir == null) { + success = false; + return; + } + + ArrayList<VfModuleFileData> vfModuleFileLineList = new ArrayList<VfModuleFileData>(); + + String feedDir = homeDir + "/" + configDir + "/" + "migration-input-files/radcom-changes/"; + String fileName = feedDir+ "INPUT-MODEL.csv"; + int genericVnfFileLineCounter = 0; + + logger.info(fileName); + logger.info("---------- Reading all file types and vf-modules ----------"); + ArrayList<String> fileTypeList = new ArrayList<String>(); + try { + List<String> lines = Files.readAllLines(Paths.get(fileName)); + Iterator<String> lineItr = lines.iterator(); + int typeFileLineCounter = 0; + while (lineItr.hasNext()){ + String line = lineItr.next().replace("\n", "").replace("\r", ""); + if (!line.isEmpty()) { + if (typeFileLineCounter != 0) { + String[] colList = line.split("\\s*,\\s*", -1); + if(!colList[0].equalsIgnoreCase(SERVICE_MODEL_TYPE) && !colList[0].equalsIgnoreCase(RESOURCE_MODEL_TYPE)) { + vfModuleFileLineList.add(new VfModuleFileData(colList[0], colList[5], colList[6])); + } + if(!colList[1].isEmpty() && !fileTypeList.contains(colList[1])) { + fileTypeList.add(colList[1]); + } + } else { + this.headerLength = line.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength + "\n"); + if (this.headerLength != 7 ){ + logger.info(MIGRATION_ERROR + "ERROR: Input file should have 7 columns"); + success = false; + return; + } + } + } + typeFileLineCounter++; + } + } catch (FileNotFoundException e) { + logger.info(MIGRATION_ERROR + "ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + checkLog = true; + } catch (IOException e) { + logger.info(MIGRATION_ERROR + "ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info(MIGRATION_ERROR + "encountered exception", e); + e.printStackTrace(); + success = false; + } + + int numberOfFileTypes = fileTypeList.size(); + for(int i = 0; i < numberOfFileTypes; i++) { + String newServiceModelInvariantId = ""; + String newServiceModelVersionId = ""; + String newResourceModelInvariantId = ""; + String newResourceModelVersionId = ""; + String newResourceModelCustomizationId = ""; + ArrayList<String> genericVnfList = new ArrayList<String>(); + + + int modelFileLineCounter = 0; + genericVnfFileLineCounter = 0; + fileName = feedDir+ "INPUT-MODEL.csv"; + + logger.info(fileName); + logger.info("---------- Processing Entries from file ----------"); + try { + List<String> lines = Files.readAllLines(Paths.get(fileName)); + Iterator<String> lineItr = lines.iterator(); + while (lineItr.hasNext()){ + String line = lineItr.next().replace("\n", "").replace("\r", ""); + if (!line.isEmpty()) { + if (modelFileLineCounter != 0) { + String[] colList = line.split("\\s*,\\s*", -1); + if(colList[1].equals(fileTypeList.get(i)) && colList[0].equalsIgnoreCase(SERVICE_MODEL_TYPE)) { + newServiceModelInvariantId = colList[2]; + newServiceModelVersionId = colList[3]; + } + else if(colList[1].equals(fileTypeList.get(i)) && colList[0].equalsIgnoreCase(RESOURCE_MODEL_TYPE)) { + newResourceModelInvariantId = colList[2]; + newResourceModelVersionId = colList[3]; + newResourceModelCustomizationId = colList[4]; + } + } + } + modelFileLineCounter++; + } + fileName = feedDir+ "INPUT-VNF.csv"; + logger.info(fileName); + logger.info("---------- Processing Entries from file ----------"); + lines = Files.readAllLines(Paths.get(fileName)); + lineItr = lines.iterator(); + while (lineItr.hasNext()){ + String line = lineItr.next().replace("\n", "").replace("\r", ""); + if (!line.isEmpty()) { + if (genericVnfFileLineCounter != 0) { + String[] colList = line.split("\\s*,\\s*", -1); + if(colList[1].equals(fileTypeList.get(i))) { + genericVnfList.add(colList[0]); + } + } else { + this.headerLength = line.split("\\s*,\\s*", -1).length; + logger.info("headerLength: " + headerLength + "\n"); + if (this.headerLength != 2){ + logger.info(MIGRATION_ERROR + "ERROR: Input file should have 2 columns"); + success = false; + return; + } + } + } + genericVnfFileLineCounter++; + } + updateGenericVnfs(fileTypeList.get(i), genericVnfList, newServiceModelInvariantId, newServiceModelVersionId, + newResourceModelInvariantId, newResourceModelVersionId, newResourceModelCustomizationId, vfModuleFileLineList); + } catch (FileNotFoundException e) { + logger.info(MIGRATION_ERROR + "ERROR: Could not file file " + fileName, e.getMessage()); + success = false; + checkLog = true; + } catch (IOException e) { + logger.info(MIGRATION_ERROR + "ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info(MIGRATION_ERROR + "encountered exception", e); + e.printStackTrace(); + success = false; + } + } + logger.info ("\n \n ******* Final Summary for RADCOM Change Migration ********* \n"); + logger.info(MIGRATION_SUMMARY_COUNT + "Total generic-vnfs in File: "+(genericVnfFileLineCounter + 1)); + logger.info(MIGRATION_SUMMARY_COUNT + " generic-vnfs processed: "+ genericVnfMigrationSuccess); + logger.info(MIGRATION_SUMMARY_COUNT + " generic-vnfs failed to process: "+ genericVnfMigrationFailure); + logger.info(MIGRATION_SUMMARY_COUNT + " service-instances processed: "+ serviceInstanceMigrationSuccess); + logger.info(MIGRATION_SUMMARY_COUNT + " service-instances failed to process: "+ serviceInstanceMigrationFailure); + logger.info(MIGRATION_SUMMARY_COUNT + " vf-modules processed: "+ vfModuleMigrationSuccess); + logger.info(MIGRATION_SUMMARY_COUNT + " vf-modules failed to process: "+ vfModuleMigrationFailure); + logger.info(MIGRATION_SUMMARY_COUNT + " images processed: "+ imageMigrationSuccess); + logger.info(MIGRATION_SUMMARY_COUNT + " images failed to process: "+ imageMigrationFailure +"\n"); + } + + private void updateGenericVnfs(String vnfType, ArrayList<String> genericVnfList, String newServiceModelInvariantId, + String newServiceModelVersionId, String newResourceModelInvariantId, String newResourceModelVersionId, + String newResourceModelCustomizationId, ArrayList<VfModuleFileData> vfModuleFileLineList) { + int numberOfNames = genericVnfList.size(); + Vertex newModelVerNode = null; + GraphTraversal<Vertex, Vertex> modelVerNodeList = g.V().has(AAIProperties.NODE_TYPE, MODEL). + has(MODEL_INVARIANT_ID, newResourceModelInvariantId).in("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, MODEL_VER).has(MODEL_VERSION_ID, newResourceModelVersionId); + if(!modelVerNodeList.hasNext()) { + logger.info(MIGRATION_ERROR + "ERROR: Model " + newResourceModelInvariantId + " with model-ver " + + newResourceModelVersionId + " does not exist in database \n"); + for(int i = 0; i < numberOfNames; i++) { + genericVnfMigrationFailure++; + } + } + else { + newModelVerNode = modelVerNodeList.next(); + for(int i = 0; i < numberOfNames; i++) { + GraphTraversal<Vertex, Vertex> genericVnfNodeList = g.V().has(AAIProperties.NODE_TYPE, GENERIC_VNF). + has(VNF_NAME, genericVnfList.get(i)).has(VNF_TYPE, vnfType); + if(!genericVnfNodeList.hasNext()) { + logger.info(MIGRATION_ERROR + "ERROR: Failure to update generic-vnf " + genericVnfList.get(i) + + " Graph Traversal failed \n"); + genericVnfMigrationFailure++; + } + while (genericVnfNodeList.hasNext()) { + Vertex genericVnfVtx = genericVnfNodeList.next(); + boolean updateSuccess = false; + if (genericVnfVtx != null) { + logger.info("Updating generic-vnf " + genericVnfVtx.value(VNF_NAME) + " with " + + "current model-invariant-id " + + (genericVnfVtx.property(MODEL_INVARIANT_ID).isPresent() + ? genericVnfVtx.value(MODEL_INVARIANT_ID) : "null") + + ", current model-version-id " + + (genericVnfVtx.property(MODEL_VERSION_ID).isPresent() + ? genericVnfVtx.value(MODEL_VERSION_ID) : "null") + + ", and current model-customization-id " + + (genericVnfVtx.property(MODEL_CUSTOMIZATION_ID).isPresent() + ? genericVnfVtx.value(MODEL_CUSTOMIZATION_ID) : "null") + + " to use model-invariant-id " + newResourceModelInvariantId + "," + + " model-version-id " + newResourceModelVersionId + " and model-customization-id " + + newResourceModelCustomizationId); + try { + Vertex oldModelVerNode = null; + GraphTraversal<Vertex, Vertex> modelVerQuery= g.V(genericVnfVtx).out("org.onap.relationships.inventory.IsA") + .has(AAIProperties.NODE_TYPE, MODEL_VER); + if(modelVerQuery.hasNext()) { + oldModelVerNode = modelVerQuery.next(); + } + genericVnfVtx.property(MODEL_INVARIANT_ID_LOCAL, newResourceModelInvariantId); + genericVnfVtx.property(MODEL_VERSION_ID_LOCAL, newResourceModelVersionId); + genericVnfVtx.property(MODEL_CUSTOMIZATION_ID, newResourceModelCustomizationId); + if(newModelVerNode.property(MODEL_VERSION).isPresent()) { + genericVnfVtx.property(PERSONA_MODEL_VERSION, newModelVerNode.value(MODEL_VERSION)); + } + this.touchVertexProperties(genericVnfVtx, false); + if(oldModelVerNode != null) { + this.swingEdges(oldModelVerNode, newModelVerNode, GENERIC_VNF, "org.onap.relationships.inventory.IsA", "IN"); + } + else { + this.createPrivateEdge(newModelVerNode, genericVnfVtx); + } + updateSuccess = true; + } catch (Exception e) { + logger.info(e.toString()); + logger.info(MIGRATION_ERROR + "ERROR: Failure to update generic-vnf " + genericVnfList.get(i) + "\n"); + genericVnfMigrationFailure++; + } + if(updateSuccess) { + String dmaapMsg = System.nanoTime() + "_" + genericVnfVtx.id().toString() + "_" + + genericVnfVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + logger.info("Update of generic-vnf " + genericVnfList.get(i) + " successful \n"); + genericVnfMigrationSuccess++; + updateServiceInstances(vnfType, genericVnfList.get(i), newServiceModelInvariantId, + newServiceModelVersionId); + updateVfModules(vnfType, genericVnfList.get(i), newResourceModelInvariantId, newResourceModelVersionId, + vfModuleFileLineList); + } + } + else { + logger.info(MIGRATION_ERROR + "ERROR: Failure to update generic-vnf " + genericVnfList.get(i) + + " Graph Traversal returned an empty vertex \n"); + genericVnfMigrationFailure++; + } + } + } + } + } + + private void updateServiceInstances(String vnfType, String vnfName, String newServiceModelInvariantId, + String newServiceModelVersionId) { + GraphTraversal<Vertex, Vertex> serviceInstanceNodeList = g.V(). + has(AAIProperties.NODE_TYPE, GENERIC_VNF).has(VNF_NAME, vnfName).has(VNF_TYPE, vnfType). + in("org.onap.relationships.inventory.ComposedOf").has(AAIProperties.NODE_TYPE, SERVICE_INSTANCE); + Vertex newModelVerNode = null; + GraphTraversal<Vertex, Vertex> modelVerNodeList = g.V().has(AAIProperties.NODE_TYPE, MODEL). + has(MODEL_INVARIANT_ID, newServiceModelInvariantId).in("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, MODEL_VER).has(MODEL_VERSION_ID, newServiceModelVersionId); + if(!modelVerNodeList.hasNext()) { + logger.info(MIGRATION_ERROR + "ERROR: Model " + newServiceModelInvariantId + " with model-ver " + + newServiceModelVersionId + " does not exist in database \n"); + while(serviceInstanceNodeList.hasNext()) { + serviceInstanceNodeList.next(); + serviceInstanceMigrationFailure++; + } + } + else { + newModelVerNode = modelVerNodeList.next(); + while (serviceInstanceNodeList.hasNext()) { + Vertex serviceInstanceVtx = serviceInstanceNodeList.next(); + boolean updateSuccess = false; + if (serviceInstanceVtx != null) { + logger.info("Updating service-instance " + serviceInstanceVtx.value(SERVICE_INSTANCE_ID) + + " with current model-invariant-id " + + (serviceInstanceVtx.property(MODEL_INVARIANT_ID).isPresent() + ? serviceInstanceVtx.value(MODEL_INVARIANT_ID) : "null") + + " and current model-version-id " + + (serviceInstanceVtx.property(MODEL_VERSION_ID).isPresent() + ? serviceInstanceVtx.value(MODEL_VERSION_ID) : "null") + + " to use model-invariant-id " + newServiceModelInvariantId + " and" + + " model-version-id " + newServiceModelVersionId); + try { + Vertex oldModelVerNode = null; + GraphTraversal<Vertex, Vertex> modelVerQuery= g.V(serviceInstanceVtx).out("org.onap.relationships.inventory.IsA") + .has(AAIProperties.NODE_TYPE, MODEL_VER); + if(modelVerQuery.hasNext()) { + oldModelVerNode = modelVerQuery.next(); + } + serviceInstanceVtx.property(MODEL_INVARIANT_ID_LOCAL, newServiceModelInvariantId); + serviceInstanceVtx.property(MODEL_VERSION_ID_LOCAL, newServiceModelVersionId); + if(newModelVerNode.property(MODEL_VERSION).isPresent()) { + serviceInstanceVtx.property(PERSONA_MODEL_VERSION, newModelVerNode.value(MODEL_VERSION)); + } + this.touchVertexProperties(serviceInstanceVtx, false); + if(oldModelVerNode != null) { + this.swingEdges(oldModelVerNode, newModelVerNode, SERVICE_INSTANCE, "org.onap.relationships.inventory.IsA", "IN"); + } + else { + this.createPrivateEdge(newModelVerNode, serviceInstanceVtx); + } + updateSuccess = true; + } catch (Exception e) { + logger.info(e.toString()); + logger.info(MIGRATION_ERROR + "ERROR: Failure to update service-instance " + + serviceInstanceVtx.value(SERVICE_INSTANCE_ID) + "\n"); + serviceInstanceMigrationFailure++; + } + if(updateSuccess) { + String dmaapMsg = System.nanoTime() + "_" + serviceInstanceVtx.id().toString() + "_" + + serviceInstanceVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + logger.info("Update of service-instance " + + serviceInstanceVtx.value(SERVICE_INSTANCE_ID) + " successful \n"); + serviceInstanceMigrationSuccess++; + } + } + } + } + } + + private void updateVfModules(String vnfType, String vnfName, String newResourceModelInvariantId, + String newResourceModelVersionId, ArrayList<VfModuleFileData> vfModuleFileLineList) { + int numberOfLines = vfModuleFileLineList.size(); + ArrayList<Integer> processedNodes = new ArrayList<Integer>(); + for(int i = 0; i < numberOfLines; i++) { + VfModuleFileData currentLine = vfModuleFileLineList.get(i); + String vfModuleId = currentLine.getVfModuleId(); + String vfModuleModelName = currentLine.getVfModuleModelName(); + String imageName = currentLine.getImageName(); + String vfModuleInvariantId = ""; + String vfModuleVersionId = ""; + GraphTraversal<Vertex, Vertex> vfModuleNodeList = g.V(). + has(AAIProperties.NODE_TYPE, GENERIC_VNF).has(VNF_NAME, vnfName).has(VNF_TYPE, vnfType). + in("org.onap.relationships.inventory.BelongsTo").has(AAIProperties.NODE_TYPE, VF_MODULE). + has(VF_MODULE_ID, vfModuleId); + if(vfModuleNodeList.hasNext()) { + GraphTraversal<Vertex, Vertex> modelElementNodeList = g.V(). + has(AAIProperties.NODE_TYPE, MODEL).has(MODEL_INVARIANT_ID, newResourceModelInvariantId). + in("org.onap.relationships.inventory.BelongsTo").has(AAIProperties.NODE_TYPE, MODEL_VER). + has(MODEL_VERSION_ID, newResourceModelVersionId).in("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, MODEL_ELEMENT); + while(modelElementNodeList.hasNext()) { + Vertex modelElement = modelElementNodeList.next(); + GraphTraversal<Vertex, Vertex> modelVersionLookup = g.V(modelElement).out("org.onap.relationships.inventory.IsA"). + has(AAIProperties.NODE_TYPE, MODEL_VER); + while(modelVersionLookup.hasNext()) { + Vertex modelVersionVertex = modelVersionLookup.next(); + if(modelVersionVertex.value(MODEL_NAME).equals(vfModuleModelName)) { + vfModuleVersionId = modelVersionVertex.value(MODEL_VERSION_ID); + vfModuleInvariantId = g.V(modelVersionVertex).out("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, MODEL).next().value(MODEL_INVARIANT_ID); + break; + } + } + if(!vfModuleVersionId.isEmpty() && !vfModuleInvariantId.isEmpty()) { + break; + } + GraphTraversal<Vertex, Vertex> modelElementLookup = g.V(modelElement).in("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, MODEL_ELEMENT); + while(modelElementLookup.hasNext()) { + ArrayList<String> returnedValues = recursiveSearchForModelName(vfModuleModelName, modelElementLookup.next()); + if(!returnedValues.isEmpty()) { + vfModuleInvariantId = returnedValues.get(0); + vfModuleVersionId = returnedValues.get(1); + break; + } + } + if(!vfModuleVersionId.isEmpty() && !vfModuleInvariantId.isEmpty()) { + break; + } + } + while (vfModuleNodeList.hasNext()) { + Vertex vfModuleVtx = vfModuleNodeList.next(); + boolean updateSuccess = false; + if (vfModuleVtx != null) { + if(vfModuleInvariantId.isEmpty() && vfModuleVersionId.isEmpty()) { + logger.info(MIGRATION_ERROR + "ERROR: Failure to update vf-module " +vfModuleVtx.value(VF_MODULE_ID) + + ". model-invariant-id and model-version-id not found \n"); + vfModuleMigrationFailure++; + } + else if(vfModuleInvariantId.isEmpty()) { + logger.info(MIGRATION_ERROR + "ERROR: Failure to update vf-module " +vfModuleVtx.value(VF_MODULE_ID) + + ". model-invariant-id not found \n"); + vfModuleMigrationFailure++; + } + else if(vfModuleVersionId.isEmpty()) { + logger.info(MIGRATION_ERROR + "ERROR: Failure to update vf-module " +vfModuleVtx.value(VF_MODULE_ID) + + ". model-version-id not found \n"); + vfModuleMigrationFailure++; + } + else { + logger.info("Updating vf-module " + vfModuleVtx.value(VF_MODULE_ID) + + " with current model-invariant-id " + + (vfModuleVtx.property(MODEL_INVARIANT_ID).isPresent() + ? vfModuleVtx.value(MODEL_INVARIANT_ID) : "null") + + " and current model-version-id " + + (vfModuleVtx.property(MODEL_VERSION_ID).isPresent() + ? vfModuleVtx.value(MODEL_VERSION_ID) : "null") + + " to use model-invariant-id " + vfModuleInvariantId + " and" + + " model-version-id " + vfModuleVersionId); + Vertex newModelVerNode = null; + GraphTraversal<Vertex, Vertex> modelVerNodeList = g.V().has(AAIProperties.NODE_TYPE, MODEL). + has(MODEL_INVARIANT_ID, vfModuleInvariantId).in("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, MODEL_VER).has(MODEL_VERSION_ID, vfModuleVersionId); + if(!modelVerNodeList.hasNext()) { + logger.info(MIGRATION_ERROR + "ERROR: Model " + vfModuleInvariantId + " with model-ver " + + vfModuleVersionId + " could not be found in traversal, error in finding vf-module model \n"); + vfModuleMigrationFailure++; + } + else { + newModelVerNode = modelVerNodeList.next(); + try { + Vertex oldModelVerNode = null; + GraphTraversal<Vertex, Vertex> modelVerQuery= g.V(vfModuleVtx).out("org.onap.relationships.inventory.IsA") + .has(AAIProperties.NODE_TYPE, MODEL_VER); + if(modelVerQuery.hasNext()) { + oldModelVerNode = modelVerQuery.next(); + } + vfModuleVtx.property(MODEL_INVARIANT_ID_LOCAL, vfModuleInvariantId); + vfModuleVtx.property(MODEL_VERSION_ID_LOCAL, vfModuleVersionId); + if(newModelVerNode.property(MODEL_VERSION).isPresent()) { + vfModuleVtx.property(PERSONA_MODEL_VERSION, newModelVerNode.value(MODEL_VERSION)); + } + this.touchVertexProperties(vfModuleVtx, false); + if(oldModelVerNode != null) { + this.swingEdges(oldModelVerNode, newModelVerNode, VF_MODULE, "org.onap.relationships.inventory.IsA", "IN"); + } + else { + this.createPrivateEdge(newModelVerNode, vfModuleVtx); + } + updateSuccess = true; + } catch (Exception e) { + logger.info(e.toString()); + logger.info(MIGRATION_ERROR + "ERROR: Failure to update vf-module " + + vfModuleVtx.value(VF_MODULE_ID) + "\n"); + vfModuleMigrationFailure++; + } + } + } + } + if(updateSuccess) { + String dmaapMsg = System.nanoTime() + "_" + vfModuleVtx.id().toString() + "_" + + vfModuleVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + logger.info("Update of vf-module " + + vfModuleVtx.value(VF_MODULE_ID) + " successful \n"); + vfModuleMigrationSuccess++; + if(!processedNodes.contains(i)) { + processedNodes.add(i); + } + updateVserverAndImage(vfModuleId, imageName); + } + } + } + } + int processedNodesNum = processedNodes.size(); + for (int i = 0; i < processedNodesNum; i++) { + vfModuleFileLineList.remove(i); + } + } + + private ArrayList<String> recursiveSearchForModelName(String vfModuleModelName, Vertex modelElement) { + ArrayList<String> returnedValues = new ArrayList<String>(); + GraphTraversal<Vertex, Vertex> modelVersionLookup = g.V(modelElement).out("org.onap.relationships.inventory.IsA"). + has(AAIProperties.NODE_TYPE, MODEL_VER); + while(modelVersionLookup.hasNext()) { + Vertex modelVersionVertex = modelVersionLookup.next(); + if(modelVersionVertex.value(MODEL_NAME).equals(vfModuleModelName)) { + returnedValues.add(modelVersionVertex.value(MODEL_VERSION_ID)); + returnedValues.add(0, g.V(modelVersionVertex).out("org.onap.relationships.inventory.BelongsTo") + .next().value(MODEL_INVARIANT_ID)); + return returnedValues; + } + } + GraphTraversal<Vertex, Vertex> modelElementLookup = g.V(modelElement).in("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, MODEL_ELEMENT); + while(modelElementLookup.hasNext()) { + returnedValues = recursiveSearchForModelName(vfModuleModelName, modelElementLookup.next()); + if(!returnedValues.isEmpty()) { + return returnedValues; + } + } + return returnedValues; + } + + private void updateVserverAndImage(String vfModuleId, String imageName) { + GraphTraversal<Vertex, Vertex> vserverNodeList = g.V(). + has(AAIProperties.NODE_TYPE, VF_MODULE).has(VF_MODULE_ID, vfModuleId). + out("org.onap.relationships.inventory.Uses").has(AAIProperties.NODE_TYPE, VSERVER); + while (vserverNodeList.hasNext()) { + Vertex vserverVtx = vserverNodeList.next(); + boolean updateSuccess = false; + GraphTraversal<Vertex, Vertex> oldImageLookup = g.V(vserverVtx).out("org.onap.relationships.inventory.Uses"). + has(AAIProperties.NODE_TYPE, IMAGE); + Vertex oldImageVtx = null; + if(oldImageLookup.hasNext()) { + oldImageVtx = oldImageLookup.next(); + } + GraphTraversal<Vertex, Vertex> newImageLookup = g.V(vserverVtx).out("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, TENANT).out("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, CLOUD_REGION).in("org.onap.relationships.inventory.BelongsTo"). + has(AAIProperties.NODE_TYPE, IMAGE).has(IMAGE_NAME, imageName); + Vertex newImageVtx = null; + if(newImageLookup.hasNext()) { + newImageVtx = newImageLookup.next(); + } + if (vserverVtx != null && newImageVtx!= null) { + logger.info("Updating vserver " + vserverVtx.value(VSERVER_ID) + + " to replace all current image relationships with relationship to new image " + imageName); + try { + if(oldImageVtx != null) { + this.swingEdges(oldImageVtx, newImageVtx, VSERVER, "org.onap.relationships.inventory.Uses", "IN"); + } + else { + this.createEdgeIfPossible(EdgeType.COUSIN, vserverVtx, newImageVtx); + } + updateSuccess = true; + } catch (Exception e) { + logger.info(e.toString()); + logger.info(MIGRATION_ERROR + "ERROR: Failure to update vserver " + + vserverVtx.value(VSERVER_ID) + " with image " + imageName + "\n"); + imageMigrationFailure++; + } + if(updateSuccess) { + logger.info("Update of vserver " + + vserverVtx.value(VSERVER_ID) + " with image " + newImageVtx.value(IMAGE_NAME) + + " successful \n"); + imageMigrationSuccess++; + } + } + } + } + + @Override + public Status getStatus() { + if (checkLog) { + return Status.CHECK_LOGS; + } + else if (success) { + return Status.SUCCESS; + } + else { + return Status.FAILURE; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{GENERIC_VNF, SERVICE_INSTANCE, VF_MODULE, VSERVER, IMAGE}); + } + + @Override + public String getMigrationName() { + return "MigrateRadcomChanges"; + } + + @Override + public List<Pair<Vertex, Vertex>> getAffectedNodePairs() { + return null; + } + + @Override + public String getNodeTypeRestriction() { + return VSERVER; + } + + @Override + public String getEdgeLabelRestriction() { + return "org.onap.relationships.inventory.Uses"; + } + + @Override + public String getEdgeDirRestriction() { + return "IN"; + } + + @Override + public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) { + } +} |