diff options
Diffstat (limited to 'src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartTwo.java')
| -rw-r--r-- | src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartTwo.java | 508 |
1 files changed, 508 insertions, 0 deletions
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartTwo.java b/src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartTwo.java new file mode 100644 index 0000000..bb525c3 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v13/MigrateBadWidgetModelsPartTwo.java @@ -0,0 +1,508 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v13; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.javatuples.Pair; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.*; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; + + +@Enabled +@MigrationPriority(21) +@MigrationDangerRating(100) +public class MigrateBadWidgetModelsPartTwo extends EdgeSwingMigrator { + private boolean success = true; + private final GraphTraversalSource g; + + // NOTE -- this migration is for "model-ver" nodes only. It needs to be run AFTER + // the MigrateWidgetModelsPartOne. + // + + // migration restrictions that we will use for this migration + private final String NODE_TYPE_RESTRICTION = "model-element"; + private final String EDGE_LABEL_RESTRICTION = "org.onap.relationships.inventory.IsA"; + private final String EDGE_DIR_RESTRICTION = "IN"; + + GraphTraversal<Vertex, Vertex> widgetModelTraversal; + GraphTraversal<Vertex, Vertex> widgetModelVersionTraversal; + GraphTraversal<Vertex, Vertex> validModVerTraversal; + + + + public MigrateBadWidgetModelsPartTwo(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + this.g = this.engine.asAdmin().getTraversalSource(); + } + + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{"model", "model-element", "model-ver"}); + } + + @Override + public String getMigrationName() { + return "MigrateBadWidgetModelsPartTwo"; + } + + + /** + * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate. + * @return + */ + @Override + public List<Pair<Vertex, Vertex>> getAffectedNodePairs() { + logAndPrintInfo("--------- GET AFFECTED NODE PAIRS -------------"); + // Read the json file to populate the validWidgetModelVesionIdHash and also + // validWidgetModelInvIdHash which will be used to figure out which data is in the db with + // an invalid id. + ArrayList <String> fileLines = readInValidWidgetInfoFile(); + + // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType + // Note - we currently only have one valid version per model for widget models. + HashMap <String,String> validModelVersionIdHash = getModelVersionIdHash( fileLines ); + + // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType + // Note - we currently only have one valid version per model for widget models. + HashMap <String,String> validModelInvariantIdHash = getModelInvariantIdHash( fileLines ); + + // Now we will see what is actually in the DB + List<Pair<Vertex, Vertex>> fromToVertPairList = new ArrayList<Pair<Vertex, Vertex>>(); + widgetModelTraversal = this.engine.asAdmin().getTraversalSource().V() + .has("aai-node-type", "model") + .has("model-type", "widget"); + + if(!(widgetModelTraversal.hasNext())){ + logAndPrintInfo("unable to find widget models in database. "); + } + + int validModelVerCount = 0; + while (widgetModelTraversal.hasNext()) { + Vertex widgetModVertex = widgetModelTraversal.next(); + String invId = widgetModVertex.property("model-invariant-id").value().toString(); + + // Find the model-version nodes that belong to this model. + // We expect just one per model, but there could be more. + widgetModelVersionTraversal = this.engine.asAdmin().getTraversalSource() + .V(widgetModVertex) + .in("org.onap.relationships.inventory.BelongsTo") + .has("aai-node-type", "model-ver"); + + if(!(widgetModelVersionTraversal.hasNext())){ + logAndPrintInfo("unable to find widget model version in database for model-invariant-id = [" + invId + "]."); + } + + while (widgetModelVersionTraversal.hasNext()) { + Vertex widgetModVersionVertex = widgetModelVersionTraversal.next(); + String modVersionIdInDb = widgetModVersionVertex.property("model-version-id").value().toString(); + String nodeType = widgetModVersionVertex.property("model-name").value().toString(); + + if( validModelVersionIdHash.containsKey(nodeType) ){ + // We know what the model-version-id SHOULD be, so make sure we're using it. + String validModVerId = validModelVersionIdHash.get(nodeType); + if( !modVersionIdInDb.equals(validModVerId) ){ + logAndPrintInfo(" Bad model-version-id found in DB for model-name = " + nodeType + ", verId = " + modVersionIdInDb ); + validModVerTraversal = this.engine.asAdmin().getTraversalSource() + .V() + .has("model-version-id",validModVerId) + .has("aai-node-type","model-ver"); + if(!(validModVerTraversal.hasNext())){ + logAndPrintInfo("unable to find widget model version in database for valid model-version-id = [" + validModVerId + "]."); + } + int ct = 0; + while (validModVerTraversal.hasNext()) { + ct++; + if( ct > 1 ){ + logAndPrintInfo("ERROR - More than one model-ver found for model-version-id = [" + validModVerId + "]."); + break; + } + Vertex toVert = validModVerTraversal.next(); + fromToVertPairList.add(new Pair<>(widgetModVersionVertex, toVert)); + } + } + else { + validModelVerCount++; + logAndPrintInfo("Valid model-version-id used in DB for model-name = [" + nodeType + "]."); + } + } + else { + logAndPrintInfo("unable to find a valid widget model-ver in database for model-name = [" + nodeType + "]."); + } + } + } + + return fromToVertPairList; + } + + + public String getNodeTypeRestriction(){ + return NODE_TYPE_RESTRICTION; + } + + public String getEdgeLabelRestriction(){ + return EDGE_LABEL_RESTRICTION; + } + + public String getEdgeDirRestriction(){ + return EDGE_DIR_RESTRICTION; + } + + /** + * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate. + * @return + */ + public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) { + + // The first node in each pair is the model-ver that we were migrating edges AWAY FROM because + // it is an invalid model-ver node. + // Delete those as well as their parent model node (if the parent model node has no other users + // and is not on the validModelInvIdList). + + int badModelVerCount = 0; + int modelVerDelCount = 0; + int modelDelCount = 0; + int parentPreventValidDelCount = 0; + + HashMap <String,String> parentPreventInEdgeIdHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it. + HashMap <String,String> parentPreventOutEdgeIdHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it. + HashMap <String,String> parentPreventIsaEdgeDelHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it. + + ArrayList <String> fileLines = readInValidWidgetInfoFile(); + // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType + // Note - we currently only have one valid version per model for widget models. + HashMap <String,String> validModelInvariantIdHash = getModelInvariantIdHash( fileLines ); + + try { + for (Pair<Vertex, Vertex> nodePair : nodePairL) { + // The "fromNode" is the "bad/old" model-ver node that we moved off of + badModelVerCount++; + Vertex oldNode = nodePair.getValue0(); + String oldModVerId = oldNode.property("model-version-id").value().toString(); + Vertex parentModelNode = null; + + //DOUBLE CHECK THAT THIS IS NOT a valid model-version-id + + + boolean okToDelete = true; + //---- delete the oldNode if the only edge it has is its "belongsTo/OUT" edge to its parent model. + // AND if its parent node does not have any named-query edges ("IsA" edges) pointing to it. + Iterator <Edge> edgeInIter = oldNode.edges(Direction.IN); + while( edgeInIter.hasNext() ){ + Edge inE = edgeInIter.next(); + Vertex otherSideNode4ThisEdge = inE.inVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + // If there are any IN edges, we won't delete this thing. + okToDelete = false; + logAndPrintInfo("We will not delete old model-ver node because it still has IN edges. This model-version-id = [" + + oldModVerId + "], has IN edge from a [" + otherSideNodeType + "] node. "); + } + if( okToDelete ){ + // there were no OUT edges, make sure the only OUT edge is to it's parent + Iterator <Edge> edgeOutIter = oldNode.edges(Direction.OUT); + int edgeCount = 0; + while( edgeOutIter.hasNext() ){ + Edge badModVerE = edgeOutIter.next(); + edgeCount++; + if( edgeCount > 1 ){ + // If there are more than one OUT edges, we won't delete this thing. + okToDelete = false; + parentModelNode = null; + logAndPrintInfo("We will not delete old model-ver node because it still has > 1 OUT-edges. model-version-id = [" + oldModVerId + "]."); + } + else { + String eLabel = badModVerE.label().toString(); + Vertex otherSideNode4ThisEdge = badModVerE.inVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + if( ! eLabel.equals("org.onap.relationships.inventory.BelongsTo") ){ + logAndPrintInfo("We will not delete old model-ver node because it still has a non 'belongsTo' OUT-edge. model-version-id = [" + + oldModVerId + "], edgeLabel = [" + eLabel + "] edge goes to a [" + otherSideNodeType + "]. "); + okToDelete = false; + } + else { + if( ! otherSideNodeType.equals("model") ){ + logAndPrintInfo("We will not delete old model-ver node (model-version-id = [" + oldModVerId + "]) " + + " because it still has an OUT edge to a [" + otherSideNodeType + "] node. "); + okToDelete = false; + parentModelNode = null; + } + else { + parentModelNode = otherSideNode4ThisEdge; + String parentInvId = parentModelNode.property("model-invariant-id").value().toString(); + Iterator <Edge> pInIter = parentModelNode.edges(Direction.IN); + while( pInIter.hasNext() ){ + Edge inE = pInIter.next(); + String inELabel = inE.label().toString(); + if( ! inELabel.equals("org.onap.relationships.inventory.BelongsTo") ){ + Vertex otherSideNode = inE.outVertex(); + String otherSideNT = otherSideNode.value(AAIProperties.NODE_TYPE); + // If there are any IN edges still on the parent, + // we won't delete this model-ver since once the model-ver + // is gone, its hard to know what nodeType the model was + // for - so it would be hard to know what valid model-invariant-id + // to migrate its edges to. + okToDelete = false; + parentPreventIsaEdgeDelHash.put(parentInvId,""); + logAndPrintInfo("We will not delete old model-ver node because its" + + " parent model still has IN edges. The model with model-invariant-id = [" + + parentInvId + "], has an non-belongsTo IN edge, label = [" + + inELabel + "] from a [" + otherSideNT + "] node. "); + } + } + } + } + } + } + } + + if( okToDelete ){ + logAndPrintInfo(" >>> DELETEING model-ver node with model-version-id = [" + oldModVerId + "]" ); + modelVerDelCount++; + oldNode.remove(); + } + + if( parentModelNode != null && okToDelete ){ + // Delete the corresponding parent model IF it now has no + // edges anymore (and is not in our known valid model list) + // and we were deleting the model-ver also. + boolean okToDelParent = true; + String parentModInvId = parentModelNode.property("model-invariant-id").value().toString(); + + if( validModelInvariantIdHash.containsValue(parentModInvId) ){ + okToDelParent = false; + logAndPrintInfo("We will not delete old model node because it is on our valid widget list. " + + " model-invariant-id = [" + parentModInvId + "] "); + parentPreventValidDelCount++; + } + else { + Iterator <Edge> pInIter = parentModelNode.edges(Direction.IN); + while( pInIter.hasNext() ){ + Edge inE = pInIter.next(); + String inELabel = inE.label().toString(); + Vertex otherSideNode4ThisEdge = inE.outVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + // If there are any IN edges, we won't delete this thing. + okToDelParent = false; + parentPreventInEdgeIdHash.put(parentModInvId, ""); + logAndPrintInfo("We will not delete old model node (yet) because it still has IN edges. This model-invariant-id = [" + + parentModInvId + "], has IN edge, label = [" + + inELabel + "] from a [" + otherSideNodeType + "] node. "); + } + Iterator <Edge> pOutIter = parentModelNode.edges(Direction.OUT); + while( pOutIter.hasNext() ){ + Edge outE = pOutIter.next(); + String outELabel = outE.label().toString(); + Vertex otherSideNode4ThisEdge = outE.inVertex(); + String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE); + // If there are any OUT edges, we won't delete this thing. + okToDelParent = false; + parentPreventOutEdgeIdHash.put(parentModInvId, ""); + logAndPrintInfo("We will not delete old model node because it still has OUT edges. This model-invariant-id = [" + + parentModInvId + "], has OUT edge, label = [" + + outELabel + "] to a [" + otherSideNodeType + "] node. "); + } + } + + if( okToDelParent ){ + if( parentPreventInEdgeIdHash.containsKey(parentModInvId) ){ + // This parent had been prevented from being deleted until all its + // child model-ver's were deleted (it must have had more than one). + // So we can now remove it from the list of parent guys that + // could not be deleted. + parentPreventInEdgeIdHash.remove(parentModInvId); + } + logAndPrintInfo(" >>> DELETEING model node which was the parent of model-ver with model-version-id = [" + + oldModVerId + "]. This model-invariant-id = [" + parentModInvId + "]" ); + modelDelCount++; + parentModelNode.remove(); + } + } + } + + logAndPrintInfo(" >>> SUMMARY: total number of bad model-ver nodes found = " + badModelVerCount ); + logAndPrintInfo(" >>> SUMMARY: number of model-ver nodes deleted = " + modelVerDelCount ); + logAndPrintInfo(" >>> SUMMARY: number of model nodes deleted = " + modelDelCount ); + logAndPrintInfo(" >>> SUMMARY: number of model-ver nodes not deleted because their PARENT still had IsA edges = " + + parentPreventIsaEdgeDelHash.size() ); + logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they were valid = " + + parentPreventValidDelCount); + logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they had IN edges = " + + parentPreventInEdgeIdHash.size() ); + logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they had OUT edges = " + + parentPreventOutEdgeIdHash.size() ); + + + } catch (Exception e) { + logger.error("error encountered", e ); + success = false; + } + + } + + private ArrayList <String> readInValidWidgetInfoFile(){ + + ArrayList <String> fileLines = new ArrayList <String> (); + String homeDir = System.getProperty("AJSC_HOME"); + String configDir = System.getProperty("BUNDLECONFIG_DIR"); + if (homeDir == null) { + logAndPrintInfo("ERROR: Could not find sys prop AJSC_HOME"); + success = false; + return fileLines; + } + if (configDir == null) { + logAndPrintInfo("ERROR: Could not find sys prop BUNDLECONFIG_DIR"); + success = false; + return fileLines; + } + String fileName = homeDir + "/" + configDir + "/" + "migration-input-files/widget-model-migration-data/widget-model-migration-input.csv"; + try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { + String modelInfoLine; + while ((modelInfoLine = br.readLine()) != null) { + modelInfoLine = modelInfoLine.replace("\n", "").replace("\r", ""); + if (!modelInfoLine.isEmpty()) { + fileLines.add(modelInfoLine); + } + } + } + catch (FileNotFoundException e) { + logger.error("ERROR: Could not find file " + fileName, e); + success = false; + } catch (IOException e) { + logger.error("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.error("encountered exception", e); + e.printStackTrace(); + success = false; + } + return fileLines; + } + + + HashMap <String,String> getModelVersionIdHash( ArrayList <String> fileLines ){ + + HashMap <String, String> versionIdHash = new HashMap <String,String> (); + + if( fileLines == null ){ + logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash"); + success = false; + return versionIdHash; + } + + for(int i = 0; i < fileLines.size(); i++ ){ + String mLine = fileLines.get(i); + String[] fields = mLine.split("\\,"); + if (fields.length != 3) { + logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line."); + success = false; + } + else { + versionIdHash.put(fields[0],fields[1]); + } + } + + // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is + // the correct model info for "virtual-data-center". Problem is that there is no vdc nodeType, but + // there are named-queries pointing at a bad widget-model for "vdc". + String virtDataCenterVerId = versionIdHash.get("virtual-data-center"); + if( virtDataCenterVerId != null ){ + versionIdHash.put("vdc",virtDataCenterVerId ); + } + + return versionIdHash; + } + + + HashMap <String,String> getModelInvariantIdHash( ArrayList <String> fileLines ){ + HashMap <String, String> invIdHash = new HashMap <String,String> (); + + if( fileLines == null ){ + logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash"); + success = false; + return invIdHash; + } + + for(int i = 0; i < fileLines.size(); i++ ){ + String mLine = fileLines.get(i); + String[] fields = mLine.split("\\,"); + if (fields.length != 3) { + logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line."); + success = false; + } + else { + invIdHash.put(fields[0],fields[2]); + } + } + + // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is + // the correct model info for "virtual-data-center". Problem is that there is no vdc nodeType, but + // there are named-queries pointing at a bad widget-model for "vdc". + String virtDataCenterInvId = invIdHash.get("virtual-data-center"); + if( invIdHash != null ){ + invIdHash.put("vdc",virtDataCenterInvId ); + } + return invIdHash; + } + + /** + * Log and print. + * + * @param msg + * the msg + */ + protected void logAndPrintInfo(String msg) { + System.out.println(msg); + logger.info(msg); + } + + + +}
\ No newline at end of file |