diff options
Diffstat (limited to 'src/main/java/org/onap/aai/audit/AuditGraphson2Sql.java')
| -rw-r--r-- | src/main/java/org/onap/aai/audit/AuditGraphson2Sql.java | 605 |
1 files changed, 605 insertions, 0 deletions
diff --git a/src/main/java/org/onap/aai/audit/AuditGraphson2Sql.java b/src/main/java/org/onap/aai/audit/AuditGraphson2Sql.java new file mode 100644 index 0000000..ff29157 --- /dev/null +++ b/src/main/java/org/onap/aai/audit/AuditGraphson2Sql.java @@ -0,0 +1,605 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.audit; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.javatuples.Triplet; +import org.onap.aai.restclient.PropertyPasswordConfiguration; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.EdgeRule; +import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; +import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.rest.client.ApertureService; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; +import org.onap.aai.util.AAIConstants; +import org.onap.aai.util.ExceptionTranslator; +import org.onap.aai.util.FormatDate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.stream.Collectors; + +import com.att.eelf.configuration.Configuration; +import org.springframework.stereotype.Component; + +@Component +public class AuditGraphson2Sql { + + private EdgeIngestor ei; + private Loader loader; + private static final Logger LOGGER = LoggerFactory.getLogger(AuditGraphson2Sql.class.getSimpleName()); + private SchemaVersions schemaVersions; + private ApertureService apertureService; + + public static final String DEFAULT_SRC_DIR = "logs/data/dataSnapshots/"; + public static final int DEFAULT_THRESHOLD_PERCENT = 10; + public static final String DEFAULT_OUTPUT_DIR = "logs/data/audit"; + + //DEBUG -- should be getting default-src-dir, default-output-dir and rdbms-db-name from param file + @Autowired + public AuditGraphson2Sql( EdgeIngestor ei, SchemaVersions schemaVersions, LoaderFactory loaderFactory, ApertureService apertureService) { + this.schemaVersions = schemaVersions; + this.loader = loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion()); + this.ei = ei; + this.apertureService = apertureService; + } + + public String runAudit(String dbName, String sourceFName, String sourceDir ) + throws Exception { + + if( sourceDir == null || sourceDir.isEmpty() ){ + sourceDir = DEFAULT_SRC_DIR; + } + + HashMap <String,Integer> gsonTableCounts = new HashMap <> (); + try { + gsonTableCounts = getCountsFromGraphsonSnapshot(dbName, sourceFName, sourceDir); + } catch( Exception e ) { + LOGGER.error(" ERROR when calling getCountsFromGraphsonSnapshot(" + + dbName + "," + + sourceFName + "). Exception = " + + e.getMessage() ); + throw e; + } + + long timestamp = 0L; + try { + timestamp = getTimestampFromFileName(sourceFName); + } catch( Exception e ) { + LOGGER.error(" ERROR getting timestamp from filename: " + + e.getMessage() ); + throw e; + } + + JsonObject rdbmsJsonResults = new JsonObject (); + try { + rdbmsJsonResults = getRdbmsTableCounts(timestamp, dbName); + } catch( Exception e ) { + LOGGER.error(" ERROR getting table count info from mySql. timestamp = [" + + timestamp + "], dbName = [" + dbName + + "]. Exception = " + e.getMessage() ); + throw e; + } + HashMap <String,Integer> sqlNodeCounts = getNodeCountsFromSQLRes(rdbmsJsonResults); + HashMap <String,Integer> sqlEdgeCounts = getEdgeCountsFromSQLRes(rdbmsJsonResults); + + String resultString = "Audit ran and logged to file: "; + try { + String titleInfo = "Comparing data from GraphSon file: " + + sourceFName + ", and Aperture data using timeStamp = [" + + timestamp + "] "; + String outFileName = compareResults(gsonTableCounts, sqlNodeCounts, + sqlEdgeCounts, DEFAULT_OUTPUT_DIR, titleInfo); + resultString = resultString + outFileName; + } + catch( IOException ie) { + LOGGER.error(" ERROR writing to output file. [" + ie.getMessage() + "]" ); + throw new Exception( ie.getMessage() ); + } + + return resultString; + + } + + + public String compareResults(HashMap <String,Integer> gsonTableCounts, + HashMap <String,Integer> sqlNodeCounts, + HashMap <String,Integer> sqlEdgeCounts, + String outputDir, String titleInfo) + throws IOException { + + FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); + String dteStr = fd.getDateTime(); + String fName = outputDir + "/" + "gson-sql-audit" + dteStr; + File auditOutFile = new File(fName); + auditOutFile.getParentFile().mkdirs(); + BufferedWriter outWriter = new BufferedWriter(new FileWriter(auditOutFile)); + outWriter.write(titleInfo); + outWriter.newLine(); + outWriter.write("Totals from Graphson: " + getGsonTotals(gsonTableCounts)); + outWriter.newLine(); + outWriter.write("Totals from mySql: " + getMSqlTotals(sqlNodeCounts,sqlEdgeCounts)); + outWriter.newLine(); + outWriter.newLine(); + + for (Map.Entry<String, Integer> entry : gsonTableCounts.entrySet()) { + String tblKey = entry.getKey(); + int gVal = entry.getValue(); + if(!sqlNodeCounts.containsKey(tblKey) && !sqlEdgeCounts.containsKey(tblKey)) { + String msg = "> Entry for Table: [" + tblKey + + "] not found in the SQL db but had " + + gVal + " entries in the GraphSon db"; + LOGGER.error(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + continue; + } + int sVal = 0; + if( sqlNodeCounts.containsKey(tblKey) ) { + sVal = sqlNodeCounts.get(tblKey); + }else { + sVal = sqlEdgeCounts.get(tblKey); + } + if ((gVal > 0) && (sVal == 0)){ + String msg = "For Table [" + tblKey + "], GSon count = " + + gVal + ", zero found in SQL db."; + LOGGER.error(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + } + String msg = tblKey + ": gson/sql = " + gVal + "/" + sVal; + LOGGER.debug(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + } + for (Map.Entry<String, Integer> entry : sqlNodeCounts.entrySet()) { + // check for Node types that are no longer on the Graphson side, but are + // still in the SQL DB + String tblKey = entry.getKey(); + int sVal = entry.getValue(); + if(!gsonTableCounts.containsKey(tblKey)) { + String msg = " Entry for Table [" + tblKey + + "] not found in the Graphson db, but had " + + sVal + " entries in the SQL db."; + LOGGER.error(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + continue; + } + } + for (Map.Entry<String, Integer> entry : sqlEdgeCounts.entrySet()) { + // check for Edge+Label combos that are no longer on the Graphson side, but are + // still in the SQL DB + String tblKey = entry.getKey(); + int sVal = entry.getValue(); + if(!gsonTableCounts.containsKey(tblKey)) { + String msg = " Entry for edge+Label combo [" + tblKey + + "] not found in the Graphson db, but had " + + sVal + " entries in the SQL db."; + LOGGER.error(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + continue; + } + } + outWriter.close(); + String msg = "Audit Results written to: " + fName; + LOGGER.debug(msg); + System.out.println(msg); + return fName; + + } + + + public HashMap <String,Integer> getNodeCountsFromSQLRes(JsonObject sqlJsonData) { + + HashMap<String,Integer> nodeCtHash = new HashMap<>(); + if( sqlJsonData != null ) { + for (Object e : sqlJsonData.entrySet()) { + Map.Entry entry = (Map.Entry) e; + String tableName = String.valueOf(entry.getKey()); + if (!tableName.startsWith("edge__")) { + nodeCtHash.put(String.valueOf(entry.getKey()), Integer.parseInt(entry.getValue().toString())); + } + } + } + return nodeCtHash; + } + + + public HashMap <String,Integer> getEdgeCountsFromSQLRes(JsonObject sqlJsonData){ + + HashMap<String,Integer> edgeCtHash = new HashMap<>(); + if( sqlJsonData != null ) { + for (Object e : sqlJsonData.entrySet()) { + Map.Entry entry = (Map.Entry) e; + String tableName = String.valueOf(entry.getKey()); + if (tableName.startsWith("edge__")) { + edgeCtHash.put(String.valueOf(entry.getKey()), Integer.parseInt(entry.getValue().toString())); + + } + } + } + return edgeCtHash; + } + + + public JsonObject getRdbmsTableCounts(long timestamp, String dbName) + throws Exception { + + return apertureService.runAudit(timestamp, dbName); + } + + + public String getGsonTotals(HashMap <String,Integer> tableCounts){ + int nodeCount = 0; + int edgeCount = 0; + for (Map.Entry<String, Integer> entry : tableCounts.entrySet()) { + String tblKey = entry.getKey(); + if( tblKey != null && tblKey.startsWith("edge__")){ + edgeCount = edgeCount + entry.getValue(); + } else { + nodeCount = nodeCount + entry.getValue(); + } + } + String countStr = " nodes = " + nodeCount + ", edges = " + edgeCount; + return countStr; + } + + public String getMSqlTotals(HashMap <String,Integer> nodeCounts, + HashMap <String,Integer> edgeCounts){ + int nodeCount = 0; + int edgeCount = 0; + for (Map.Entry<String, Integer> entry : nodeCounts.entrySet()) { + nodeCount = nodeCount + entry.getValue(); + } + for (Map.Entry<String, Integer> entry : edgeCounts.entrySet()) { + edgeCount = edgeCount + entry.getValue(); + } + String countStr = " nodes = " + nodeCount + ", edges = " + edgeCount; + return countStr; + } + + public Long getTimestampFromFileName(String fName) throws Exception { + // Note -- we are expecting filenames that look like our + // current snapshot filenames (without the ".PXX" suffix) + // Ie. dataSnapshot.graphSON.201908151845 + + String datePiece = getDateTimePieceOfFileName(fName); + final SimpleDateFormat dateFormat = new SimpleDateFormat("YYYYMMddHHmm"); + Date date = null; + try { + date = dateFormat.parse(datePiece); + } + catch (ParseException pe) { + throw new Exception ("Error trying to parse this to a Date-Timestamp [" + + datePiece + "]: " + pe.getMessage() ); + } + final long timestamp = date.getTime(); + return timestamp; + + } + + + public String getDateTimePieceOfFileName(String fName) throws Exception { + // Note -- we are expecting filenames that look like our + // current snapshot filenames (without the ".PXX" suffix) + // Ie. dataSnapshot.graphSON.201908151845 + + if( fName == null || fName.isEmpty() || fName.length() < 12) { + throw new Exception ("File name must end with .yyyymmddhhmm "); + } + int index = fName.lastIndexOf('.'); + + if ( index == -1 ) { + throw new Exception ("File name must end with .yyyymmddhhmm "); + } + index++; + if(fName.length() <= index) { + throw new Exception ("File name must end with .yyyymmddhhmm "); + } + String datePiece = fName.substring(index); + if( datePiece.length() != 12 ) { + throw new Exception ("File name must end date in the format .yyyymmddhhmm "); + } + + return datePiece; + } + + + public Map<String, String> getEdgeMapKeys() throws EdgeRuleNotFoundException { + + int edKeyCount = 0; + Map<String, String> edgeKeys = new HashMap<>(); + // EdgeKey will look like: nodeTypeA__nodeTypeB + // The value will be the key with "edge__" pre-pended + + for (Map.Entry<String, Collection<EdgeRule>> rulePairings : ei.getAllCurrentRules().asMap().entrySet()) { + for (EdgeRule er : rulePairings.getValue()) { + String a = er.getTo(); + String b = er.getFrom(); + if (a.compareTo(b) >= 0) { + er.flipDirection(); + } + if (!loader.getAllObjects().containsKey(er.getTo()) || !loader.getAllObjects().containsKey(er.getFrom())) { + // we don't have one of these nodeTypes, so skip this entry + continue; + } + + final String to = er.getTo().replace("-", "_"); + final String from = er.getFrom().replace("-", "_"); + final String edKey = from + "__" + to; + + if (edgeKeys.containsKey(edKey)) { + continue; + } + edKeyCount++; + edgeKeys.put(edKey, "edge__"+ edKey); + + } + } + System.out.println("DEBUG --> There are " + edKeyCount + " edge table keys defined. " ); + LOGGER.debug(" -- There are " + edKeyCount + " edge table keys defined. " ); + return edgeKeys; + } + + + public HashMap <String,Integer> getCountsFromGraphsonSnapshot(String databaseName, + String fileNamePart, String srcDir) throws Exception { + + final JsonParser jsonParser = new JsonParser(); + final Set<String> uris = new HashSet<>(); + final Set<String> uuids = new HashSet<>(); + final Map<Long, String> idToUri = new HashMap<>(); + final Map<Long, String> idToType = new HashMap<>(); + final Map<Long, String> idToUuid = new HashMap<>(); + final Set<Triplet<Long, Long, String>> idIdLabelOfEdges = new HashSet<>(); + + final HashMap<String,Integer> tableCountHash = new HashMap<>(); + + + String sourceDir = DEFAULT_SRC_DIR; // Default for now + if( srcDir != null && srcDir.length() > 1 ) { + // If they passed one in, then we'll use it. + sourceDir = srcDir; + } + String graphsonDir = sourceDir + "/"; + + if( fileNamePart == null || fileNamePart.trim().isEmpty() ){ + String msg = "ERROR -- fileName is required to be passed in. "; + LOGGER.error(msg); + System.out.println(msg); + throw new Exception(msg); + } + + final List<File> graphsons = Files.walk(Paths.get(graphsonDir)) + .filter(Files::isRegularFile) + .map(Path::toFile) + .sorted() + .collect(Collectors.toList()); + + int skippedNodeCount = 0; + int nodeCounter = 0; + int edgeCounter = 0; + for (File graphson : graphsons) { + if( !graphson.getName().contains(fileNamePart) ) { + continue; + } + + try (BufferedReader reader = new BufferedReader(new FileReader(graphson))) { + String msg = "Processing snapshot file " + graphson.getName(); + LOGGER.debug(msg); + System.out.println(msg); + String line; + + while ((line = reader.readLine()) != null) { + JsonObject vertex = jsonParser.parse(line).getAsJsonObject(); + long id = vertex.get("id").getAsLong(); + + if ((vertex.get("properties") == null) || + !vertex.get("properties").getAsJsonObject().has("aai-uri") || + !vertex.get("properties").getAsJsonObject().has("aai-node-type") || + !vertex.get("properties").getAsJsonObject().has("aai-uuid")) { + + msg = "DEBUG -- Could not find keys for this line: [" + + line + "] ------"; + LOGGER.debug(msg); + System.out.println(msg); + skippedNodeCount++; + continue; + } + + String uri = vertex.get("properties").getAsJsonObject().get("aai-uri").getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); + String nodeType = vertex.get("properties").getAsJsonObject().get("aai-node-type").getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); + String nodeTypeKey = nodeType.replaceAll("-", "_"); + String uuid = vertex.get("properties").getAsJsonObject().get("aai-uuid").getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); + + try { + loader.introspectorFromName(nodeType); + } catch (Exception e) { + msg = "DEBUG -- loader introspector for nodeType error: [" + + e.getMessage() + "], [" + e.getLocalizedMessage() + "]------"; + LOGGER.debug(msg); + System.out.println(msg); + skippedNodeCount++; + continue; + } + + if (uris.contains(uri)) { + msg = "DEBUG -- SKIP Uri because it has been seen before: [" + + uri + "] ------"; + LOGGER.debug(msg); + System.out.println(msg); + skippedNodeCount++; + continue; + } + else if (uuids.contains(uuid)) { + msg = "DEBUG -- SKIP UUID because it has been seen before: [" + + uuid + "] ------"; + LOGGER.debug(msg); + System.out.println(msg); + skippedNodeCount++; + continue; + } + + uris.add(uri); + uuids.add(uuid); + idToUri.put(id, uri); + idToType.put(id, nodeType); + idToUuid.put(id, uuid); + + // Collect Edge Info for this node + if (vertex.has("inE")) { + vertex.get("inE").getAsJsonObject().entrySet().forEach(es -> { + String label = es.getKey(); + es.getValue().getAsJsonArray().forEach(e -> { + long otherId = e.getAsJsonObject().get("outV").getAsLong(); + idIdLabelOfEdges.add(new Triplet<>(id, otherId, label)); + }); + }); + } + + if( !tableCountHash.containsKey(nodeTypeKey)) { + int ct = 1; + tableCountHash.put(nodeTypeKey, ct); + } + else { + int tmpCt = tableCountHash.get(nodeTypeKey); + tmpCt++; + tableCountHash.remove(nodeTypeKey); + tableCountHash.put(nodeTypeKey, tmpCt); + } + nodeCounter++; + }// end of looping over this file + } catch (IOException e) { + String msg = "DEBUG -- Error while processing nodes ------"; + LOGGER.debug(msg); + System.out.println(msg); + e.printStackTrace(); + } + }// End of looping over all files + + String msg = "DEBUG -- Found this many Kinds of nodes: " + tableCountHash.size(); + LOGGER.debug(msg); + System.out.println(msg); + + msg = "DEBUG -- Found this many total nodes: " + nodeCounter; + LOGGER.debug(msg); + System.out.println(msg); + + msg = " >> Skipped a total of " + skippedNodeCount + " Node Records ------"; + LOGGER.debug(msg); + System.out.println(msg); + + + msg = "DEBUG -- Begin Processing Edges ------"; + LOGGER.debug(msg); + System.out.println(msg); + + int edgeTableCounter = 0; + int edgeSkipCounter = 0; + + Map<String, String> edgeKeys = this.getEdgeMapKeys(); + for (Triplet<Long, Long, String> edge : idIdLabelOfEdges) { + if (!idToType.containsKey(edge.getValue0())) { + LOGGER.info(" Edge Skipped because ID not found: [" + edge.getValue0() + "]"); + System.out.println(" Edge Skipped because ID not found: [" + edge.getValue0() + "]"); + edgeSkipCounter++; + continue; + } + else if (!idToType.containsKey(edge.getValue1())) { + System.out.println(" Edge Skipped because ID not found: [" + edge.getValue1() + "]"); + LOGGER.info(" Edge Skipped because ID not found: [" + edge.getValue1() + "]"); + edgeSkipCounter++; + continue; + } + else { + String colA = idToType.get(edge.getValue1()).replace("-","_"); + String colB = idToType.get(edge.getValue0()).replace("-","_"); + + String edLabel = edge.getValue2(); // if we start using edLabel to sort + String edKey = colA + "__" + colB; + // Note the edgeKeys table has nodeTypeX__nodeTypeY as the key + // The value stored for that key just has "edge__" pre-pended which + // is the key used by the tableCount thing + String tcEdKey = ""; + if (!edgeKeys.containsKey(edKey)) { + tcEdKey = edgeKeys.get(colB + "__" + colA ); + } else { + tcEdKey = edgeKeys.get(edKey); + } + + if( !tableCountHash.containsKey(tcEdKey)) { + int ct = 1; + tableCountHash.put(tcEdKey, ct); + edgeTableCounter++; + } + else { + int tmpCt = tableCountHash.get(tcEdKey); + tmpCt++; + tableCountHash.remove(tcEdKey); + tableCountHash.put(tcEdKey, tmpCt); + } + edgeCounter++; + } + } + + msg = " Processed a total of " + edgeCounter + " Edge Records ------"; + LOGGER.debug(msg); + System.out.println(msg); + + msg = " Found data for this many edgeTables: " + edgeTableCounter; + LOGGER.debug(msg); + System.out.println(msg); + + msg = " >> Skipped a total of " + edgeSkipCounter + " Edge Records ------"; + LOGGER.debug(msg); + System.out.println(msg); + + return tableCountHash; + + } + + + +} |