diff options
Diffstat (limited to 'src/main')
50 files changed, 2595 insertions, 510 deletions
diff --git a/src/main/assembly/descriptor.xml b/src/main/assembly/descriptor.xml index 91e8e18..af77dee 100644 --- a/src/main/assembly/descriptor.xml +++ b/src/main/assembly/descriptor.xml @@ -1,32 +1,35 @@ -<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>build</id>
- <includeBaseDirectory>false</includeBaseDirectory>
- <formats>
- <format>dir</format>
- </formats>
- <fileSets>
- <fileSet>
- <directory>${project.basedir}/src/main/resources</directory>
- <outputDirectory>/resources</outputDirectory>
- <includes>
- <include>**/*</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/src/main/scripts</directory>
- <outputDirectory>/bin</outputDirectory>
- <includes>
- <include>**/*</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${project.build.directory}</directory>
- <outputDirectory>/lib</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- </fileSets>
-</assembly>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd"> + <id>build</id> + <includeBaseDirectory>false</includeBaseDirectory> + <formats> + <format>dir</format> + </formats> + <fileSets> + <fileSet> + <directory>${project.basedir}/src/main/resources</directory> + <outputDirectory>/resources</outputDirectory> + <includes> + <include>**/*</include> + </includes> + <fileMode>755</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/src/main/scripts</directory> + <outputDirectory>/bin</outputDirectory> + <includes> + <include>**/*</include> + </includes> + <fileMode>777</fileMode> + </fileSet> + <fileSet> + <directory>${project.build.directory}</directory> + <outputDirectory>/lib</outputDirectory> + <includes> + <include>${project.artifactId}-${project.version}.jar</include> + </includes> + <fileMode>755</fileMode> + </fileSet> + </fileSets> +</assembly> diff --git a/src/main/java/org/onap/aai/GraphAdminApp.java b/src/main/java/org/onap/aai/GraphAdminApp.java index 3f7abf3..1339604 100644 --- a/src/main/java/org/onap/aai/GraphAdminApp.java +++ b/src/main/java/org/onap/aai/GraphAdminApp.java @@ -19,16 +19,21 @@ */ package org.onap.aai; +import com.att.eelf.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.onap.aai.aailog.logs.AaiDebugLog; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbmap.AAIGraph; +import java.util.Properties; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.nodes.NodeIngestor; import org.onap.aai.util.AAIConfig; +import org.onap.aai.util.AAIConstants; import org.onap.aai.util.ExceptionTranslator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -36,6 +41,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.core.env.Environment; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; @@ -61,10 +67,13 @@ import java.util.UUID; "org.onap.aai.datagrooming", "org.onap.aai.dataexport", "org.onap.aai.datacleanup", - "org.onap.aai.aailog" + "org.onap.aai.aailog", + "org.onap.aai.failover", + "org.onap.aai.audit" }) @EnableAsync @EnableScheduling +@EnableAspectJAutoProxy @EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, HibernateJpaAutoConfiguration.class}) public class GraphAdminApp { diff --git a/src/main/java/org/onap/aai/audit/AuditGraphson2Sql.java b/src/main/java/org/onap/aai/audit/AuditGraphson2Sql.java new file mode 100644 index 0000000..ff29157 --- /dev/null +++ b/src/main/java/org/onap/aai/audit/AuditGraphson2Sql.java @@ -0,0 +1,605 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.audit; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.javatuples.Triplet; +import org.onap.aai.restclient.PropertyPasswordConfiguration; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.EdgeRule; +import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; +import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.rest.client.ApertureService; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; +import org.onap.aai.util.AAIConstants; +import org.onap.aai.util.ExceptionTranslator; +import org.onap.aai.util.FormatDate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.stream.Collectors; + +import com.att.eelf.configuration.Configuration; +import org.springframework.stereotype.Component; + +@Component +public class AuditGraphson2Sql { + + private EdgeIngestor ei; + private Loader loader; + private static final Logger LOGGER = LoggerFactory.getLogger(AuditGraphson2Sql.class.getSimpleName()); + private SchemaVersions schemaVersions; + private ApertureService apertureService; + + public static final String DEFAULT_SRC_DIR = "logs/data/dataSnapshots/"; + public static final int DEFAULT_THRESHOLD_PERCENT = 10; + public static final String DEFAULT_OUTPUT_DIR = "logs/data/audit"; + + //DEBUG -- should be getting default-src-dir, default-output-dir and rdbms-db-name from param file + @Autowired + public AuditGraphson2Sql( EdgeIngestor ei, SchemaVersions schemaVersions, LoaderFactory loaderFactory, ApertureService apertureService) { + this.schemaVersions = schemaVersions; + this.loader = loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion()); + this.ei = ei; + this.apertureService = apertureService; + } + + public String runAudit(String dbName, String sourceFName, String sourceDir ) + throws Exception { + + if( sourceDir == null || sourceDir.isEmpty() ){ + sourceDir = DEFAULT_SRC_DIR; + } + + HashMap <String,Integer> gsonTableCounts = new HashMap <> (); + try { + gsonTableCounts = getCountsFromGraphsonSnapshot(dbName, sourceFName, sourceDir); + } catch( Exception e ) { + LOGGER.error(" ERROR when calling getCountsFromGraphsonSnapshot(" + + dbName + "," + + sourceFName + "). Exception = " + + e.getMessage() ); + throw e; + } + + long timestamp = 0L; + try { + timestamp = getTimestampFromFileName(sourceFName); + } catch( Exception e ) { + LOGGER.error(" ERROR getting timestamp from filename: " + + e.getMessage() ); + throw e; + } + + JsonObject rdbmsJsonResults = new JsonObject (); + try { + rdbmsJsonResults = getRdbmsTableCounts(timestamp, dbName); + } catch( Exception e ) { + LOGGER.error(" ERROR getting table count info from mySql. timestamp = [" + + timestamp + "], dbName = [" + dbName + + "]. Exception = " + e.getMessage() ); + throw e; + } + HashMap <String,Integer> sqlNodeCounts = getNodeCountsFromSQLRes(rdbmsJsonResults); + HashMap <String,Integer> sqlEdgeCounts = getEdgeCountsFromSQLRes(rdbmsJsonResults); + + String resultString = "Audit ran and logged to file: "; + try { + String titleInfo = "Comparing data from GraphSon file: " + + sourceFName + ", and Aperture data using timeStamp = [" + + timestamp + "] "; + String outFileName = compareResults(gsonTableCounts, sqlNodeCounts, + sqlEdgeCounts, DEFAULT_OUTPUT_DIR, titleInfo); + resultString = resultString + outFileName; + } + catch( IOException ie) { + LOGGER.error(" ERROR writing to output file. [" + ie.getMessage() + "]" ); + throw new Exception( ie.getMessage() ); + } + + return resultString; + + } + + + public String compareResults(HashMap <String,Integer> gsonTableCounts, + HashMap <String,Integer> sqlNodeCounts, + HashMap <String,Integer> sqlEdgeCounts, + String outputDir, String titleInfo) + throws IOException { + + FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); + String dteStr = fd.getDateTime(); + String fName = outputDir + "/" + "gson-sql-audit" + dteStr; + File auditOutFile = new File(fName); + auditOutFile.getParentFile().mkdirs(); + BufferedWriter outWriter = new BufferedWriter(new FileWriter(auditOutFile)); + outWriter.write(titleInfo); + outWriter.newLine(); + outWriter.write("Totals from Graphson: " + getGsonTotals(gsonTableCounts)); + outWriter.newLine(); + outWriter.write("Totals from mySql: " + getMSqlTotals(sqlNodeCounts,sqlEdgeCounts)); + outWriter.newLine(); + outWriter.newLine(); + + for (Map.Entry<String, Integer> entry : gsonTableCounts.entrySet()) { + String tblKey = entry.getKey(); + int gVal = entry.getValue(); + if(!sqlNodeCounts.containsKey(tblKey) && !sqlEdgeCounts.containsKey(tblKey)) { + String msg = "> Entry for Table: [" + tblKey + + "] not found in the SQL db but had " + + gVal + " entries in the GraphSon db"; + LOGGER.error(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + continue; + } + int sVal = 0; + if( sqlNodeCounts.containsKey(tblKey) ) { + sVal = sqlNodeCounts.get(tblKey); + }else { + sVal = sqlEdgeCounts.get(tblKey); + } + if ((gVal > 0) && (sVal == 0)){ + String msg = "For Table [" + tblKey + "], GSon count = " + + gVal + ", zero found in SQL db."; + LOGGER.error(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + } + String msg = tblKey + ": gson/sql = " + gVal + "/" + sVal; + LOGGER.debug(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + } + for (Map.Entry<String, Integer> entry : sqlNodeCounts.entrySet()) { + // check for Node types that are no longer on the Graphson side, but are + // still in the SQL DB + String tblKey = entry.getKey(); + int sVal = entry.getValue(); + if(!gsonTableCounts.containsKey(tblKey)) { + String msg = " Entry for Table [" + tblKey + + "] not found in the Graphson db, but had " + + sVal + " entries in the SQL db."; + LOGGER.error(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + continue; + } + } + for (Map.Entry<String, Integer> entry : sqlEdgeCounts.entrySet()) { + // check for Edge+Label combos that are no longer on the Graphson side, but are + // still in the SQL DB + String tblKey = entry.getKey(); + int sVal = entry.getValue(); + if(!gsonTableCounts.containsKey(tblKey)) { + String msg = " Entry for edge+Label combo [" + tblKey + + "] not found in the Graphson db, but had " + + sVal + " entries in the SQL db."; + LOGGER.error(msg); + System.out.println(msg); + outWriter.write(msg); + outWriter.newLine(); + continue; + } + } + outWriter.close(); + String msg = "Audit Results written to: " + fName; + LOGGER.debug(msg); + System.out.println(msg); + return fName; + + } + + + public HashMap <String,Integer> getNodeCountsFromSQLRes(JsonObject sqlJsonData) { + + HashMap<String,Integer> nodeCtHash = new HashMap<>(); + if( sqlJsonData != null ) { + for (Object e : sqlJsonData.entrySet()) { + Map.Entry entry = (Map.Entry) e; + String tableName = String.valueOf(entry.getKey()); + if (!tableName.startsWith("edge__")) { + nodeCtHash.put(String.valueOf(entry.getKey()), Integer.parseInt(entry.getValue().toString())); + } + } + } + return nodeCtHash; + } + + + public HashMap <String,Integer> getEdgeCountsFromSQLRes(JsonObject sqlJsonData){ + + HashMap<String,Integer> edgeCtHash = new HashMap<>(); + if( sqlJsonData != null ) { + for (Object e : sqlJsonData.entrySet()) { + Map.Entry entry = (Map.Entry) e; + String tableName = String.valueOf(entry.getKey()); + if (tableName.startsWith("edge__")) { + edgeCtHash.put(String.valueOf(entry.getKey()), Integer.parseInt(entry.getValue().toString())); + + } + } + } + return edgeCtHash; + } + + + public JsonObject getRdbmsTableCounts(long timestamp, String dbName) + throws Exception { + + return apertureService.runAudit(timestamp, dbName); + } + + + public String getGsonTotals(HashMap <String,Integer> tableCounts){ + int nodeCount = 0; + int edgeCount = 0; + for (Map.Entry<String, Integer> entry : tableCounts.entrySet()) { + String tblKey = entry.getKey(); + if( tblKey != null && tblKey.startsWith("edge__")){ + edgeCount = edgeCount + entry.getValue(); + } else { + nodeCount = nodeCount + entry.getValue(); + } + } + String countStr = " nodes = " + nodeCount + ", edges = " + edgeCount; + return countStr; + } + + public String getMSqlTotals(HashMap <String,Integer> nodeCounts, + HashMap <String,Integer> edgeCounts){ + int nodeCount = 0; + int edgeCount = 0; + for (Map.Entry<String, Integer> entry : nodeCounts.entrySet()) { + nodeCount = nodeCount + entry.getValue(); + } + for (Map.Entry<String, Integer> entry : edgeCounts.entrySet()) { + edgeCount = edgeCount + entry.getValue(); + } + String countStr = " nodes = " + nodeCount + ", edges = " + edgeCount; + return countStr; + } + + public Long getTimestampFromFileName(String fName) throws Exception { + // Note -- we are expecting filenames that look like our + // current snapshot filenames (without the ".PXX" suffix) + // Ie. dataSnapshot.graphSON.201908151845 + + String datePiece = getDateTimePieceOfFileName(fName); + final SimpleDateFormat dateFormat = new SimpleDateFormat("YYYYMMddHHmm"); + Date date = null; + try { + date = dateFormat.parse(datePiece); + } + catch (ParseException pe) { + throw new Exception ("Error trying to parse this to a Date-Timestamp [" + + datePiece + "]: " + pe.getMessage() ); + } + final long timestamp = date.getTime(); + return timestamp; + + } + + + public String getDateTimePieceOfFileName(String fName) throws Exception { + // Note -- we are expecting filenames that look like our + // current snapshot filenames (without the ".PXX" suffix) + // Ie. dataSnapshot.graphSON.201908151845 + + if( fName == null || fName.isEmpty() || fName.length() < 12) { + throw new Exception ("File name must end with .yyyymmddhhmm "); + } + int index = fName.lastIndexOf('.'); + + if ( index == -1 ) { + throw new Exception ("File name must end with .yyyymmddhhmm "); + } + index++; + if(fName.length() <= index) { + throw new Exception ("File name must end with .yyyymmddhhmm "); + } + String datePiece = fName.substring(index); + if( datePiece.length() != 12 ) { + throw new Exception ("File name must end date in the format .yyyymmddhhmm "); + } + + return datePiece; + } + + + public Map<String, String> getEdgeMapKeys() throws EdgeRuleNotFoundException { + + int edKeyCount = 0; + Map<String, String> edgeKeys = new HashMap<>(); + // EdgeKey will look like: nodeTypeA__nodeTypeB + // The value will be the key with "edge__" pre-pended + + for (Map.Entry<String, Collection<EdgeRule>> rulePairings : ei.getAllCurrentRules().asMap().entrySet()) { + for (EdgeRule er : rulePairings.getValue()) { + String a = er.getTo(); + String b = er.getFrom(); + if (a.compareTo(b) >= 0) { + er.flipDirection(); + } + if (!loader.getAllObjects().containsKey(er.getTo()) || !loader.getAllObjects().containsKey(er.getFrom())) { + // we don't have one of these nodeTypes, so skip this entry + continue; + } + + final String to = er.getTo().replace("-", "_"); + final String from = er.getFrom().replace("-", "_"); + final String edKey = from + "__" + to; + + if (edgeKeys.containsKey(edKey)) { + continue; + } + edKeyCount++; + edgeKeys.put(edKey, "edge__"+ edKey); + + } + } + System.out.println("DEBUG --> There are " + edKeyCount + " edge table keys defined. " ); + LOGGER.debug(" -- There are " + edKeyCount + " edge table keys defined. " ); + return edgeKeys; + } + + + public HashMap <String,Integer> getCountsFromGraphsonSnapshot(String databaseName, + String fileNamePart, String srcDir) throws Exception { + + final JsonParser jsonParser = new JsonParser(); + final Set<String> uris = new HashSet<>(); + final Set<String> uuids = new HashSet<>(); + final Map<Long, String> idToUri = new HashMap<>(); + final Map<Long, String> idToType = new HashMap<>(); + final Map<Long, String> idToUuid = new HashMap<>(); + final Set<Triplet<Long, Long, String>> idIdLabelOfEdges = new HashSet<>(); + + final HashMap<String,Integer> tableCountHash = new HashMap<>(); + + + String sourceDir = DEFAULT_SRC_DIR; // Default for now + if( srcDir != null && srcDir.length() > 1 ) { + // If they passed one in, then we'll use it. + sourceDir = srcDir; + } + String graphsonDir = sourceDir + "/"; + + if( fileNamePart == null || fileNamePart.trim().isEmpty() ){ + String msg = "ERROR -- fileName is required to be passed in. "; + LOGGER.error(msg); + System.out.println(msg); + throw new Exception(msg); + } + + final List<File> graphsons = Files.walk(Paths.get(graphsonDir)) + .filter(Files::isRegularFile) + .map(Path::toFile) + .sorted() + .collect(Collectors.toList()); + + int skippedNodeCount = 0; + int nodeCounter = 0; + int edgeCounter = 0; + for (File graphson : graphsons) { + if( !graphson.getName().contains(fileNamePart) ) { + continue; + } + + try (BufferedReader reader = new BufferedReader(new FileReader(graphson))) { + String msg = "Processing snapshot file " + graphson.getName(); + LOGGER.debug(msg); + System.out.println(msg); + String line; + + while ((line = reader.readLine()) != null) { + JsonObject vertex = jsonParser.parse(line).getAsJsonObject(); + long id = vertex.get("id").getAsLong(); + + if ((vertex.get("properties") == null) || + !vertex.get("properties").getAsJsonObject().has("aai-uri") || + !vertex.get("properties").getAsJsonObject().has("aai-node-type") || + !vertex.get("properties").getAsJsonObject().has("aai-uuid")) { + + msg = "DEBUG -- Could not find keys for this line: [" + + line + "] ------"; + LOGGER.debug(msg); + System.out.println(msg); + skippedNodeCount++; + continue; + } + + String uri = vertex.get("properties").getAsJsonObject().get("aai-uri").getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); + String nodeType = vertex.get("properties").getAsJsonObject().get("aai-node-type").getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); + String nodeTypeKey = nodeType.replaceAll("-", "_"); + String uuid = vertex.get("properties").getAsJsonObject().get("aai-uuid").getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); + + try { + loader.introspectorFromName(nodeType); + } catch (Exception e) { + msg = "DEBUG -- loader introspector for nodeType error: [" + + e.getMessage() + "], [" + e.getLocalizedMessage() + "]------"; + LOGGER.debug(msg); + System.out.println(msg); + skippedNodeCount++; + continue; + } + + if (uris.contains(uri)) { + msg = "DEBUG -- SKIP Uri because it has been seen before: [" + + uri + "] ------"; + LOGGER.debug(msg); + System.out.println(msg); + skippedNodeCount++; + continue; + } + else if (uuids.contains(uuid)) { + msg = "DEBUG -- SKIP UUID because it has been seen before: [" + + uuid + "] ------"; + LOGGER.debug(msg); + System.out.println(msg); + skippedNodeCount++; + continue; + } + + uris.add(uri); + uuids.add(uuid); + idToUri.put(id, uri); + idToType.put(id, nodeType); + idToUuid.put(id, uuid); + + // Collect Edge Info for this node + if (vertex.has("inE")) { + vertex.get("inE").getAsJsonObject().entrySet().forEach(es -> { + String label = es.getKey(); + es.getValue().getAsJsonArray().forEach(e -> { + long otherId = e.getAsJsonObject().get("outV").getAsLong(); + idIdLabelOfEdges.add(new Triplet<>(id, otherId, label)); + }); + }); + } + + if( !tableCountHash.containsKey(nodeTypeKey)) { + int ct = 1; + tableCountHash.put(nodeTypeKey, ct); + } + else { + int tmpCt = tableCountHash.get(nodeTypeKey); + tmpCt++; + tableCountHash.remove(nodeTypeKey); + tableCountHash.put(nodeTypeKey, tmpCt); + } + nodeCounter++; + }// end of looping over this file + } catch (IOException e) { + String msg = "DEBUG -- Error while processing nodes ------"; + LOGGER.debug(msg); + System.out.println(msg); + e.printStackTrace(); + } + }// End of looping over all files + + String msg = "DEBUG -- Found this many Kinds of nodes: " + tableCountHash.size(); + LOGGER.debug(msg); + System.out.println(msg); + + msg = "DEBUG -- Found this many total nodes: " + nodeCounter; + LOGGER.debug(msg); + System.out.println(msg); + + msg = " >> Skipped a total of " + skippedNodeCount + " Node Records ------"; + LOGGER.debug(msg); + System.out.println(msg); + + + msg = "DEBUG -- Begin Processing Edges ------"; + LOGGER.debug(msg); + System.out.println(msg); + + int edgeTableCounter = 0; + int edgeSkipCounter = 0; + + Map<String, String> edgeKeys = this.getEdgeMapKeys(); + for (Triplet<Long, Long, String> edge : idIdLabelOfEdges) { + if (!idToType.containsKey(edge.getValue0())) { + LOGGER.info(" Edge Skipped because ID not found: [" + edge.getValue0() + "]"); + System.out.println(" Edge Skipped because ID not found: [" + edge.getValue0() + "]"); + edgeSkipCounter++; + continue; + } + else if (!idToType.containsKey(edge.getValue1())) { + System.out.println(" Edge Skipped because ID not found: [" + edge.getValue1() + "]"); + LOGGER.info(" Edge Skipped because ID not found: [" + edge.getValue1() + "]"); + edgeSkipCounter++; + continue; + } + else { + String colA = idToType.get(edge.getValue1()).replace("-","_"); + String colB = idToType.get(edge.getValue0()).replace("-","_"); + + String edLabel = edge.getValue2(); // if we start using edLabel to sort + String edKey = colA + "__" + colB; + // Note the edgeKeys table has nodeTypeX__nodeTypeY as the key + // The value stored for that key just has "edge__" pre-pended which + // is the key used by the tableCount thing + String tcEdKey = ""; + if (!edgeKeys.containsKey(edKey)) { + tcEdKey = edgeKeys.get(colB + "__" + colA ); + } else { + tcEdKey = edgeKeys.get(edKey); + } + + if( !tableCountHash.containsKey(tcEdKey)) { + int ct = 1; + tableCountHash.put(tcEdKey, ct); + edgeTableCounter++; + } + else { + int tmpCt = tableCountHash.get(tcEdKey); + tmpCt++; + tableCountHash.remove(tcEdKey); + tableCountHash.put(tcEdKey, tmpCt); + } + edgeCounter++; + } + } + + msg = " Processed a total of " + edgeCounter + " Edge Records ------"; + LOGGER.debug(msg); + System.out.println(msg); + + msg = " Found data for this many edgeTables: " + edgeTableCounter; + LOGGER.debug(msg); + System.out.println(msg); + + msg = " >> Skipped a total of " + edgeSkipCounter + " Edge Records ------"; + LOGGER.debug(msg); + System.out.println(msg); + + return tableCountHash; + + } + + + +} diff --git a/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java index 8fc6295..99b1619 100644 --- a/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java +++ b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java @@ -18,14 +18,25 @@ * ============LICENSE_END========================================================= */ package org.onap.aai.datacleanup; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.nio.file.Files; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog; import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.logging.LogFormatTools; +import org.onap.aai.exceptions.AAIException; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; import org.onap.logging.filter.base.ONAPComponents; -import org.onap.logging.ref.slf4j.ONAPLogConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -33,18 +44,6 @@ import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - @Component @PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties") public class DataCleanupTasks { @@ -73,6 +72,7 @@ public class DataCleanupTasks { String archiveDir = dataGroomingDir + AAIConstants.AAI_FILESEP + "ARCHIVE"; String dataGroomingArcDir = archiveDir + AAIConstants.AAI_FILESEP + "dataGrooming"; File path = new File(dataGroomingDir); + File archivepath = new File(archiveDir); File dataGroomingPath = new File(dataGroomingArcDir); logger.debug("The logDir is " + logDir); @@ -138,7 +138,9 @@ public class DataCleanupTasks { * */ public boolean directoryExists(String dir) { - return new File(dir).exists(); + File path = new File(dir); + boolean exists = path.exists(); + return exists; } public Date getZipDate(Integer days) { @@ -177,7 +179,7 @@ public class DataCleanupTasks { logger.debug("Inside the archive folder"); String filename = file.getName(); logger.debug("file name is " +filename); - + String zipFile = afterArchiveDir + AAIConstants.AAI_FILESEP + filename; File dataGroomingPath = new File(afterArchiveDir); @@ -230,8 +232,10 @@ public class DataCleanupTasks { */ @Scheduled(cron = "${datasnapshotcleanup.cron}" ) public void dataSnapshotCleanup() { - - logger.info(ONAPLogConstants.Markers.ENTRY, "Started cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date())); + + auditLog.logBefore("dataSnapshotCleanup", ONAPComponents.AAI.toString() ); + + logger.debug("Started cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date())); try { String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs"; @@ -289,11 +293,88 @@ public class DataCleanupTasks { } } } + dmaapEventsDataCleanup(newAgeDelete); + dataMigrationCleanup(); } catch (Exception e) { ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e)); logger.debug("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e)); } - logger.info(ONAPLogConstants.Markers.EXIT, "Ended cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date())); - } + logger.debug("Ended cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date())); + auditLog.logAfter(); + } + public void dmaapEventsDataCleanup(Date deleteAge) { + + logger.debug("Started dmaapEventsDataCleanup @ " + simpleDateFormat.format(new Date())); + + try { + String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs"; + String dmaapEventsDataDir = logDir + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dmaapEvents"; + File path = new File(dmaapEventsDataDir); + + logger.debug("The logDir is " + logDir); + logger.debug("The dmaapEventsDataDir is " + dmaapEventsDataDir); + + //Iterate through the files + File[] listFiles = path.listFiles(); + if(listFiles != null) { + for(File listFile : listFiles) { + if(listFile.isFile()){ + logger.debug("The file name in dmaapEvents is: " +listFile.getName()); + Date fileCreateDate = fileCreationMonthDate(listFile); + logger.debug("The fileCreateDate in dmaapEvents is " + fileCreateDate); + if( fileCreateDate.compareTo(deleteAge) < 0) { + delete(listFile); + logger.debug("Deleted " + listFile.getName()); + } + } + } + } + + } + catch (Exception e) { + ErrorLogHelper.logError("AAI_4000", "Exception in dmaapEventsDataCleanup"); + logger.debug("AAI_4000", "Exception in dmaapEventsDataCleanup "+LogFormatTools.getStackTop(e)); + } + logger.debug("Ended cron dmaapEventsDataCleanup @ " + simpleDateFormat.format(new Date())); + } + + public void dataMigrationCleanup() throws AAIException { + Integer ageDeleteSnapshot = AAIConfig.getInt("aai.datamigration.agedelete"); + + Date deleteAge = getZipDate(ageDeleteSnapshot); + + logger.debug("Started dataMigrationCleanup @ " + simpleDateFormat.format(new Date())); + + try { + String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs"; + String dataMigrationCleanupDir = logDir + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migration-input-files"; + File path = new File(dataMigrationCleanupDir); + + logger.debug("The logDir is " + logDir); + logger.debug("The migrationInputFilesDir is " + dataMigrationCleanupDir); + + //Iterate through the files + File[] listFiles = path.listFiles(); + if(listFiles != null) { + for(File listFile : listFiles) { + if(listFile.isFile()){ + logger.debug("The file name in migration-input-files is: " +listFile.getName()); + Date fileCreateDate = fileCreationMonthDate(listFile); + logger.debug("The fileCreateDate in migration-input-files is " + fileCreateDate); + if( fileCreateDate.compareTo(deleteAge) < 0) { + delete(listFile); + logger.debug("Deleted " + listFile.getName()); + } + } + } + } + + } + catch (Exception e) { + ErrorLogHelper.logError("AAI_4000", "Exception in dataMigrationCleanup"); + logger.debug("AAI_4000", "Exception in dataMigrationCleanup "+LogFormatTools.getStackTop(e)); + } + logger.debug("Ended cron dataMigrationCleanup @ " + simpleDateFormat.format(new Date())); + } } diff --git a/src/main/java/org/onap/aai/dataexport/DataExportTasks.java b/src/main/java/org/onap/aai/dataexport/DataExportTasks.java index 2d0625c..028c729 100644 --- a/src/main/java/org/onap/aai/dataexport/DataExportTasks.java +++ b/src/main/java/org/onap/aai/dataexport/DataExportTasks.java @@ -19,15 +19,29 @@ */ package org.onap.aai.dataexport; -import com.att.eelf.configuration.Configuration; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.comparator.LastModifiedFileComparator; -import org.apache.commons.io.filefilter.DirectoryFileFilter; -import org.apache.commons.io.filefilter.FileFileFilter; -import org.apache.commons.io.filefilter.RegexFileFilter; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Properties; +import java.util.TreeMap; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog; import org.onap.aai.dbgen.DynamicPayloadGenerator; import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.logging.LogFormatTools; @@ -35,18 +49,22 @@ import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; import org.onap.logging.filter.base.ONAPComponents; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.onap.logging.ref.slf4j.ONAPLogConstants; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import java.io.*; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import com.att.eelf.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.apache.commons.io.comparator.LastModifiedFileComparator; +import org.apache.commons.io.filefilter.DirectoryFileFilter; +import org.apache.commons.io.filefilter.FileFileFilter; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.RegexFileFilter; /** * DataExportTasks obtains a graph snapshot and invokes DynamicPayloadGenerator @@ -54,11 +72,12 @@ import java.util.regex.Pattern; */ @Component @PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties") +@ConditionalOnProperty(prefix = "dataexporttask", name = "cron") public class DataExportTasks { private static final Logger LOGGER; private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); - + static { System.setProperty("aai.service.name", DataExportTasks.class.getSimpleName()); Properties props = System.getProperties(); @@ -94,11 +113,19 @@ public class DataExportTasks { /** * The exportTask method. * + * @throws AAIException, Exception */ - public void exportTask() throws Exception { - AaiScheduledTaskAuditLog auditLog = new AaiScheduledTaskAuditLog(); + public void exportTask() throws AAIException, Exception { + AaiScheduledTaskAuditLog auditLog = new AaiScheduledTaskAuditLog(); auditLog.logBefore("dataExportTask", ONAPComponents.AAI.toString()); LOGGER.info("Started exportTask: " + dateFormat.format(new Date())); + try { + String isDataExportEnabled = AAIConfig.get("aai.dataexport.enable"); + } catch (AAIException ex){ + LOGGER.info("Ended exportTask: " + dateFormat.format(new Date()) + " " + ex.getMessage()); + auditLog.logAfter(); + throw ex; + } if (AAIConfig.get("aai.dataexport.enable").equalsIgnoreCase("false")) { LOGGER.debug("Data Export is not enabled"); return; @@ -208,7 +235,7 @@ public class DataExportTasks { LOGGER.debug("Exception while running the check to see if dataExport is running "+ LogFormatTools.getStackTop(e)); } - return count > 0; + return count > 0; } /** @@ -275,7 +302,7 @@ public class DataExportTasks { String g1 = m.group(1); LOGGER.debug ("Found group " + g1); if ( !fileMap.containsKey(g1) ) { - ArrayList<File> l = new ArrayList<>(); + ArrayList<File> l = new ArrayList<File>(); l.add(f); fileMap.put(g1, l); } @@ -317,6 +344,7 @@ public class DataExportTasks { /** * The deletePayload method deletes all the payload files that it finds at targetDirectory * @param targetDirFile the directory that contains payload files + * @throws AAIException */ private static void deletePayload(File targetDirFile) { diff --git a/src/main/java/org/onap/aai/datagrooming/DataGrooming.java b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java index 88bfebb..3884a01 100644 --- a/src/main/java/org/onap/aai/datagrooming/DataGrooming.java +++ b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java @@ -47,7 +47,7 @@ import org.apache.tinkerpop.gremlin.structure.Property; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.onap.aai.GraphAdminApp; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.util.GraphAdminConstants; import org.onap.logging.ref.slf4j.ONAPLogConstants; import org.onap.aai.dbmap.AAIGraph; diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java index 217d6c0..946489b 100644 --- a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java +++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java @@ -317,7 +317,7 @@ public class DataSnapshot { } else if( command.equals("MULTITHREAD_RELOAD") ){ // Note - this will use as many threads as the snapshot file is - // broken up into. (up to a limit) + // broken up into. (up to a limit - whatever the 'threadCount' variable is set to) if (args.length >= 2) { // Since they are re-loading, they need to pass the snapshot file name to use. // We expected the file to be found in our snapshot directory. Note - if @@ -357,8 +357,14 @@ public class DataSnapshot { } } - - threadCount4Create = cArgs.threadCount; + try { + threadCount4Create = cArgs.threadCount; + } + catch ( NumberFormatException nfe ){ + ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) threadCount found for DataSnapshot [" + cArgs.threadCount + "]"); + LOGGER.debug("Bad (non-integer) threadCount found by DataSnapshot [" + cArgs.threadCount + "]"); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } maxNodesPerFile4Create = cArgs.maxNodesPerFile; //Print Defaults LOGGER.debug("DataSnapshot command is [" + cArgs.command + "]"); @@ -375,8 +381,7 @@ public class DataSnapshot { LOGGER.debug("VertToEdgeProcDelay is [" + cArgs.vertToEdgeProcDelay + "]"); LOGGER.debug("StaggerThreadDelay is [" + cArgs.staggerThreadDelay + "]"); LOGGER.debug("Caller process is ["+ cArgs.caller + "]"); - - + //Print non-default values if (!AAIConfig.isEmpty(cArgs.fileName)){ LOGGER.debug("Snapshot file name (if not default) to use is [" + cArgs.fileName + "]"); @@ -553,6 +558,9 @@ public class DataSnapshot { // NOTE - it will only use as many threads as the number of files the // snapshot is written to. Ie. if you have a single-file snapshot, // then this will be single-threaded. + // If the number of files is greater than the 'threadCount' parameter, + // then we will use more than one pass to keep the number of simultaneous + // threads below the threadCount param. // LOGGER.debug(" Command = " + command ); @@ -561,114 +569,129 @@ public class DataSnapshot { } ArrayList <File> snapFilesArr = getFilesToProcess(targetDir, oldSnapshotFileName, false); int fCount = snapFilesArr.size(); + int threadPassesNeeded = (int) Math.ceil((double)fCount / (double)threadCount4Create); + int filesPerPass = (int) Math.ceil((double)fCount / (double)threadPassesNeeded); + JanusGraph graph1 = AAIGraph.getInstance().getGraph(); long timeStart = System.nanoTime(); HashMap <String,String> old2NewVertIdMap = new HashMap <String,String> (); - // We're going to try loading in the vertices - without edges or properties - // using Separate threads - - ExecutorService executor = Executors.newFixedThreadPool(fCount); - List<Future<HashMap<String,String>>> list = new ArrayList<Future<HashMap<String,String>>>(); - - for( int i=0; i < fCount; i++ ){ - File f = snapFilesArr.get(i); + ExecutorService executor = Executors.newFixedThreadPool(fCount); + int threadFailCount = 0; + + LOGGER.debug(" -- vertAddDelayMs = " + vertAddDelayMs + + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs + + ", maxErrorsPerThread = " + maxErrorsPerThread ); + + // -------------------------------------- + // Step 1 -- Load empty vertices + // -------------------------------------- + int fileNo = 0; + for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){ + List<Future<HashMap<String,String>>> listFutV = new ArrayList<Future<HashMap<String,String>>>(); + + int thisPassCount = 0; + while( (thisPassCount < filesPerPass) && (fileNo < fCount) ){ + File f = snapFilesArr.get(fileNo); String fname = f.getName(); String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname; Thread.sleep(cArgs.staggerThreadDelay); // Stagger the threads a bit LOGGER.debug(" -- Read file: [" + fullSnapName + "]"); - LOGGER.debug(" -- Call the PartialVertexLoader to just load vertices ----"); - LOGGER.debug(" -- vertAddDelayMs = " + vertAddDelayMs - + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs - + ", maxErrorsPerThread = " + maxErrorsPerThread ); Callable <HashMap<String,String>> vLoader = new PartialVertexLoader(graph1, fullSnapName, vertAddDelayMs, failureDelayMs, retryDelayMs, maxErrorsPerThread, LOGGER); Future <HashMap<String,String>> future = (Future<HashMap<String, String>>) executor.submit(vLoader); - // add Future to the list, we can get return value using Future - list.add(future); - LOGGER.debug(" -- Starting PartialDbLoad VERT_ONLY thread # "+ i ); + // add future to the list, we can get return value later + listFutV.add(future); + LOGGER.debug(" -- Starting PartialDbLoad VERT_ONLY file # "+ fileNo + + "( passNo = " + passNo + ", passIndex = " + thisPassCount + ")"); + + thisPassCount++; + fileNo++; } - + int threadCount4Reload = 0; - int threadFailCount = 0; - for(Future<HashMap<String,String>> fut : list){ - threadCount4Reload++; - try { - old2NewVertIdMap.putAll(fut.get()); - LOGGER.debug(" -- back from PartialVertexLoader. returned thread # " + threadCount4Reload + - ", current size of old2NewVertMap is: " + old2NewVertIdMap.size() ); - } - catch (InterruptedException e) { - threadFailCount++; - AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); + for(Future<HashMap<String,String>> fut : listFutV){ + threadCount4Reload++; + try { + old2NewVertIdMap.putAll(fut.get()); + LOGGER.debug(" -- back from PartialVertexLoader. returned pass # " + + passNo + ", thread # " + + threadCount4Reload + + ", current size of old2NewVertMap is: " + old2NewVertIdMap.size() ); + } + catch (InterruptedException e) { + threadFailCount++; + AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); ErrorLogHelper.logException(ae); - } - catch (ExecutionException e) { - threadFailCount++; - AAIException ae = new AAIException("AAI_6128", e , "ExecutionException"); + } + catch (ExecutionException e) { + threadFailCount++; + AAIException ae = new AAIException("AAI_6128", e , "ExecutionException"); ErrorLogHelper.logException(ae); - } - } - - executor.shutdown(); - - if( threadFailCount > 0 ) { - String emsg = " FAILURE >> " + threadFailCount + " Vertex-loader thread(s) failed to complete successfully. "; - LOGGER.debug(emsg); - throw new Exception( emsg ); + } } + } // end of passes for loading empty vertices + + executor.shutdown(); + if( threadFailCount > 0 ) { + String emsg = " FAILURE >> " + threadFailCount + " Vertex-loader thread(s) failed to complete successfully. "; + LOGGER.debug(emsg); + throw new Exception( emsg ); + } - long timeX = System.nanoTime(); - long diffTime = timeX - timeStart; - long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); - long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); - LOGGER.debug(" -- To reload just the vertex ids from the snapshot files, it took: " + - minCount + " minutes, " + secCount + " seconds " ); - - // Give the DB a little time to chew on all those vertices - Thread.sleep(vertToEdgeProcDelay); - - // ---------------------------------------------------------------------------------------- - LOGGER.debug("\n\n\n -- Now do the edges/props ----------------------"); - // ---------------------------------------------------------------------------------------- + long timeX = System.nanoTime(); + long diffTime = timeX - timeStart; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To reload just the vertex ids from the snapshot files, it took: " + + minCount + " minutes, " + secCount + " seconds " ); + // Give the DB a little time to chew on all those new vertices + Thread.sleep(vertToEdgeProcDelay); - // We're going to try loading in the edges and missing properties - // Note - we're passing the whole oldVid2newVid mapping to the PartialPropAndEdgeLoader - // so that the String-updates to the GraphSON will happen in the threads instead of - // here in the un-threaded calling method. - executor = Executors.newFixedThreadPool(fCount); - ArrayList<Future<ArrayList<String>>> listEdg = new ArrayList<Future<ArrayList<String>>>(); - for( int i=0; i < fCount; i++ ){ - File f = snapFilesArr.get(i); + + // ------------------------------------------------------------- + // Step 2 -- Load Edges and properties onto the empty vertices + // ------------------------------------------------------------- + LOGGER.debug("\n\n\n -- Now load the edges/properties ----------------------"); + executor = Executors.newFixedThreadPool(fCount); + + fileNo = 0; + for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){ + ArrayList<Future<ArrayList<String>>> listFutEdg = new ArrayList<Future<ArrayList<String>>>(); + + int thisPassCount = 0; + while( (thisPassCount < filesPerPass) && (fileNo < fCount) ){ + File f = snapFilesArr.get(fileNo); String fname = f.getName(); String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname; Thread.sleep(cArgs.staggerThreadDelay); // Stagger the threads a bit LOGGER.debug(" -- Read file: [" + fullSnapName + "]"); - LOGGER.debug(" -- Call the PartialPropAndEdgeLoader for Properties and EDGEs ----"); - LOGGER.debug(" -- edgeAddDelayMs = " + vertAddDelayMs - + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs - + ", maxErrorsPerThread = " + maxErrorsPerThread ); - - Callable eLoader = new PartialPropAndEdgeLoader(graph1, fullSnapName, + Callable eLoader = new PartialPropAndEdgeLoader(graph1, fullSnapName, edgeAddDelayMs, failureDelayMs, retryDelayMs, old2NewVertIdMap, maxErrorsPerThread, LOGGER); Future <ArrayList<String>> future = (Future<ArrayList<String>>) executor.submit(eLoader); - //add Future to the list, we can get return value using Future - listEdg.add(future); - LOGGER.debug(" -- Starting PartialPropAndEdge thread # "+ i ); + // add future to the list, we can wait for it below + listFutEdg.add(future); + LOGGER.debug(" -- Starting PartialPropAndEdge file # " + + fileNo + " (pass # " + passNo + ", passIndex " + + thisPassCount + ")" ); + + thisPassCount++; + fileNo++; } - threadCount4Reload = 0; - for(Future<ArrayList<String>> fut : listEdg){ - threadCount4Reload++; - try{ - fut.get(); // DEBUG -- should be doing something with the return value if it's not empty - ie. errors - LOGGER.debug(" -- back from PartialPropAndEdgeLoader. thread # " + threadCount4Reload ); - } + int threadCount4Reload = 0; + for( Future<ArrayList<String>> fut : listFutEdg ){ + threadCount4Reload++; + try{ + fut.get(); + LOGGER.debug(" -- back from PartialPropAndEdgeLoader. pass # " + + passNo + ", thread # " + threadCount4Reload ); + } catch (InterruptedException e) { threadFailCount++; AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); @@ -680,30 +703,32 @@ public class DataSnapshot { ErrorLogHelper.logException(ae); } } + + } // end of passes for reloading edges and properties - executor.shutdown(); + executor.shutdown(); - if( threadFailCount > 0 ) { - String emsg = " FAILURE >> " + threadFailCount + " Property/Edge-loader thread(s) failed to complete successfully. "; - LOGGER.debug(emsg); - throw new Exception( emsg ); - } + if( threadFailCount > 0 ) { + String emsg = " FAILURE >> " + threadFailCount + " Property/Edge-loader thread(s) failed to complete successfully. "; + LOGGER.debug(emsg); + throw new Exception( emsg ); + } - // This is needed so we can see the data committed by the called threads - graph1.tx().commit(); + // This is needed so we can see the data committed by the called threads + graph1.tx().commit(); - long timeEnd = System.nanoTime(); - diffTime = timeEnd - timeX; - minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); - secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); - LOGGER.debug(" -- To reload the edges and properties from snapshot files, it took: " + - minCount + " minutes, " + secCount + " seconds " ); + long timeEnd = System.nanoTime(); + diffTime = timeEnd - timeX; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To reload the edges and properties from snapshot files, it took: " + + minCount + " minutes, " + secCount + " seconds " ); - long totalDiffTime = timeEnd - timeStart; - long totalMinCount = TimeUnit.NANOSECONDS.toMinutes(totalDiffTime); - long totalSecCount = TimeUnit.NANOSECONDS.toSeconds(totalDiffTime) - (60 * totalMinCount); - LOGGER.debug(" -- TOTAL multi-threaded reload time: " + - totalMinCount + " minutes, " + totalSecCount + " seconds " ); + long totalDiffTime = timeEnd - timeStart; + long totalMinCount = TimeUnit.NANOSECONDS.toMinutes(totalDiffTime); + long totalSecCount = TimeUnit.NANOSECONDS.toSeconds(totalDiffTime) - (60 * totalMinCount); + LOGGER.debug(" -- TOTAL multi-threaded reload time: " + + totalMinCount + " minutes, " + totalSecCount + " seconds " ); } else if (command.equals("CLEAR_ENTIRE_DATABASE")) { // ------------------------------------------------------------------ diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java index 8d250d7..9aba8cf 100644 --- a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java +++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java @@ -40,7 +40,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.tinkerpop.gremlin.structure.io.IoCore; @@ -75,8 +75,8 @@ import com.beust.jcommander.ParameterException; public class DataSnapshot4HistInit { - private static Logger LOGGER; - + private static Logger LOGGER = LoggerFactory.getLogger(DataSnapshot4HistInit.class); + /* Using realtime d */ private static final String REALTIME_DB = "realtime"; @@ -109,6 +109,13 @@ public class DataSnapshot4HistInit { */ public static void main(String[] args) { + // Set the logging file properties to be used by EELFManager + System.setProperty("aai.service.name", DataSnapshot4HistInit.class.getSimpleName()); + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); + + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); initializer.initialize(ctx); @@ -149,13 +156,6 @@ public class DataSnapshot4HistInit { public boolean executeCommand(String[] args) { - // Set the logging file properties to be used by EELFManager - System.setProperty("aai.service.name", DataSnapshot4HistInit.class.getSimpleName()); - Properties props = System.getProperties(); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); - LOGGER = LoggerFactory.getLogger(DataSnapshot4HistInit.class); - Boolean dbClearFlag = false; JanusGraph graph = null; String command = "UNKNOWN"; @@ -576,19 +576,25 @@ public class DataSnapshot4HistInit { // NOTE - it will only use as many threads as the number of files the // snapshot is written to. Ie. if you have a single-file snapshot, // then this will be single-threaded. + // If the number of files is greater than the 'threadCount' parameter, + // then we will use more than one pass to keep the number of simultaneous + // threads below the threadCount param. // LOGGER.debug(" Command = " + command ); + if (cArgs.oldFileDir != null && cArgs.oldFileDir != ""){ targetDir = cArgs.oldFileDir; } ArrayList <File> snapFilesArr = getFilesToProcess(targetDir, oldSnapshotFileName, false); int fCount = snapFilesArr.size(); + int threadPassesNeeded = (int) Math.ceil((double)fCount / (double)threadCount4Create); + int filesPerPass = (int) Math.ceil((double)fCount / (double)threadPassesNeeded); + JanusGraph graph1 = AAIGraph.getInstance().getGraph(); - GraphAdminDBUtils.logConfigs(graph1.configuration()); long timeStart = System.nanoTime(); - HashMap <String,String> old2NewVertIdMap = new <String,String> HashMap (); - HashMap <String,ArrayList<String>> nodeKeyNames = new <String,ArrayList<String>> HashMap (); - + GraphAdminDBUtils.logConfigs(graph1.configuration()); + HashMap <String,String> old2NewVertIdMap = new HashMap <String,String> (); + HashMap <String,ArrayList<String>> nodeKeyNames = new HashMap <String,ArrayList<String>> (); try { LOGGER.debug("call getNodeKeyNames ()" ); nodeKeyNames = getNodeKeyNames(); @@ -597,142 +603,162 @@ public class DataSnapshot4HistInit { ErrorLogHelper.logException(ae); AAISystemExitUtil.systemExitCloseAAIGraph(1); } + + ExecutorService executor = Executors.newFixedThreadPool(fCount); + int threadFailCount = 0; - // We're going to try loading in the vertices - without edges or properties - // using Separate threads + LOGGER.debug(" -- vertAddDelayMs = " + vertAddDelayMs + + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs + + ", maxErrorsPerThread = " + maxErrorsPerThread ); + + // -------------------------------------- + // Step 1 -- Load empty vertices + // -------------------------------------- + int fileNo = 0; + for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){ + List<Future<HashMap<String,String>>> listFutV = new ArrayList<Future<HashMap<String,String>>>(); - ExecutorService executor = Executors.newFixedThreadPool(fCount); - List<Future<HashMap<String,String>>> list = new ArrayList<Future<HashMap<String,String>>>(); - for( int i=0; i < fCount; i++ ){ - File f = snapFilesArr.get(i); + int thisPassCount = 0; + while( (thisPassCount < filesPerPass) && (fileNo < fCount) ){ + File f = snapFilesArr.get(fileNo); String fname = f.getName(); String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname; Thread.sleep(cArgs.staggerThreadDelay); // Stagger the threads a bit LOGGER.debug(" -- Read file: [" + fullSnapName + "]"); - LOGGER.debug(" -- Call the PartialVertexLoader to just load vertices ----"); - LOGGER.debug(" -- vertAddDelayMs = " + vertAddDelayMs - + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs - + ", maxErrorsPerThread = " + maxErrorsPerThread ); Callable <HashMap<String,String>> vLoader = new PartialVertexLoader(graph1, fullSnapName, vertAddDelayMs, failureDelayMs, retryDelayMs, maxErrorsPerThread, LOGGER); Future <HashMap<String,String>> future = (Future<HashMap<String, String>>) executor.submit(vLoader); - // add Future to the list, we can get return value using Future - list.add(future); - LOGGER.debug(" -- Starting PartialDbLoad VERT_ONLY thread # "+ i ); + // add future to the list, we can get return value later + listFutV.add(future); + LOGGER.debug(" -- Starting PartialDbLoad VERT_ONLY file # "+ fileNo + + "( passNo = " + passNo + ", passIndex = " + thisPassCount + ")"); + + thisPassCount++; + fileNo++; } - + int threadCount4Reload = 0; - int threadFailCount = 0; - for(Future<HashMap<String,String>> fut : list){ - threadCount4Reload++; - try { - old2NewVertIdMap.putAll(fut.get()); - LOGGER.debug(" -- back from PartialVertexLoader. returned thread # " + threadCount4Reload + - ", current size of old2NewVertMap is: " + old2NewVertIdMap.size() ); - } - catch (InterruptedException e) { - threadFailCount++; - AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); - ErrorLogHelper.logException(ae); - } - catch (ExecutionException e) { - threadFailCount++; - AAIException ae = new AAIException("AAI_6128", e , "ExecutionException"); - ErrorLogHelper.logException(ae); - } - } - executor.shutdown(); - - if( threadFailCount > 0 ) { - String emsg = " FAILURE >> " + threadFailCount + " Vertex-loader thread(s) failed to complete successfully. "; - LOGGER.debug(emsg); - throw new Exception( emsg ); + for(Future<HashMap<String,String>> fut : listFutV){ + threadCount4Reload++; + try { + old2NewVertIdMap.putAll(fut.get()); + LOGGER.debug(" -- back from PartialVertexLoader. returned pass # " + + passNo + ", thread # " + + threadCount4Reload + + ", current size of old2NewVertMap is: " + old2NewVertIdMap.size() ); + } + catch (InterruptedException e) { + threadFailCount++; + AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); + ErrorLogHelper.logException(ae); + } + catch (ExecutionException e) { + threadFailCount++; + AAIException ae = new AAIException("AAI_6128", e , "ExecutionException"); + ErrorLogHelper.logException(ae); + } } + } // end of passes for loading empty vertices + + executor.shutdown(); + if( threadFailCount > 0 ) { + String emsg = " FAILURE >> " + threadFailCount + " Vertex-loader thread(s) failed to complete successfully. "; + LOGGER.debug(emsg); + throw new Exception( emsg ); + } - long timeX = System.nanoTime(); - long diffTime = timeX - timeStart; - long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); - long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); - LOGGER.debug(" -- To reload just the vertex ids from the snapshot files, it took: " + - minCount + " minutes, " + secCount + " seconds " ); + long timeX = System.nanoTime(); + long diffTime = timeX - timeStart; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To reload just the vertex ids from the snapshot files, it took: " + + minCount + " minutes, " + secCount + " seconds " ); - // Give the DB a little time to chew on all those vertices - Thread.sleep(vertToEdgeProcDelay); + // Give the DB a little time to chew on all those new vertices + Thread.sleep(vertToEdgeProcDelay); - // ---------------------------------------------------------------------------------------- - LOGGER.debug("\n\n\n -- Now do the edges/props ----------------------"); - // ---------------------------------------------------------------------------------------- - // We're going to try loading in the edges and missing properties - // Note - we're passing the whole oldVid2newVid mapping to the PartialPropAndEdgeLoader - // so that the String-updates to the GraphSON will happen in the threads instead of - // here in the un-threaded calling method. - executor = Executors.newFixedThreadPool(fCount); - ArrayList<Future<ArrayList<String>>> listEdg = new ArrayList<Future<ArrayList<String>>>(); - for( int i=0; i < fCount; i++ ){ - File f = snapFilesArr.get(i); + // ------------------------------------------------------------- + // Step 2 -- Load Edges and properties onto the empty vertices + // ------------------------------------------------------------- + LOGGER.debug("\n\n\n -- Now load the edges/properties ----------------------"); + executor = Executors.newFixedThreadPool(fCount); + + fileNo = 0; + for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){ + ArrayList<Future<ArrayList<String>>> listFutEdg = new ArrayList<Future<ArrayList<String>>>(); + + int thisPassCount = 0; + while( (thisPassCount < filesPerPass) && (fileNo < fCount) ){ + File f = snapFilesArr.get(fileNo); String fname = f.getName(); String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname; Thread.sleep(cArgs.staggerThreadDelay); // Stagger the threads a bit LOGGER.debug(" -- Read file: [" + fullSnapName + "]"); - LOGGER.debug(" -- Call the PartialPropAndEdgeLoader4HistInit for Properties and EDGEs ----"); - LOGGER.debug(" -- edgeAddDelayMs = " + vertAddDelayMs - + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs - + ", maxErrorsPerThread = " + maxErrorsPerThread ); - - - Callable eLoader = new PartialPropAndEdgeLoader4HistInit(graph1, fullSnapName, + Callable eLoader = new PartialPropAndEdgeLoader4HistInit(graph1, fullSnapName, edgeAddDelayMs, failureDelayMs, retryDelayMs, old2NewVertIdMap, maxErrorsPerThread, LOGGER, scriptStartTime, nodeKeyNames); + Future <ArrayList<String>> future = (Future<ArrayList<String>>) executor.submit(eLoader); - //add Future to the list, we can get return value using Future - listEdg.add(future); - LOGGER.debug(" -- Starting PartialPropAndEdge thread # "+ i ); + // add future to the list, we can wait for it below + listFutEdg.add(future); + LOGGER.debug(" -- Starting PartialPropAndEdgeLoader4HistInit file # " + + fileNo + " (pass # " + passNo + ", passIndex " + + thisPassCount + ")" ); + + thisPassCount++; + fileNo++; } - threadCount4Reload = 0; - for(Future<ArrayList<String>> fut : listEdg){ - threadCount4Reload++; - try{ - fut.get(); // DEBUG -- should be doing something with the return value if it's not empty - ie. errors - LOGGER.debug(" -- back from PartialPropAndEdgeLoader. thread # " + threadCount4Reload ); - } + + int threadCount4Reload = 0; + for( Future<ArrayList<String>> fut : listFutEdg ){ + threadCount4Reload++; + try{ + fut.get(); + LOGGER.debug(" -- back from PartialPropAndEdgeLoader4HistInit. pass # " + + passNo + ", thread # " + threadCount4Reload ); + } catch (InterruptedException e) { threadFailCount++; AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); - ErrorLogHelper.logException(ae); + ErrorLogHelper.logException(ae); } catch (ExecutionException e) { threadFailCount++; AAIException ae = new AAIException("AAI_6128", e , "ExecutionException"); - ErrorLogHelper.logException(ae); + ErrorLogHelper.logException(ae); } } + + } // end of passes for reloading edges and properties - executor.shutdown(); - if( threadFailCount > 0 ) { - String emsg = " FAILURE >> " + threadFailCount + " Property/Edge-loader thread(s) failed to complete successfully. "; - LOGGER.debug(emsg); - throw new Exception( emsg ); - } + executor.shutdown(); - // This is needed so we can see the data committed by the called threads - graph1.tx().commit(); + if( threadFailCount > 0 ) { + String emsg = " FAILURE >> " + threadFailCount + " Property/Edge-loader thread(s) failed to complete successfully. "; + LOGGER.debug(emsg); + throw new Exception( emsg ); + } - long timeEnd = System.nanoTime(); - diffTime = timeEnd - timeX; - minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); - secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); - LOGGER.debug(" -- To reload the edges and properties from snapshot files, it took: " + - minCount + " minutes, " + secCount + " seconds " ); + // This is needed so we can see the data committed by the called threads + graph1.tx().commit(); - long totalDiffTime = timeEnd - timeStart; - long totalMinCount = TimeUnit.NANOSECONDS.toMinutes(totalDiffTime); - long totalSecCount = TimeUnit.NANOSECONDS.toSeconds(totalDiffTime) - (60 * totalMinCount); - LOGGER.debug(" -- TOTAL multi-threaded reload time: " + - totalMinCount + " minutes, " + totalSecCount + " seconds " ); + long timeEnd = System.nanoTime(); + diffTime = timeEnd - timeX; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To reload the edges and properties from snapshot files, it took: " + + minCount + " minutes, " + secCount + " seconds " ); + + long totalDiffTime = timeEnd - timeStart; + long totalMinCount = TimeUnit.NANOSECONDS.toMinutes(totalDiffTime); + long totalSecCount = TimeUnit.NANOSECONDS.toSeconds(totalDiffTime) - (60 * totalMinCount); + LOGGER.debug(" -- TOTAL multi-threaded reload time: " + + totalMinCount + " minutes, " + totalSecCount + " seconds " ); + } else if (command.equals("CLEAR_ENTIRE_DATABASE")) { // ------------------------------------------------------------------ // They are calling this to clear the db before re-loading it @@ -1075,4 +1101,4 @@ public class DataSnapshot4HistInit { } -} +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/db/schema/ScriptDriver.java b/src/main/java/org/onap/aai/db/schema/ScriptDriver.java index a634e0b..3d9ec69 100644 --- a/src/main/java/org/onap/aai/db/schema/ScriptDriver.java +++ b/src/main/java/org/onap/aai/db/schema/ScriptDriver.java @@ -27,7 +27,7 @@ import org.apache.commons.configuration.PropertiesConfiguration; import org.codehaus.jackson.JsonGenerationException; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.JanusGraphFactory; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbmap.AAIGraphConfig; import org.onap.aai.edges.EdgeIngestor; import org.onap.aai.exceptions.AAIException; diff --git a/src/main/java/org/onap/aai/dbgen/DupeTool.java b/src/main/java/org/onap/aai/dbgen/DupeTool.java index d8d3ce0..0532e2e 100644 --- a/src/main/java/org/onap/aai/dbgen/DupeTool.java +++ b/src/main/java/org/onap/aai/dbgen/DupeTool.java @@ -28,7 +28,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; import org.apache.tinkerpop.gremlin.structure.*; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.JanusGraphFactory; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.dbmap.AAIGraphConfig; import org.onap.aai.edges.enums.AAIDirection; diff --git a/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java b/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java index 2e1bc4b..ba5fad4 100644 --- a/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java +++ b/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java @@ -30,7 +30,7 @@ import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.node.ObjectNode; import org.codehaus.jackson.type.TypeReference; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.db.props.AAIProperties; import org.onap.aai.dbmap.InMemoryGraph; import org.onap.aai.edges.EdgeIngestor; diff --git a/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java b/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java index 7bb0555..c12f3f3 100644 --- a/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java +++ b/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java @@ -100,29 +100,24 @@ public class ForceDeleteTool { for (int i = 0; i < args.length; i++) { String thisArg = args[i]; argStr4Msg = argStr4Msg + " " + thisArg; + if (thisArg.equals("-action")) { i++; if (i >= args.length) { logger.error(" No value passed with -action option. "); - System.out.println(" No value passed with -action option. "); exit(0); } - else { - actionVal = args[i]; - argStr4Msg = argStr4Msg + " " + actionVal; - } + actionVal = args[i]; + argStr4Msg = argStr4Msg + " " + actionVal; } else if (thisArg.equals("-userId")) { i++; if (i >= args.length) { logger.error(" No value passed with -userId option. "); - System.out.println(" No value passed with -userId option. "); exit(0); } - else { - userIdVal = args[i]; - argStr4Msg = argStr4Msg + " " + userIdVal; - } + userIdVal = args[i]; + argStr4Msg = argStr4Msg + " " + userIdVal; } else if (thisArg.equals("-overRideProtection")) { overRideProtection = true; @@ -134,54 +129,40 @@ public class ForceDeleteTool { i++; if (i >= args.length) { logger.error(" No value passed with -vertexId option. "); - System.out.println(" No value passed with -vertexId option. "); exit(0); } - else { - String nextArg = args[i]; - argStr4Msg = argStr4Msg + " " + nextArg; - - try { - vertexIdLong = Long.parseLong(nextArg); - } catch (Exception e) { - logger.error("Bad value passed with -vertexId option: [" + String nextArg = args[i]; + argStr4Msg = argStr4Msg + " " + nextArg; + try { + vertexIdLong = Long.parseLong(nextArg); + } catch (Exception e) { + logger.error("Bad value passed with -vertexId option: [" + nextArg + "]"); - System.out.println("Bad value passed with -vertexId option: [" - + nextArg + "]"); - exit(0); - } + exit(0); } } else if (thisArg.equals("-params4Collect")) { i++; if (i >= args.length) { logger.error(" No value passed with -params4Collect option. "); - System.out.println(" No value passed with -params4Collect option. "); exit(0); } - else { - dataString = args[i]; - argStr4Msg = argStr4Msg + " " + dataString; - } + dataString = args[i]; + argStr4Msg = argStr4Msg + " " + dataString; } else if (thisArg.equals("-edgeId")) { i++; if (i >= args.length) { logger.error(" No value passed with -edgeId option. "); - System.out.println(" No value passed with -edgeId option. "); exit(0); } - else { - String nextArg = args[i]; - argStr4Msg = argStr4Msg + " " + nextArg; - edgeIdStr = nextArg; - } + String nextArg = args[i]; + argStr4Msg = argStr4Msg + " " + nextArg; + edgeIdStr = nextArg; } else { logger.error(" Unrecognized argument passed to ForceDeleteTool: [" + thisArg + "]. "); - System.out.println(" Unrecognized argument passed to ForceDeleteTool: [" - + thisArg + "]"); logger.error(" Valid values are: -action -userId -vertexId -edgeId -overRideProtection -params4Collect -DISPLAY_ALL_VIDS"); exit(0); } @@ -368,19 +349,18 @@ public class ForceDeleteTool { System.out.println(infMsg); exit(0); } - else { - if( fd.getEdgeDelConfirmation(logger, userIdVal, thisEdge, overRideProtection) ){ - thisEdge.remove(); - graph.tx().commit(); - String infMsg = ">>>>>>>>>> Removed edge with edgeId = " + edgeIdStr; - logger.debug( infMsg ); - System.out.println(infMsg); - } - else { - String infMsg = " Delete Cancelled. "; - System.out.println(infMsg); - logger.debug( infMsg ); - } + + if( fd.getEdgeDelConfirmation(logger, userIdVal, thisEdge, overRideProtection) ){ + thisEdge.remove(); + graph.tx().commit(); + String infMsg = ">>>>>>>>>> Removed edge with edgeId = " + edgeIdStr; + logger.debug( infMsg ); + System.out.println(infMsg); + } + else { + String infMsg = " Delete Cancelled. "; + System.out.println(infMsg); + logger.debug( infMsg ); } exit(0); } diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java index 15b4c81..6e48d4b 100644 --- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java +++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java @@ -20,7 +20,7 @@ package org.onap.aai.dbgen.schemamod; import com.att.eelf.configuration.Configuration; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Loader; import org.onap.aai.introspection.LoaderFactory; diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java index 837f8a8..4c393eb 100644 --- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java +++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java @@ -20,7 +20,7 @@ package org.onap.aai.dbgen.schemamod; import com.att.eelf.configuration.Configuration; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Loader; import org.onap.aai.introspection.LoaderFactory; diff --git a/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java b/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java index 45b5d04..e0adec0 100644 --- a/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java +++ b/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java @@ -22,7 +22,7 @@ package org.onap.aai.historytruncate; import java.util.*; import java.util.concurrent.TimeUnit; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.apache.tinkerpop.gremlin.process.traversal.P; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.structure.Edge; diff --git a/src/main/java/org/onap/aai/interceptors/pre/VersionInterceptor.java b/src/main/java/org/onap/aai/interceptors/pre/VersionInterceptor.java index f591120..5d32649 100644 --- a/src/main/java/org/onap/aai/interceptors/pre/VersionInterceptor.java +++ b/src/main/java/org/onap/aai/interceptors/pre/VersionInterceptor.java @@ -62,7 +62,7 @@ public class VersionInterceptor extends AAIContainerFilter implements ContainerR String uri = requestContext.getUriInfo().getPath(); - if (uri.startsWith("search") || uri.startsWith("util/echo") || uri.startsWith("tools")) { + if (uri.startsWith("search") || uri.startsWith("util/echo") || uri.startsWith("tools") || uri.endsWith("audit-sql-db")) { return; } diff --git a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java index 3f25119..d2a336b 100644 --- a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java +++ b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java @@ -140,7 +140,7 @@ public abstract class EdgeSwingMigrator extends Migrator { // If the nodeTypes don't match, throw an error if( !oldNodeType.equals(newNodeType) ){ - logger.debug ( "Can not swing edge from a [" + oldNodeType + "] node to a [" + + logger.info ( "Can not swing edge from a [" + oldNodeType + "] node to a [" + newNodeType + "] node. "); success = false; return; @@ -182,7 +182,7 @@ public abstract class EdgeSwingMigrator extends Migrator { } String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present"; - logger.debug ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is [" + logger.info ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is [" + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. "); // remove the old edge @@ -203,8 +203,9 @@ public abstract class EdgeSwingMigrator extends Migrator { Map.Entry pair = (Map.Entry)it.next(); newOutE.property(pair.getKey().toString(), pair.getValue().toString() ); } + logger.info("\n Edge swing of [" + eLabel + "] OUT edge successful"); }else { - logger.debug("\n Edge was not swung due to Multiplicity Rule Violation..."); + logger.info("\n Edge was not swung due to Multiplicity Rule Violation..."); } } } @@ -245,7 +246,7 @@ public abstract class EdgeSwingMigrator extends Migrator { } String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present"; - logger.debug ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is [" + logger.info ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is [" + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. "); @@ -267,8 +268,9 @@ public abstract class EdgeSwingMigrator extends Migrator { Map.Entry pair = (Map.Entry)it.next(); newInE.property(pair.getKey().toString(), pair.getValue().toString() ); } + logger.info("\n Edge swing of [" + eLabel + "] IN edge successful"); } else { - logger.debug("\t Edge was not swung due to Multiplicity Rule Violation..."); + logger.info("\t Edge was not swung due to Multiplicity Rule Violation..."); } } } @@ -276,7 +278,7 @@ public abstract class EdgeSwingMigrator extends Migrator { } } catch (Exception e) { - logger.error("error encountered", e); + logger.info("error encountered", e); success = false; } } diff --git a/src/main/java/org/onap/aai/migration/MigrationController.java b/src/main/java/org/onap/aai/migration/MigrationController.java index 8d758e3..5e969bf 100644 --- a/src/main/java/org/onap/aai/migration/MigrationController.java +++ b/src/main/java/org/onap/aai/migration/MigrationController.java @@ -19,7 +19,7 @@ */ package org.onap.aai.migration; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.edges.EdgeIngestor; import org.onap.aai.exceptions.AAIException; diff --git a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java index 7cb71ca..97ed045 100644 --- a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java +++ b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java @@ -99,8 +99,8 @@ public class MigrationControllerInternal { props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml"); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); - logger = LoggerFactory.getLogger(MigrationControllerInternal.class.getSimpleName()); MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); + logger = LoggerFactory.getLogger(MigrationControllerInternal.class.getSimpleName()); CommandLineArgs cArgs = new CommandLineArgs(); diff --git a/src/main/java/org/onap/aai/migration/Migrator.java b/src/main/java/org/onap/aai/migration/Migrator.java index 7d6a7c1..9498cd1 100644 --- a/src/main/java/org/onap/aai/migration/Migrator.java +++ b/src/main/java/org/onap/aai/migration/Migrator.java @@ -21,7 +21,6 @@ package org.onap.aai.migration; import java.io.File; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.file.Files; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; @@ -39,6 +38,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.json.JSONException; import org.json.JSONObject; +import org.onap.aai.aailog.logs.AaiDebugLog; import org.onap.aai.edges.EdgeIngestor; import org.onap.aai.edges.enums.EdgeType; import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException; @@ -54,9 +54,9 @@ import org.onap.aai.serialization.db.exceptions.NoEdgeRuleFoundException; import org.onap.aai.serialization.engines.TransactionalGraphEngine; import org.onap.aai.setup.SchemaVersion; import org.onap.aai.setup.SchemaVersions; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; /** * This class defines an A&AI Migration @@ -81,6 +81,12 @@ public abstract class Migrator implements Runnable { protected static final String MIGRATION_ERROR = "Migration Error: "; protected static final String MIGRATION_SUMMARY_COUNT = "Migration Summary Count: "; + + private static AaiDebugLog debugLog = new AaiDebugLog(); + static { + debugLog.setupMDC(); + } + /** * Instantiates a new migrator. @@ -96,8 +102,9 @@ public abstract class Migrator implements Runnable { this.schemaVersions = schemaVersions; initDBSerializer(); this.notificationHelper = new NotificationHelper(loader, serializer, loaderFactory, schemaVersions, engine, "AAI-MIGRATION", this.getMigrationName()); - logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); - logger.debug("\tInitilization of " + this.getClass().getSimpleName() + " migration script complete."); + MDC.put("logFilenameAppender", this.getClass().getSimpleName()); + logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + logAndPrint(logger,"\tInitilization of " + this.getClass().getSimpleName() + " migration script complete."); } /** @@ -127,7 +134,7 @@ public abstract class Migrator implements Runnable { */ public void createDmaapFiles(List<String> dmaapMsgList) { String fileName = getMigrationName() + "-" + UUID.randomUUID(); - String logDirectory = System.getProperty("AJSC_HOME") + "/logs/migration/dmaapEvents"; + String logDirectory = System.getProperty("AJSC_HOME") + "/logs/data/dmaapEvents"; File f = new File(logDirectory); f.mkdirs(); @@ -136,10 +143,12 @@ public abstract class Migrator implements Runnable { try { Files.write(Paths.get(logDirectory+"/"+fileName), (Iterable<String>)dmaapMsgList.stream()::iterator); } catch (IOException e) { + System.out.println("Unable to generate file with dmaap msgs for " + getMigrationName() + + " Exception is: " + e.getMessage()); logger.error("Unable to generate file with dmaap msgs for " + getMigrationName(), e); } } else { - logger.debug("No dmaap msgs detected for " + getMigrationName()); + logAndPrint(logger,"No dmaap msgs detected for " + getMigrationName()); } } @@ -150,7 +159,7 @@ public abstract class Migrator implements Runnable { public void createDmaapFilesForDelete(List<Introspector> dmaapDeleteIntrospectorList) {try { System.out.println("dmaapDeleteIntrospectorList :: " + dmaapDeleteIntrospectorList.size()); String fileName = "DELETE-"+ getMigrationName() + "-" + UUID.randomUUID(); - String logDirectory = System.getProperty("AJSC_HOME") + "/logs/migration/dmaapEvents/"; + String logDirectory = System.getProperty("AJSC_HOME") + "/logs/data/dmaapEvents/"; File f = new File(logDirectory); f.mkdirs(); @@ -169,6 +178,8 @@ public abstract class Migrator implements Runnable { finalStr=svIntr.getName() + "#@#" + svIntr.getURI() + "#@#" + str+"\n"; Files.write(Paths.get(logDirectory + "/" + fileName),finalStr.getBytes(),StandardOpenOption.APPEND); } catch (IOException e) { + System.out.println("Unable to generate file with dmaap msgs for " + getMigrationName() + + " Exception is: " + e.getMessage()); logger.error("Unable to generate file with dmaap msgs for "+getMigrationName(), e); } @@ -201,6 +212,7 @@ public abstract class Migrator implements Runnable { result.put(pk.key(), pk.value()); } } catch (JSONException e) { + System.out.println("Warning error reading vertex: " + e.getMessage()); logger.error("Warning error reading vertex: " + e); } @@ -223,6 +235,7 @@ public abstract class Migrator implements Runnable { result.put(pk.key(), pk.value()); } } catch (JSONException e) { + System.out.println("Warning error reading edge: " + e.getMessage()); logger.error("Warning error reading edge: " + e); } @@ -404,4 +417,15 @@ public abstract class Migrator implements Runnable { public NotificationHelper getNotificationHelper() { return this.notificationHelper; } + + /** + * Log and print. + * + * @param logger the logger + * @param msg the msg + */ + protected void logAndPrint(Logger logger, String msg) { + System.out.println(msg); + logger.info(msg); + } } diff --git a/src/main/java/org/onap/aai/migration/NotificationHelper.java b/src/main/java/org/onap/aai/migration/NotificationHelper.java index f10a824..f08c56d 100644 --- a/src/main/java/org/onap/aai/migration/NotificationHelper.java +++ b/src/main/java/org/onap/aai/migration/NotificationHelper.java @@ -39,6 +39,7 @@ import org.onap.aai.serialization.engines.query.QueryEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import org.onap.aai.setup.SchemaVersions; /** @@ -46,7 +47,7 @@ import org.onap.aai.setup.SchemaVersions; */ public class NotificationHelper { - private static final Logger LOGGER = LoggerFactory.getLogger(NotificationHelper.class); + protected Logger LOGGER = null; protected final DBSerializer serializer; protected final Loader loader; protected final TransactionalGraphEngine engine; @@ -61,6 +62,9 @@ public class NotificationHelper { this.transactionId = transactionId; this.sourceOfTruth = sourceOfTruth; this.notification = new UEBNotification(loader, loaderFactory, schemaVersions); + MDC.put("logFilenameAppender", this.getClass().getSimpleName()); + LOGGER = LoggerFactory.getLogger(this.getClass().getSimpleName()); + } public void addEvent(Vertex v, Introspector obj, EventAction action, URI uri, String basePath) throws UnsupportedEncodingException, AAIException { diff --git a/src/main/java/org/onap/aai/migration/ValueMigrator.java b/src/main/java/org/onap/aai/migration/ValueMigrator.java index f45b20b..685855c 100644 --- a/src/main/java/org/onap/aai/migration/ValueMigrator.java +++ b/src/main/java/org/onap/aai/migration/ValueMigrator.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.structure.Vertex; @@ -113,6 +112,8 @@ public abstract class ValueMigrator extends Migrator { } } } catch (Exception e) { + System.out.println(String.format("caught exception updating aai-node-type %s's property %s's value to " + + "%s: %s", nodeType, property, newValue.toString(), e.getMessage())); logger.error(String.format("caught exception updating aai-node-type %s's property %s's value to " + "%s: %s", nodeType, property, newValue.toString(), e.getMessage())); logger.error(e.getMessage()); @@ -121,12 +122,12 @@ public abstract class ValueMigrator extends Migrator { this.nodeTotalSuccess.put(nodeType, Integer.toString(this.subTotal)); } - logger.debug ("\n \n ******* Final Summary for " + " " + getMigrationName() +" ********* \n"); + logAndPrint(logger, " ******* Final Summary for " + " " + getMigrationName() +" ********* "); for (Map.Entry<String, String> migratedNode: nodeTotalSuccess.entrySet()) { - logger.debug("Total Migrated Records for " + migratedNode.getKey() +": " + migratedNode.getValue()); + logAndPrint(logger,"Total Migrated Records for " + migratedNode.getKey() +": " + migratedNode.getValue()); } - logger.debug(this.MIGRATION_SUMMARY_COUNT + "Total Migrated Records: "+ migrationSuccess); + logAndPrint(logger,this.MIGRATION_SUMMARY_COUNT + "Total Migrated Records: "+ migrationSuccess); } @@ -136,18 +137,18 @@ public abstract class ValueMigrator extends Migrator { String propertyValue = v.property(property).value().toString(); if (propertyValue.isEmpty()) { v.property(property, newValue); - logger.debug(String.format("Node Type %s: Property %s is empty, adding value %s", + logAndPrint(logger,String.format("Node Type %s: Property %s is empty, adding value %s", nodeType, property, newValue.toString())); this.touchVertexProperties(v, false); updateDmaapList(v); this.migrationSuccess++; this.subTotal++; } else { - logger.debug(String.format("Node Type %s: Property %s value already exists - skipping", + logAndPrint(logger,String.format("Node Type %s: Property %s value already exists - skipping", nodeType, property)); } } else { - logger.debug(String.format("Node Type %s: Property %s does not exist or " + + logAndPrint(logger,String.format("Node Type %s: Property %s does not exist or " + "updateExistingValues flag is set to True - adding the property with value %s", nodeType, property, newValue.toString())); v.property(property, newValue); @@ -178,7 +179,7 @@ public abstract class ValueMigrator extends Migrator { private void updateDmaapList(Vertex v){ String dmaapMsg = System.nanoTime() + "_" + v.id().toString() + "_" + v.value("resource-version").toString(); dmaapMsgList.add(dmaapMsg); - logger.debug("\tAdding Updated Vertex " + v.id().toString() + " to dmaapMsgList...."); + logAndPrint(logger,"\tAdding Updated Vertex " + v.id().toString() + " to dmaapMsgList...."); } public boolean isUpdateDmaap(){ diff --git a/src/main/java/org/onap/aai/migration/VertexMerge.java b/src/main/java/org/onap/aai/migration/VertexMerge.java index a2c814e..6480e2a 100644 --- a/src/main/java/org/onap/aai/migration/VertexMerge.java +++ b/src/main/java/org/onap/aai/migration/VertexMerge.java @@ -43,6 +43,7 @@ import org.onap.aai.serialization.db.EdgeSerializer; import org.onap.aai.serialization.engines.TransactionalGraphEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; /** * This class recursively merges two vertices passed in. @@ -52,8 +53,7 @@ import org.slf4j.LoggerFactory; */ public class VertexMerge { - private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); - + protected Logger logger = null; private final GraphTraversalSource g; private final TransactionalGraphEngine engine; private final DBSerializer serializer; @@ -70,6 +70,9 @@ public class VertexMerge { this.loader = builder.getLoader(); this.notificationHelper = builder.getHelper(); this.hasNotifications = builder.isHasNotifications(); + MDC.put("logFilenameAppender", this.getClass().getSimpleName()); + logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + } /** diff --git a/src/main/java/org/onap/aai/migration/v20/MigrateL2DefaultToFalse.java b/src/main/java/org/onap/aai/migration/v20/MigrateL2DefaultToFalse.java new file mode 100644 index 0000000..cbe15de --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v20/MigrateL2DefaultToFalse.java @@ -0,0 +1,79 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v20; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Status; +import org.onap.aai.migration.ValueMigrator; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(20) +@MigrationDangerRating(2) +@Enabled +public class MigrateL2DefaultToFalse extends ValueMigrator { + + protected static final String L_INTERFACE_NODE_TYPE = "l-interface"; + protected static final String L2_MULTI_PROPERTY = "l2-multicasting"; + + private static Map<String, Map<String, Boolean>> map; + private static Map<String, Boolean> pair; + + public MigrateL2DefaultToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setL2ToFalse(), false); + } + + private static Map<String, Map<String, Boolean>> setL2ToFalse(){ + map = new HashMap<>(); + pair = new HashMap<>(); + + pair.put(L2_MULTI_PROPERTY, false); + + map.put(L_INTERFACE_NODE_TYPE, pair); + + return map; + } + + @Override + public Status getStatus() { + return Status.SUCCESS; + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{L_INTERFACE_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateL2DefaultToFalse"; + } + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v20/MigrateVlanTag.java b/src/main/java/org/onap/aai/migration/v20/MigrateVlanTag.java new file mode 100644 index 0000000..0519cad --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v20/MigrateVlanTag.java @@ -0,0 +1,391 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v20; + +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.javatuples.Pair; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.migration.EdgeSwingMigrator; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Status; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConstants; + +@MigrationPriority(20) +@MigrationDangerRating(2) +@Enabled +public class MigrateVlanTag extends EdgeSwingMigrator { + + protected static final String CLOUD_REGION_NODE_TYPE = "cloud-region"; + protected static final String CLOUD_OWNER = "cloud-owner"; + protected static final String CLOUD_REGION_ID = "cloud-region-id"; + protected static final String VLAN_RANGE = "vlan-range"; + protected static final String VLAN_RANGE_ID = "vlan-range-id"; + protected static final String VLAN_TAG = "vlan-tag"; + protected static final String VLAN_TAG_ID = "vlan-tag-id"; + protected static final String UPGRADE_CYCLE = "upgrade-cycle"; + + protected final AtomicInteger skippedRowsCount = new AtomicInteger(0); + protected final AtomicInteger processedRowsCount = new AtomicInteger(0); + private static List<String> dmaapMsgList = new ArrayList<String>(); + private static List<Introspector> dmaapDeleteIntrospectorList = new ArrayList<Introspector>(); + private static int vlanRangeCount = 0; + private static int vlanRangeFailureCount = 0; + private static int vlanTagCount = 0; + private static int vlanTagFailureCount = 0; + + private boolean success = true; + protected int headerLength; + + protected final AtomicInteger falloutRowsCount = new AtomicInteger(0); + + public MigrateVlanTag(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, + EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions); + } + + @Override + public void run() { + logger.info("---------- Start Updating vlan-tags under cloud-region ----------"); + + String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs"; + String feedDir = logDir + AAIConstants.AAI_FILESEP + "data" + + AAIConstants.AAI_FILESEP + "migration-input-files" + + AAIConstants.AAI_FILESEP; + String fileName = feedDir + "dslResults.json"; + logger.info(fileName); + logger.info("---------- Processing vlan-tags from file ----------"); + + Map cloudOwnerRegionIdWiseVertexMap = new HashMap(); + Map<String, Vertex> vlanTagIdWiseVertexMap = new HashMap<>(); + + try { + List<Vertex> cloudRegionList = this.engine.asAdmin().getTraversalSource().V() + .has(AAIProperties.NODE_TYPE, CLOUD_REGION_NODE_TYPE).toList(); + + List<Vertex> vlantagIdList = this.engine.asAdmin().getTraversalSource().V() + .has(AAIProperties.NODE_TYPE, VLAN_TAG).toList(); + + String cloudOwner, cloudRegionId; + for (Vertex vertex : cloudRegionList) { + cloudOwner = getCloudOwnerNodeValue(vertex); + cloudRegionId = getCloudRegionIdNodeValue(vertex); + if (!cloudOwner.isEmpty() && !cloudRegionId.isEmpty()) { + cloudOwnerRegionIdWiseVertexMap.put(cloudOwner + "/" + cloudRegionId, vertex); + } + } + + String vlantagId; + for (Vertex vertex : vlantagIdList) { + vlantagId = getVlanTagIdNodeValue(vertex); + if (!vlantagId.isEmpty() && + !(vertex.property(AAIProperties.AAI_URI).isPresent() && + vertex.property(AAIProperties.AAI_URI).toString().contains(CLOUD_REGION_NODE_TYPE))) { + vlanTagIdWiseVertexMap.put(vlantagId, vertex); + } + } + + JSONParser jsonParser = new JSONParser(); + Object obj = jsonParser + .parse(new FileReader(fileName)); + JSONObject vlanRangeResultJsonObj = (JSONObject) obj; + JSONArray vlanRangeResultJsonArray = (JSONArray) vlanRangeResultJsonObj.get("results"); + + JSONObject vlanRangeJsonObject, vlanTagsJsonObject, vlanTagJsonObject; + JSONArray vlanTagJsonArray; + String vlanTagId; + boolean isFirstTime; + Vertex vlanRangeVtx; + Set<String> matchedVlanTagIdSet = new HashSet<String>(); + for (int i = 0; i < vlanRangeResultJsonArray.size(); i++) { + isFirstTime = true; + vlanRangeVtx = null; + vlanRangeJsonObject = (JSONObject) vlanRangeResultJsonArray.get(i); + cloudOwner = getCloudOwnerValueFromUrl((String) vlanRangeJsonObject.get("url")); + cloudRegionId = getCloudRegionIdValueFromUrl((String) vlanRangeJsonObject.get("url")); + if (cloudOwnerRegionIdWiseVertexMap.containsKey(cloudOwner + "/" + cloudRegionId)) {// ap1 key contains + // api2 key + JSONObject vlanRangeInnerObject = (JSONObject) vlanRangeJsonObject.get("vlan-range"); + String vlanRangeId = (String) vlanRangeInnerObject.get("vlan-range-id"); + vlanTagsJsonObject = (JSONObject) vlanRangeInnerObject.get("vlan-tags"); + vlanTagJsonArray = (JSONArray) vlanTagsJsonObject.get("vlan-tag"); + + for (int j = 0; j < vlanTagJsonArray.size(); j++) { + vlanTagJsonObject = (JSONObject) vlanTagJsonArray.get(j); + vlanTagId = (String) vlanTagJsonObject.get("vlan-tag-id"); + if (vlanTagIdWiseVertexMap.containsKey(vlanTagId)) {// ap1 key contains api2 key + matchedVlanTagIdSet.add(vlanTagId); + if (isFirstTime) { + isFirstTime = false; + vlanRangeVtx = createNewVlanRangeFromCloudRegion(vlanRangeInnerObject, cloudOwner, + cloudRegionId, vlanRangeId); + } + Vertex vertex = vlanTagIdWiseVertexMap.get(vlanTagId); + createNewVlanTagFromVlanRange(vlanTagJsonObject, vlanRangeVtx, cloudOwner, cloudRegionId, + vlanRangeId, vertex); + vlanTagIdWiseVertexMap.remove(vlanTagId); + } + } + } + } + logger.info("******* Final Summary of Migrate vlan-tag Migration *********"); + if(!vlanTagIdWiseVertexMap.isEmpty()) { + logger.info("The following vlan-ids in A&AI graph were not found in the Narad input" + + "file and not migrated: "); + for(Vertex vlanTagIdWiseVertex : vlanTagIdWiseVertexMap.values()) { + logger.info(vlanTagIdWiseVertex.property("vlan-tag-id").toString()); + } + } + logger.info(MIGRATION_SUMMARY_COUNT+"Total Vlan Ranges Updated Count:" + vlanRangeCount); + logger.info(MIGRATION_ERROR+"Total Vlan Ranges Not Updated Count:" + vlanRangeFailureCount); + logger.info(MIGRATION_SUMMARY_COUNT+"Total Vlan Tags Updated Sucessfully Count: " + vlanTagCount); + logger.info(MIGRATION_ERROR+"Total Vlan Tags Not Updated Count: " + vlanTagFailureCount); + } catch (FileNotFoundException e) { + logger.info("ERROR: Could not file file " + fileName, e.getMessage()); + logger.error("ERROR: Could not file file " + fileName, e); + success = false; + } catch (IOException e) { + logger.info("ERROR: Issue reading file " + fileName, e.getMessage()); + logger.error("ERROR: Issue reading file " + fileName, e); + success = false; + } catch (Exception e) { + logger.info("encountered exception", e.getMessage()); + logger.error("encountered exception", e); + e.printStackTrace(); + success = false; + } + } + + private Vertex createNewVlanRangeFromCloudRegion(JSONObject vlanRangeJsonObject, String cloudOwner, + String cloudRegionId, String vlanRangeId) { + + Vertex vlanRangeVtx = null; + try { + + GraphTraversal<Vertex, Vertex> cloudRegionVtxList = this.engine.asAdmin().getTraversalSource().V() + .has(AAIProperties.NODE_TYPE, CLOUD_REGION_NODE_TYPE).has("cloud-owner", cloudOwner) + .has("cloud-region-id", cloudRegionId); + + if (cloudRegionVtxList != null && cloudRegionVtxList.hasNext()) { + Vertex cloudRegionVtx = cloudRegionVtxList.next(); + if (cloudRegionVtx != null) { + Introspector vlanRange = loader.introspectorFromName("vlan-range"); + vlanRangeVtx = serializer.createNewVertex(vlanRange); + vlanRange.setValue("vlan-range-id", vlanRangeId);// required + vlanRange.setValue("vlan-id-lower", (Long) vlanRangeJsonObject.get("vlan-id-lower"));// required + vlanRange.setValue("vlan-id-upper", (Long) vlanRangeJsonObject.get("vlan-id-upper"));// required + vlanRange.setValue("vlan-type", (String) vlanRangeJsonObject.get("vlan-type"));// required + this.createTreeEdge(cloudRegionVtx, vlanRangeVtx); + vlanRangeVtx.property(AAIProperties.AAI_URI, + "/cloud-infrastructure/" + "cloud-regions/cloud-region/" + cloudOwner + "/" + cloudRegionId + + "/vlan-ranges/vlan-range/" + vlanRangeId); + serializer.serializeSingleVertex(vlanRangeVtx, vlanRange, "migrations"); + + logger.info("\t Created new vlan-range " + vlanRangeVtx + " with vlan-range-id = " + + (String) vlanRangeJsonObject.get("vlan-range-id")); + + String dmaapMsg = System.nanoTime() + "_" + vlanRangeVtx.id().toString() + "_" + + vlanRangeVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + vlanRangeCount++; + } + } + + } catch (Exception e) { + vlanRangeFailureCount++; + logger.error("vlan-range failure: ", e); + logger.info("vlan-range with id : " + vlanRangeId + " failed: ", e.getMessage()); + + } + return vlanRangeVtx; + + } + + private Vertex createNewVlanTagFromVlanRange(JSONObject vlanTagJsonObject, Vertex vlanRangeVtx, String cloudOwner, + String cloudRegionId, String vlanRangeId, Vertex oldVlanTagVtx) { + + Vertex vlanTagVtx = null; + try { + Introspector vlanTag = loader.introspectorFromName("vlan-tag"); + vlanTagVtx = serializer.createNewVertex(vlanTag); + String vlanTagId = (String) vlanTagJsonObject.get("vlan-tag-id"); + vlanTag.setValue("vlan-tag-id", vlanTagId);// required + vlanTag.setValue("vlan-tag-role", (String) vlanTagJsonObject.get("vlan-tag-role"));// required + vlanTag.setValue("is-private", (Boolean) vlanTagJsonObject.get("is-private"));// required + if (vlanTagJsonObject.containsKey("vlan-id-inner")) + vlanTag.setValue("vlan-id-inner", (Long) vlanTagJsonObject.get("vlan-id-inner")); + if (vlanTagJsonObject.containsKey("vlan-id-outer")) + vlanTag.setValue("vlan-id-outer", (Long) vlanTagJsonObject.get("vlan-id-outer")); + if (vlanTagJsonObject.containsKey("vlan-tag-function")) + vlanTag.setValue("vlan-tag-function", (String) vlanTagJsonObject.get("vlan-tag-function")); + if (vlanTagJsonObject.containsKey("config-phase")) + vlanTag.setValue("config-phase", (String) vlanTagJsonObject.get("config-phase")); + if (vlanTagJsonObject.containsKey("vlan-tag-type")) + vlanTag.setValue("vlan-tag-type", (String) vlanTagJsonObject.get("vlan-tag-type")); + this.createTreeEdge(vlanRangeVtx, vlanTagVtx); + vlanTagVtx.property(AAIProperties.AAI_URI, + "/cloud-infrastructure/" + "cloud-regions/cloud-region/" + cloudOwner + "/" + cloudRegionId + + "/vlan-ranges/vlan-range/" + vlanRangeId + "/vlan-tags/vlan-tag/" + vlanTagId); + executeModifyOperation(oldVlanTagVtx, vlanTagVtx); + Introspector deletedVlanEvent = serializer.getLatestVersionView(oldVlanTagVtx); + dmaapDeleteIntrospectorList.add(deletedVlanEvent); + oldVlanTagVtx.remove(); + serializer.serializeSingleVertex(vlanTagVtx, vlanTag, "migrations"); + + logger.info("\t Created new vlan-tag " + vlanTagVtx + " with vlan-tag-id = " + + (String) vlanTagJsonObject.get("vlan-tag-id")); + + String dmaapMsg = System.nanoTime() + "_" + vlanTagVtx.id().toString() + "_" + + vlanRangeVtx.value("resource-version").toString(); + dmaapMsgList.add(dmaapMsg); + vlanTagCount++; + } catch (Exception e) { + vlanTagFailureCount++; + logger.error("vlan-tag failure: ", e); + if(vlanTagJsonObject != null && vlanTagJsonObject.get("vlan-tag-id") != null){ + logger.info("vlan-tag with id : " + vlanTagJsonObject.get("vlan-tag-id") + " failed: ", e.getMessage()); + } + else { + logger.info("vlan-tag failure: ", e.getMessage()); + } + } + return vlanRangeVtx; + + } + + private String getCloudRegionIdNodeValue(Vertex vertex) { + String propertyValue = ""; + if (vertex != null && vertex.property(CLOUD_REGION_ID).isPresent()) { + propertyValue = vertex.property(CLOUD_REGION_ID).value().toString(); + } + return propertyValue; + } + + private String getCloudOwnerNodeValue(Vertex vertex) { + String propertyValue = ""; + if (vertex != null && vertex.property(CLOUD_OWNER).isPresent()) { + propertyValue = vertex.property(CLOUD_OWNER).value().toString(); + } + return propertyValue; + } + + private String getVlanTagIdNodeValue(Vertex vertex) { + String propertyValue = ""; + if (vertex != null && vertex.property(VLAN_TAG_ID).isPresent()) { + propertyValue = vertex.property(VLAN_TAG_ID).value().toString(); + } + return propertyValue; + } + + private String getCloudOwnerValueFromUrl(String url) { + String arr[] = url.split("/"); + return arr[6]; + } + + private String getCloudRegionIdValueFromUrl(String url) { + String arr[] = url.split("/"); + return arr[7]; + } + + protected void executeModifyOperation(Vertex fromNode, Vertex toNode) { + try { + + this.swingEdges(fromNode, toNode, "l3-network", "none", "BOTH"); + this.swingEdges(fromNode, toNode, "cp", "none", "BOTH"); + } catch (Exception e) { + logger.error("error encountered", e); + success = false; + } + } + + @Override + public void commit() { + engine.commit(); + createDmaapFiles(dmaapMsgList); + createDmaapFilesForDelete(dmaapDeleteIntrospectorList); + } + + @Override + public Status getStatus() { + if (success) { + return Status.SUCCESS; + } else { + return Status.FAILURE; + } + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[] { CLOUD_REGION_NODE_TYPE }); + } + + @Override + public String getMigrationName() { + return "MigrateVlanTag"; + } + + @Override + public List<Pair<Vertex, Vertex>> getAffectedNodePairs() { + return null; + } + + @Override + public String getNodeTypeRestriction() { + return null; + } + + @Override + public String getEdgeLabelRestriction() { + return null; + } + + @Override + public String getEdgeDirRestriction() { + return null; + } + + @Override + public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) { + } + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/rest/AuditSqlDbConsumer.java b/src/main/java/org/onap/aai/rest/AuditSqlDbConsumer.java new file mode 100644 index 0000000..aec7224 --- /dev/null +++ b/src/main/java/org/onap/aai/rest/AuditSqlDbConsumer.java @@ -0,0 +1,120 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.rest; + +import org.onap.aai.audit.AuditGraphson2Sql; +import org.onap.aai.concurrent.AaiCallable; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.restcore.HttpMethod; +import org.onap.aai.restcore.RESTAPI; +import org.onap.aai.util.AAIConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.ws.rs.*; +import javax.ws.rs.core.*; +import javax.ws.rs.core.Response.Status; + +@Component +@Path("{version: v1}/audit-sql-db") +public class AuditSqlDbConsumer extends RESTAPI { + + private static String SOURCE_FILE_NAME = "sourceFileName"; + private static String SOURCE_FILE_DIR_DEFAULT = "logs/data/dataSnapshots/"; + + private static final Logger LOGGER = LoggerFactory.getLogger(AuditSqlDbConsumer.class); + + private String rdbmsDbName; + private AuditGraphson2Sql auditGraphson2Sql; + + @Autowired + public AuditSqlDbConsumer( + AuditGraphson2Sql auditGraphson2Sql, + @Value("${aperture.rdbmsname}") String rdbmsDbName + ){ + this.auditGraphson2Sql = auditGraphson2Sql; + this.rdbmsDbName = rdbmsDbName; + } + + + @GET + @Consumes({ MediaType.APPLICATION_JSON}) + @Produces({ MediaType.APPLICATION_JSON}) + public Response executeAudit(String content, + @PathParam("uri") @Encoded String uri, + @Context HttpHeaders headers, + @Context UriInfo info){ + return runner(AAIConstants.AAI_GRAPHADMIN_TIMEOUT_ENABLED, + AAIConstants.AAI_GRAPHADMIN_TIMEOUT_APP, + AAIConstants.AAI_GRAPHADMIN_TIMEOUT_LIMIT, + headers, + info, + HttpMethod.GET, + new AaiCallable<Response>() { + @Override + public Response process() { + return processExecuteAudit(content, uri, headers, info); + } + } + ); + } + + + public Response processExecuteAudit(String content, + @PathParam("uri") @Encoded String uri, + @Context HttpHeaders headers, + @Context UriInfo info) { + + Response response = null; + try { + this.checkParams(info.getQueryParameters()); + + String resStr = auditGraphson2Sql.runAudit( rdbmsDbName, + info.getQueryParameters().getFirst(SOURCE_FILE_NAME), + SOURCE_FILE_DIR_DEFAULT ); + + LOGGER.info ("Completed"); + + response = Response.status(Status.OK) + .type(MediaType.APPLICATION_JSON) + .entity(resStr).build(); + + } catch (AAIException e) { + response = consumerExceptionResponseGenerator(headers, info, HttpMethod.GET, e); + } catch (Exception e ) { + AAIException ex = new AAIException("AAI_4000", e); + response = consumerExceptionResponseGenerator(headers, info, HttpMethod.GET, ex); + } + + return response; + } + + + public void checkParams(MultivaluedMap<String, String> params) throws AAIException { + + if (!params.containsKey(SOURCE_FILE_NAME)) { + throw new AAIException("AAI_6120", "parameter: sourceFileName (of snapshot file) is required. "); + } + } + +} diff --git a/src/main/java/org/onap/aai/rest/client/ApertureConfiguration.java b/src/main/java/org/onap/aai/rest/client/ApertureConfiguration.java new file mode 100644 index 0000000..34fdbec --- /dev/null +++ b/src/main/java/org/onap/aai/rest/client/ApertureConfiguration.java @@ -0,0 +1,49 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.rest.client; + +import org.onap.aai.restclient.RestClient; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; + + +@Configuration +public class ApertureConfiguration { + + @Bean(name = "apertureRestClient") + @ConditionalOnProperty(name = "aperture.service.client", havingValue = "two-way-ssl", matchIfMissing = true) + public RestClient apertureRestClientTwoWaySSL() { + return new ApertureServiceRestClient(); + } + + @Bean(name = "apertureRestClient") + @ConditionalOnProperty(name = "aperture.service.client", havingValue = "no-auth") + public RestClient apertureRestClientNoAuth() { + return new ApertureServiceNoAuthClient(); + } + + @Bean(name = "apertureRestClient") + @ConditionalOnProperty(name = "aperture.service.client", havingValue = "one-way-ssl") + public RestClient apertureRestClientOneWaySSL() { + return new ApertureServiceOneWayClient(); + } +} diff --git a/src/main/java/org/onap/aai/rest/client/ApertureService.java b/src/main/java/org/onap/aai/rest/client/ApertureService.java new file mode 100644 index 0000000..493a4cd --- /dev/null +++ b/src/main/java/org/onap/aai/rest/client/ApertureService.java @@ -0,0 +1,142 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.rest.client; + + +import com.google.gson.JsonObject; +import org.apache.http.conn.ConnectTimeoutException; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.restclient.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.util.*; + +@Service +public class ApertureService { + + /** + * Error indicating that the service trying to connect is down + */ + static final String CONNECTION_REFUSED_STRING = + "Connection refused to the Aperture microservice due to service unreachable"; + + /** + * Error indicating that the server is unable to reach the port + * Could be server related connectivity issue + */ + static final String CONNECTION_TIMEOUT_STRING = + "Connection timeout to the Aperture microservice as this could " + + "indicate the server is unable to reach port, " + + "please check on server by running: nc -w10 -z -v ${APERTURE_HOST} ${APERTURE_PORT}"; + + /** + * Error indicating that the request exceeded the allowed time + * + * Note: This means that the service could be active its + * just taking some time to process our request + */ + static final String REQUEST_TIMEOUT_STRING = + "Request to Aperture service took longer than the currently set timeout"; + + static final String APERTURE_ENDPOINT = "/v1/audit/"; + + private static final Logger LOGGER = LoggerFactory.getLogger(ApertureService.class); + private final RestClient apertureRestClient; + private final String appName; + + + @Autowired + public ApertureService( + @Qualifier("apertureRestClient") RestClient apertureRestClient, + @Value("${spring.application.name}") String appName + ){ + this.apertureRestClient = apertureRestClient; + this.appName = appName; + + LOGGER.info("Successfully initialized the aperture service"); + } + + + public JsonObject runAudit(Long tStamp, String dbName) throws AAIException { + + Map<String, String> httpHeaders = new HashMap<>(); + + httpHeaders.put("X-FromAppId", appName); + httpHeaders.put("X-TransactionID", UUID.randomUUID().toString()); + httpHeaders.put("Accept", "application/json"); + + String queryParams = "?timestamp=" + tStamp + "&dbname=" + dbName; + + ResponseEntity responseEntity = null; + try { + responseEntity = apertureRestClient.execute( + APERTURE_ENDPOINT + queryParams, + HttpMethod.GET, + httpHeaders + ); + + if(isSuccess(responseEntity)){ + LOGGER.debug("Audit returned following response status code {} and body {}", responseEntity.getStatusCodeValue(), responseEntity.getBody()); + } + + } catch(Exception e){ + // If the exception cause is client side timeout + // then proceed as if it passed validation + // resources microservice shouldn't be blocked because of validation service + // is taking too long or if the validation service is down + // Any other exception it should block the request from passing? + if(e.getCause() instanceof SocketTimeoutException){ + LOGGER.error(REQUEST_TIMEOUT_STRING, e.getCause()); + } else if(e.getCause() instanceof ConnectException){ + LOGGER.error(CONNECTION_REFUSED_STRING, e.getCause()); + } else if(e.getCause() instanceof ConnectTimeoutException){ + LOGGER.error(CONNECTION_TIMEOUT_STRING, e.getCause()); + } else { + LOGGER.error("Unknown exception thrown please investigate", e.getCause()); + } + } + + JsonObject jsonResults = new JsonObject (); + if( responseEntity != null ) { + jsonResults = (JsonObject) (responseEntity.getBody()); + } + + return jsonResults; + } + + + + boolean isSuccess(ResponseEntity responseEntity){ + return responseEntity != null && responseEntity.getStatusCode().is2xxSuccessful(); + } + + + +} diff --git a/src/main/java/org/onap/aai/rest/client/ApertureServiceNoAuthClient.java b/src/main/java/org/onap/aai/rest/client/ApertureServiceNoAuthClient.java new file mode 100644 index 0000000..778a821 --- /dev/null +++ b/src/main/java/org/onap/aai/rest/client/ApertureServiceNoAuthClient.java @@ -0,0 +1,79 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.rest.client; + +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; +import org.onap.aai.restclient.NoAuthRestClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.util.MultiValueMap; + +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + +public class ApertureServiceNoAuthClient extends NoAuthRestClient { + + private static EELFLogger logger = EELFManager.getInstance().getLogger(ApertureServiceNoAuthClient.class); + + @Value("${aperture.service.base.url}") + private String baseUrl; + + @Value("${aperture.service.timeout-in-milliseconds}") + private Integer timeout; + + @Override + protected HttpComponentsClientHttpRequestFactory getHttpRequestFactory() throws Exception { + HttpComponentsClientHttpRequestFactory requestFactory = super.getHttpRequestFactory(); + requestFactory.setConnectionRequestTimeout(timeout); + requestFactory.setReadTimeout(timeout); + requestFactory.setConnectTimeout(timeout); + return requestFactory; + } + + @Override + public String getBaseUrl() { + return baseUrl; + } + + @Override + public MultiValueMap<String, String> getHeaders(Map<String, String> headers) { + HttpHeaders httpHeaders = new HttpHeaders(); + + String defaultAccept = headers.getOrDefault(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON.toString()); + String defaultContentType = + headers.getOrDefault(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON.toString()); + + if (headers.isEmpty()) { + httpHeaders.setAccept(Collections.singletonList(MediaType.parseMediaType(defaultAccept))); + httpHeaders.setContentType(MediaType.parseMediaType(defaultContentType)); + } + + httpHeaders.add("X-FromAppId", appName); + httpHeaders.add("X-TransactionId", UUID.randomUUID().toString()); + headers.forEach(httpHeaders::add); + return httpHeaders; + } + +} diff --git a/src/main/java/org/onap/aai/rest/client/ApertureServiceOneWayClient.java b/src/main/java/org/onap/aai/rest/client/ApertureServiceOneWayClient.java new file mode 100644 index 0000000..a888536 --- /dev/null +++ b/src/main/java/org/onap/aai/rest/client/ApertureServiceOneWayClient.java @@ -0,0 +1,92 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.rest.client; + +import org.onap.aai.restclient.OneWaySSLRestClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.util.MultiValueMap; + +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + +public class ApertureServiceOneWayClient extends OneWaySSLRestClient { + + @Value("${aperture.service.base.url}") + private String baseUrl; + + @Value("${aperture.service.ssl.trust-store}") + private String truststorePath; + + @Value("${aperture.service.ssl.trust-store-password}") + private String truststorePassword; + + @Value("${aperture.service.timeout-in-milliseconds}") + private Integer timeout; + + @Override + public String getBaseUrl() { + return baseUrl; + } + + @Override + protected String getTruststorePath() { + return truststorePath; + } + + @Override + protected char[] getTruststorePassword() { + return truststorePassword.toCharArray(); + } + + @Override + protected HttpComponentsClientHttpRequestFactory getHttpRequestFactory() throws Exception { + HttpComponentsClientHttpRequestFactory requestFactory = super.getHttpRequestFactory(); + requestFactory.setConnectionRequestTimeout(timeout); + requestFactory.setReadTimeout(timeout); + requestFactory.setConnectTimeout(timeout); + return requestFactory; + } + + @Override + public MultiValueMap<String, String> getHeaders(Map<String, String> headers) { + HttpHeaders httpHeaders = new HttpHeaders(); + + String defaultAccept = headers.getOrDefault(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON.toString()); + String defaultContentType = + headers.getOrDefault(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON.toString()); + + if (headers.isEmpty()) { + httpHeaders.setAccept(Collections.singletonList(MediaType.parseMediaType(defaultAccept))); + httpHeaders.setContentType(MediaType.parseMediaType(defaultContentType)); + } + + httpHeaders.add("X-FromAppId", appName); + httpHeaders.add("X-TransactionId", UUID.randomUUID().toString()); + httpHeaders.add("X-TransactionId", appName); + headers.forEach(httpHeaders::add); + return httpHeaders; + } + +} diff --git a/src/main/java/org/onap/aai/rest/client/ApertureServiceRestClient.java b/src/main/java/org/onap/aai/rest/client/ApertureServiceRestClient.java new file mode 100644 index 0000000..da32fbe --- /dev/null +++ b/src/main/java/org/onap/aai/rest/client/ApertureServiceRestClient.java @@ -0,0 +1,106 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.rest.client; + +import org.onap.aai.restclient.TwoWaySSLRestClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.util.MultiValueMap; + +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + +public class ApertureServiceRestClient extends TwoWaySSLRestClient { + + @Value("${aperture.service.base.url}") + private String baseUrl; + + @Value("${aperture.service.ssl.key-store}") + private String keystorePath; + + @Value("${aperture.service.ssl.trust-store}") + private String truststorePath; + + @Value("${aperture.service.ssl.key-store-password}") + private String keystorePassword; + + @Value("${aperture.service.ssl.trust-store-password}") + private String truststorePassword; + + @Value("${aperture.service.timeout-in-milliseconds}") + private Integer timeout; + + @Override + public String getBaseUrl() { + return baseUrl; + } + + @Override + protected String getKeystorePath() { + return keystorePath; + } + + @Override + protected String getTruststorePath() { + return truststorePath; + } + + @Override + protected char[] getKeystorePassword() { + return keystorePassword.toCharArray(); + } + + @Override + protected char[] getTruststorePassword() { + return truststorePassword.toCharArray(); + } + + protected HttpComponentsClientHttpRequestFactory getHttpRequestFactory() throws Exception { + HttpComponentsClientHttpRequestFactory requestFactory = super.getHttpRequestFactory(); + requestFactory.setConnectionRequestTimeout(timeout); + requestFactory.setReadTimeout(timeout); + requestFactory.setConnectTimeout(timeout); + return requestFactory; + } + + @Override + public MultiValueMap<String, String> getHeaders(Map<String, String> headers) { + HttpHeaders httpHeaders = new HttpHeaders(); + + String defaultAccept = headers.getOrDefault(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON.toString()); + String defaultContentType = + headers.getOrDefault(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON.toString()); + + if (headers.isEmpty()) { + httpHeaders.setAccept(Collections.singletonList(MediaType.parseMediaType(defaultAccept))); + httpHeaders.setContentType(MediaType.parseMediaType(defaultContentType)); + } + + httpHeaders.add("X-FromAppId", appName); + httpHeaders.add("X-TransactionId", UUID.randomUUID().toString()); + headers.forEach(httpHeaders::add); + return httpHeaders; + } + +} diff --git a/src/main/java/org/onap/aai/schema/GenTester.java b/src/main/java/org/onap/aai/schema/GenTester.java index 84ecf19..160ec22 100644 --- a/src/main/java/org/onap/aai/schema/GenTester.java +++ b/src/main/java/org/onap/aai/schema/GenTester.java @@ -24,7 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.schema.JanusGraphManagement; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbgen.SchemaGenerator; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.exceptions.AAIException; diff --git a/src/main/java/org/onap/aai/schema/GenTester4Hist.java b/src/main/java/org/onap/aai/schema/GenTester4Hist.java index eefb7b0..05478e2 100644 --- a/src/main/java/org/onap/aai/schema/GenTester4Hist.java +++ b/src/main/java/org/onap/aai/schema/GenTester4Hist.java @@ -24,7 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.schema.JanusGraphManagement; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbgen.SchemaGenerator4Hist; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.exceptions.AAIException; diff --git a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java index 0bd254f..4297886 100644 --- a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java +++ b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java @@ -21,7 +21,7 @@ package org.onap.aai.util; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; diff --git a/src/main/java/org/onap/aai/util/SendMigrationNotifications.java b/src/main/java/org/onap/aai/util/SendMigrationNotifications.java index dfe2649..c51de1e 100644 --- a/src/main/java/org/onap/aai/util/SendMigrationNotifications.java +++ b/src/main/java/org/onap/aai/util/SendMigrationNotifications.java @@ -54,8 +54,8 @@ public class SendMigrationNotifications { private String config; private String path; private Set<String> notifyOn; - private long sleepInMilliSecs; - private int numToBatch; + long sleepInMilliSecs; + int numToBatch; private String requestId; private EventAction eventAction; private String eventSource; @@ -108,22 +108,25 @@ public class SendMigrationNotifications { vertexes = g.V(entry.getKey()).toList(); if (vertexes == null || vertexes.isEmpty()) { logAndPrint("Vertex " + entry.getKey() + " no longer exists." ); + continue; } else if (vertexes.size() > 1) { logAndPrint("Vertex " + entry.getKey() + " query returned " + vertexes.size() + " vertexes." ); + continue; } else { logger.debug("Processing " + entry.getKey() + "resource-version " + entry.getValue()); v = vertexes.get(0); - if ((notifyOn.isEmpty() || notifyOn.contains(v.value(AAIProperties.NODE_TYPE).toString())) - && entry.getValue().equals(v.value(AAIProperties.RESOURCE_VERSION).toString())) { - Introspector introspector = serializer.getLatestVersionView(v); - uri = this.serializer.getURIForVertex(v, false); - this.notificationHelper.addEvent(v, introspector, eventAction, uri, basePath); - count++; - if (count >= this.numToBatch) { - trigger(); - logger.debug("Triggered " + entry.getKey()); - count = 0; - Thread.sleep(this.sleepInMilliSecs); + if (notifyOn.isEmpty() || notifyOn.contains(v.value(AAIProperties.NODE_TYPE).toString())) { + if (entry.getValue().equals(v.value(AAIProperties.RESOURCE_VERSION).toString())) { + Introspector introspector = serializer.getLatestVersionView(v); + uri = this.serializer.getURIForVertex(v, false); + this.notificationHelper.addEvent(v, introspector, eventAction, uri, basePath); + count++; + if (count >= this.numToBatch) { + trigger(); + logger.debug("Triggered " + entry.getKey()); + count = 0; + Thread.sleep(this.sleepInMilliSecs); + } } } } diff --git a/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java b/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java index d3670f2..0eeaecd 100644 --- a/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java +++ b/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java @@ -21,7 +21,7 @@ package org.onap.aai.util; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.restclient.PropertyPasswordConfiguration; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; diff --git a/src/main/java/org/onap/aai/web/JerseyConfiguration.java b/src/main/java/org/onap/aai/web/JerseyConfiguration.java index 629d11e..f193b8c 100644 --- a/src/main/java/org/onap/aai/web/JerseyConfiguration.java +++ b/src/main/java/org/onap/aai/web/JerseyConfiguration.java @@ -32,6 +32,7 @@ import org.apache.tinkerpop.gremlin.structure.T; import org.glassfish.jersey.filter.LoggingFilter; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.servlet.ServletProperties; +import org.onap.aai.rest.AuditSqlDbConsumer; import org.onap.aai.rest.QueryConsumer; import org.onap.aai.rest.util.EchoResponse; import org.onap.logging.filter.base.AuditLogContainerFilter; @@ -54,7 +55,7 @@ public class JerseyConfiguration extends ResourceConfig { this.env = env; register(QueryConsumer.class); - + register(AuditSqlDbConsumer.class); register(EchoResponse.class); //Filters diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 698a6da..82780f7 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -61,7 +61,7 @@ schema.version.namespace.change.start=v12 # Specifies from which version should the client start seeing the edge label in payload schema.version.edge.label.start=v12 # Specifies the version that the application should default to -schema.version.api.default=v16 +schema.version.api.default=v19 schema.translator.list=config #schema.service.client=no-auth schema.service.base.url=https://localhost:8452/aai/schema-service/v1/ @@ -74,4 +74,14 @@ schema.service.ssl.key-store-password=password(OBF:1vn21ugu1saj1v9i1v941sar1ugw1 schema.service.ssl.trust-store=${server.local.startpath}etc/auth/aai_keystore schema.service.ssl.trust-store-password=password(OBF:1vn21ugu1saj1v9i1v941sar1ugw1vo0) -aaf.cadi.file=${server.local.startpath}/cadi.properties
\ No newline at end of file +aaf.cadi.file=${server.local.startpath}/cadi.properties + +aperture.rdbmsname=aai_relational + +aperture.service.client=no-auth +aperture.service.base.url=http://localhost:8457/aai/aperture +aperture.service.ssl.key-store=${server.local.startpath}etc/auth/aai_keystore +aperture.service.ssl.trust-store=${server.local.startpath}etc/auth/aai_keystore +aperture.service.ssl.key-store-password=password(OBF:1vn21ugu1saj1v9i1v941sar1ugw1vo0) +aperture.service.ssl.trust-store-password=password(OBF:1vn21ugu1saj1v9i1v941sar1ugw1vo0) +aperture.service.timeout-in-milliseconds=300000 diff --git a/src/main/resources/etc/appprops/aaiconfig.properties b/src/main/resources/etc/appprops/aaiconfig.properties index ae5084f..b2dd2ed 100644 --- a/src/main/resources/etc/appprops/aaiconfig.properties +++ b/src/main/resources/etc/appprops/aaiconfig.properties @@ -99,6 +99,7 @@ aai.datagrooming.agezip=5 aai.datagrooming.agedelete=30 aai.datasnapshot.agezip=5 aai.datasnapshot.agedelete=30 +aai.datamigration.agedelete=30 #used by the dataSnapshot and dataGrooming tasks aai.cron.enable.dataSnapshot=true diff --git a/src/main/resources/etc/appprops/datatoolscrons.properties b/src/main/resources/etc/appprops/datatoolscrons.properties index 74b3c9e..e37931f 100644 --- a/src/main/resources/etc/appprops/datatoolscrons.properties +++ b/src/main/resources/etc/appprops/datatoolscrons.properties @@ -8,5 +8,4 @@ datagroomingcleanup.cron=0 06 0 * * ? datasnapshotcleanup.cron=0 17 0 * * ? datasnapshottasks.cron=0 45 * * * ? -datagroomingtasks.cron=0 10 1,5,9,13,17,21 * * ? -dataexporttask.cron=0 02 3 * * ?
\ No newline at end of file +datagroomingtasks.cron=0 10 1,5,9,13,17,21 * * ?
\ No newline at end of file diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index eff961f..7561f3f 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -184,6 +184,23 @@ <appender-ref ref="translog" /> </appender> + <appender name="failover" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/failover/failover.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/failover/failover.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{1024} - %msg%n + </pattern> + </encoder> + </appender> + + <appender name="asyncFailover" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>1000</queueSize> + <includeCallerData>true</includeCallerData> + <appender-ref ref="failover" /> + </appender> + <appender name="dmaapAAIEventConsumer" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> @@ -195,7 +212,7 @@ </fileNamePattern> </rollingPolicy> <encoder> - <pattern>${"errorPattern"}</pattern> + <pattern>${errorPattern}</pattern> </encoder> </appender> <appender name="dmaapAAIEventConsumerInfo" @@ -410,7 +427,7 @@ <fileNamePattern>${logDirectory}/createDBSchema/error.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> <encoder> - <pattern>${"errorPattern"}</pattern> + <pattern>${errorPattern}</pattern> </encoder> </appender> @@ -455,7 +472,7 @@ <fileNamePattern>${logDirectory}/misc/error.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> <encoder> - <pattern>${"errorPattern"}</pattern> + <pattern>${errorPattern}</pattern> </encoder> </appender> @@ -497,9 +514,9 @@ <onMatch>ACCEPT</onMatch> <onMismatch>DENY</onMismatch> </filter> - <File>${logDirectory}/dupetool/debug.log</File> + <File>${logDirectory}/dupeTool/debug.log</File> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/dupetool/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + <fileNamePattern>${logDirectory}/dupeTool/debug.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> <encoder> <pattern>${debugPattern}</pattern> @@ -600,7 +617,7 @@ </appender> <!-- forceDelete logs ended --> - <!-- migration logs started --> + <!-- default migration logs started --> <appender name="migrationdebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>DEBUG</level> @@ -630,7 +647,38 @@ <pattern>${errorPattern}</pattern> </encoder> </appender> - <!-- migration logs ended --> + <!-- default migration logs ended --> + + <!-- other migration logs started --> + <appender name="migrationlog" class="ch.qos.logback.classic.sift.SiftingAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <!-- This is MDC value --> + <!-- We will assign a value to 'logFilenameAppender' via Java code --> + <discriminator> + <key>logFilenameAppender</key> + <defaultValue>undefined</defaultValue> + </discriminator> + <sift> + <!-- A standard RollingFileAppender, the log file is based on 'logFileName' + at runtime --> + <appender name="FILE-${logFilenameAppender}" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/migration/${logFilenameAppender}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/migration/${logFilenameAppender}.log.%d{yyyy-MM-dd} + </fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%m%n</pattern> + </encoder> + </appender> + </sift> + </appender> + <!-- other migration logs ended --> <!-- DataGrooming logs started --> <appender name="dataExportError" class="ch.qos.logback.core.rolling.RollingFileAppender"> @@ -954,6 +1002,9 @@ <logger name="org.onap.aai.logging.ErrorLogHelper" level="WARN"> <appender-ref ref="asyncERROR"/> </logger> + <logger name="org.onap.aai.failover" level="DEBUG" additivity="false"> + <appender-ref ref="asyncFailover" /> + </logger> <logger name="org.onap.aai.interceptors.post" level="DEBUG" additivity="false"> <appender-ref ref="asynctranslog" /> </logger> @@ -1028,6 +1079,15 @@ <appender-ref ref="migrationdebug" /> <appender-ref ref="migrationerror" /> </logger> + + <logger name="org.onap.aai.migration" level="ERROR" additivity="false"> + <appender-ref ref="migrationlog" /> + </logger> + + <logger name="org.onap.aai.migration" level="INFO" additivity="false"> + <appender-ref ref="migrationlog" /> + </logger> + <logger name="org.onap.aai.dataexport" level="DEBUG" additivity="false"> <appender-ref ref="dataExportError"/> @@ -1041,15 +1101,6 @@ <logger name="org.janusgraph" level="WARN" /> <logger name="com.att.aft.dme2" level="WARN" /> - <!-- ============================================================================ --> - <!-- General EELF logger --> - <!-- ============================================================================ --> - <logger name="com.att.eelf" level="WARN" additivity="false"> - <appender-ref ref="asyncDEBUG" /> - <appender-ref ref="asyncERROR" /> - <appender-ref ref="asyncMETRIC" /> - </logger> - <root level="DEBUG"> <appender-ref ref="external" /> </root> diff --git a/src/main/resources/uniquePropertyCheck-logback.xml b/src/main/resources/uniquePropertyCheck-logback.xml index b4522d5..6420690 100644 --- a/src/main/resources/uniquePropertyCheck-logback.xml +++ b/src/main/resources/uniquePropertyCheck-logback.xml @@ -11,39 +11,28 @@ <property name="p_thr" value="%thread"/> <property name="pattern" value="%nopexception${p_tim}\t${p_thr}\t${p_lvl}\t${p_log}\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t%n"/> <property name="errorPattern" value="%X{LogTimestamp}|%X{RequestID}|%thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%.-5level|%X{ErrorCode}|%X{ErrorDesc}|%msg%n" /> - <property name="debugPattern" value="%X{LogTimestamp}|%X{RequestID}|%msg\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t|^%n" /> + <property name="debugPattern" value="%X{LogTimestamp}|%X{RequestID}|${p_msg}\t${p_mdc}\t${p_exc}\t${p_mak}\t|^%n" /> <property name="auditPattern" value="%X{EntryTimestamp}|%X{LogTimestamp}|%X{RequestID}|%X{ServiceInstanceID}|%thread||%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}||||${p_mak}|${p_mdc}|||%msg%n" /> <property name="metricPattern" value="%X{InvokeTimestamp}|%X{LogTimestamp}|%X{RequestID}|%X{ServiceInstanceID}|%thread||%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}||||%X{TargetVirtualEntity}|${p_mak}|${p_mdc}|||%msg%n" /> <property name="transLogPattern" value="%X{LogTimestamp}|%date{yyyy-MM-dd'T'HH:mm:ss.SSSZ, UTC}|%X{RequestID}|%X{ServiceInstanceID}|%-10t|%X{ServerFQDN}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%replace(%replace(%X{ResponseDesc}){'\\|', '!'}){'\r|\n', '^'}|%X{InstanceUUID}|%level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{clientIpAddress}||%X{unused}|%X{processKey}|%X{customField1}|%X{customField2}|%X{customField3}|%X{customField4}|co=%X{PartnerName}:%m%n"/> - <appender name="uniquePropertyChecklog" class="ch.qos.logback.classic.sift.SiftingAppender"> - <filter class="ch.qos.logback.classic.filter.LevelFilter"> - <level>INFO</level> - <onMatch>ACCEPT</onMatch> - <onMismatch>DENY</onMismatch> - </filter> - <!-- This is MDC value --> - <!-- We will assign a value to 'logFilenameAppender' via Java code --> - <discriminator> - <key>logFilenameAppender</key> - <defaultValue>undefined</defaultValue> - </discriminator> - <sift> - <!-- A standard RollingFileAppender, the log file is based on 'logFileName' - at runtime --> - <appender name="FILE-${logFilenameAppender}" - class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${logDirectory}/uniquePropertyCheck/${logFilenameAppender}.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/uniquePropertyCheck/${logFilenameAppender}.log.%d{yyyy-MM-dd} - </fileNamePattern> - </rollingPolicy> - <encoder> - <pattern>${auditPattern}</pattern> - </encoder> - </appender> - </sift> - </appender> + <appender name="uniquePropertyChecklog" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/uniquePropertyCheck/debug.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/uniquePropertyCheck/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + <maxHistory>7</maxHistory> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> + </encoder> + </appender> + <logger name="org.reflections" level="WARN" additivity="false"> <appender-ref ref="uniquePropertyChecklog" /> diff --git a/src/main/scripts/common_functions.sh b/src/main/scripts/common_functions.sh index f279334..1fac7ce 100644 --- a/src/main/scripts/common_functions.sh +++ b/src/main/scripts/common_functions.sh @@ -16,7 +16,6 @@ check_user(){ # Sources the profile and sets the project home source_profile(){ - . /etc/profile.d/aai.sh PROJECT_HOME=/opt/app/aai-graphadmin } diff --git a/src/main/scripts/dupeTool.sh b/src/main/scripts/dupeTool.sh index 350b0bd..d4cdb9c 100644 --- a/src/main/scripts/dupeTool.sh +++ b/src/main/scripts/dupeTool.sh @@ -68,6 +68,9 @@ COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) start_date; check_user; source_profile; + +export JAVA_PRE_OPTS=${JAVA_PRE_OPTS:--Xms6g -Xmx6g}; + execute_spring_jar org.onap.aai.dbgen.DupeTool ${PROJECT_HOME}/resources/dupeTool-logback.xml "$@" end_date; -exit 0
\ No newline at end of file +exit 0 diff --git a/src/main/scripts/dynamicPayloadArchive.sh b/src/main/scripts/dynamicPayloadArchive.sh index 87cce13..89cf990 100644 --- a/src/main/scripts/dynamicPayloadArchive.sh +++ b/src/main/scripts/dynamicPayloadArchive.sh @@ -29,7 +29,7 @@ COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) . ${COMMON_ENV_PATH}/common_functions.sh -. /etc/profile.d/aai.sh + PROJECT_HOME=/opt/app/aai-graphadmin PROGNAME=$(basename $0) diff --git a/src/main/scripts/extract-events.sh b/src/main/scripts/extract-events.sh new file mode 100644 index 0000000..5a2bb3b --- /dev/null +++ b/src/main/scripts/extract-events.sh @@ -0,0 +1,20 @@ +#!/bin/bash +usage(){ + echo "Usage $0 input-file output-file event-type"; +} + +if [ "${#}" -lt 3 ]; then + usage; + exit -1 +fi; + +input_file=$1 +output_file=$2 +event_type=$3 + +grep "|${event_type}|" ${input_file} > ${output_file}.1 +sed -i -e '/InvokeReturn/s/^.*$//g' ${output_file}.1 +sed -i '/^$/d' ${output_file}.1 +cat ${output_file}.1 | awk -F '|' '{print $29}' > ${output_file} +rm ${output_file}.1 +exit 0 diff --git a/src/main/scripts/getDslResult.sh b/src/main/scripts/getDslResult.sh new file mode 100644 index 0000000..78286cd --- /dev/null +++ b/src/main/scripts/getDslResult.sh @@ -0,0 +1,151 @@ +#!/bin/ksh + +### +# ============LICENSE_START======================================================= +# org.onap.aai +# ================================================================================ +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +### + +# + +display_usage() { + cat <<EOF + Usage: $0 [options] + + 1. Usage: getDslResult.sh <base-path or optional host url> <optional input-json-filepath> <optional -XFROMAPPID> <optional -XTRANSID> + 2. This script requires one argument, a base-path + 3. Example for basepath: aai/{version}\ + 4. Adding the optional input-json-payload replaces the default dsl payload with the contents of the input-file + 5. The query result is returned in the file resources/migration-input-files/dslResults.json +EOF +} +if [ $# -eq 0 ]; then + display_usage + exit 1 +fi + +RESOURCE="dsl?format=resource_and_url&nodesOnly=true" + +BASEPATH=$1 +if [ -z $BASEPATH ]; then + echo "base-path parameter is missing" + echo "usage: $0 <base-path>" + exit 1 +fi + + +PROJECT_HOME=/opt/app/aai-graphadmin +RESULTDIR=$PROJECT_HOME/logs/data/migration-input-files + +if [ ! -d ${RESULTDIR} ]; then + mkdir -p ${RESULTDIR} + chown aaiadmin:aaiadmin ${RESULTDIR} + chmod u+w ${RESULTDIR} +fi + +RESULTPATH=$RESULTDIR/dslResults.json + +JSONFILE=$2 +TEMPFILE=/tmp/dslResult-temp.json +if [ -z $JSONFILE ]; then + JSONFILE=$TEMPFILE + echo "{ \"dsl\":\"l3-network('network-type','Tenant Network')>vlan-tag>vlan-range*\" }" > $JSONFILE +fi + +echo `date` " Starting $0 for resource $RESOURCE" + + +XFROMAPPID="AAI-TOOLS" +XTRANSID=`uuidgen` + +if [ ! -z "$3" ]; then + XFROMAPPID=$3 +fi + +if [ ! -z "$4" ]; then + XTRANSID=$4 +fi + +userid=$( id | cut -f2 -d"(" | cut -f1 -d")" ) +if [ "${userid}" != "aaiadmin" ]; then + echo "You must be aaiadmin to run $0. The id used $userid." + exit 1 +fi + +prop_file=$PROJECT_HOME/resources/etc/appprops/aaiconfig.properties +log_dir=$PROJECT_HOME/logs/misc +today=$(date +\%Y-\%m-\%d) + +RETURNRESPONSE=true + +MISSING_PROP=false +RESTURLPROP=`grep ^aai.server.url.base= $prop_file |cut -d'=' -f2 |tr -d "\015"` +if [ -z $RESTURLPROP ]; then + echo "Property [aai.server.url.base] not found in file $prop_file" + MISSING_PROP=true +else + RESTURL=`echo $RESTURLPROP | sed -e "s#aai/#$BASEPATH/#"` +fi + +if [ ! -z "$1" ]; then + if [[ $1 == *"https"* ]]; then + RESTURL=$1 + fi +fi + +USEBASICAUTH=false +BASICENABLE=`grep ^aai.tools.enableBasicAuth $prop_file |cut -d'=' -f2 |tr -d "\015"` +if [ -z $BASICENABLE ]; then + USEBASICAUTH=false +else + USEBASICAUTH=true + CURLUSER=`grep ^aai.tools.username $prop_file |cut -d'=' -f2 |tr -d "\015"` + if [ -z $CURLUSER ]; then + echo "Property [aai.tools.username] not found in file $prop_file" + MISSING_PROP=true + fi + CURLPASSWORD=`grep ^aai.tools.password $prop_file |cut -d'=' -f2 |tr -d "\015"` + if [ -z $CURLPASSWORD ]; then + echo "Property [aai.tools.password] not found in file $prop_file" + MISSING_PROP=true + fi +fi + +fname=$JSONFILE +if [ -f /tmp/$(basename $JSONFILE) ]; then + fname=/tmp/$(basename $JSONFILE) +elif [ ! -f $JSONFILE ]; then + echo "The file $JSONFILE does not exist" + exit -1 +fi + +if [ $MISSING_PROP = false ]; then + if [ $USEBASICAUTH = false ]; then + AUTHSTRING="--cert $PROJECT_HOME/resources/etc/auth/aaiClientPublicCert.pem --key $PROJECT_HOME/resources/etc/auth/aaiClientPrivateKey.pem" + else + AUTHSTRING="-u $CURLUSER:$CURLPASSWORD" + fi + curl --request PUT -k $AUTHSTRING -H "X-FromAppId: $XFROMAPPID" -H "X-TransactionId: $XTRANSID" -H "Accept: application/json" -H "Content-Type: application/json" -T $fname $RESTURL$RESOURCE | jq '.' > $RESULTPATH + RC=$? + +else + echo "usage: $0 <base-path>" + RC=-1 +fi +if [ "a$JSONFILE" = "a$TEMPFILE" ]; then + rm $TEMPFILE +fi +echo `date` " Done $0, exit code is $RC, returning result in $RESULTPATH" +exit $RC
\ No newline at end of file diff --git a/src/main/scripts/resend-dmaap-events.sh b/src/main/scripts/resend-dmaap-events.sh index 2afa3a7..b8ff73f 100644 --- a/src/main/scripts/resend-dmaap-events.sh +++ b/src/main/scripts/resend-dmaap-events.sh @@ -33,17 +33,21 @@ # # -b, (required) <string> the base url for the dmaap server # -e, (required) <file> filename containing the missed events -# -l, (optional) indicating that the script should be run it debug mode +# -l, (optional) indicating that the script should be run in debug mode # it will not send the dmaap messages to dmaap server # but it will write to a file named resend_dmaap_server.out +# -x (optional) skip resource version check # -p, (required) <string> the password for the dmaap server # -s, (required) <file> containing the data snapshot graphson file to compare the resource versions against -# -u, (required) <string> the username for the dmaap server +# partial snapshots should be concatenated into a full snapshot file +# before running the script +# -u, (required) <string> the username for the dmaap server +# -t, (required) <string> the dmaap topic # # An example of how to use the tool: # Please use right credentials and right dmaap server in the cluster # -# ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 +# ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t AAI-EVENT # # For each dmaap message in the example_events.txt, it will check # against graphson and try to send it to the dmaap server @@ -66,9 +70,11 @@ # For testing purposes, if you are trying to run this script and don't want to actually # send it to a dmaap server, then you can run either of the following: # -# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 +# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -t <other-dmaap-topic> # or -# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 +# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t <other-dmaap-topic> +# or, to skip the resource-version check +# ./resend-dmaap-events.sh -l -x -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t <other-dmaap-topic> # # Following will output what would have been sent out based on checking the datasnapshot with example_events.txt # @@ -90,18 +96,22 @@ usage(){ echo; echo " -b, <string> the base url for the dmaap server"; echo " -e, <file> filename containing the missed events"; - echo " -l, (optional) indicating that the script should be run it debug mode" + echo " -l, (optional) indicating that the script should be run in debug mode" echo " it will not send the dmaap messages to dmaap server " echo " but it will write to a file named resend_dmaap_server.out" + echo " -x, (optional) indicating that the script will skip the resource-version check" echo " -p, <string> the password for the dmaap server"; echo " -s, <file> containing the data snapshot graphson file to compare the resource versions against"; + echo " partial snapshots should be concatenated into a full snapshot file"; + echo " before running the script"; echo " -u, <string> the username for the dmaap server"; + echo " -t, <string> the dmaap topic"; echo; echo; echo " An example of how to use the tool:"; echo " Please use right credentials and right dmaap server in the cluster"; echo; - echo " ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905"; + echo " ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t AAI-EVENT"; echo; echo " For each dmaap message in the example_events.txt, it will check "; echo " against graphson and try to send it to the dmaap server"; @@ -126,7 +136,9 @@ usage(){ echo; echo " ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545"; echo " or"; - echo " ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905"; + echo " ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t AAI-EVENT"; + echo " or, to skip the resource-version check"; + echo " ./resend-dmaap-events.sh -l -x -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t AAI-EVENT"; echo; echo " Following will output what would have been sent out based on checking the datasnapshot with example_events.txt"; echo; @@ -143,25 +155,25 @@ usage(){ # Checks if the argument of the string is greater than zero # Also check if the file actually exists validate(){ - local type_of_file=$1; + type_of_file=$1; if [ $# -eq 0 ]; then echo "Error expecting the validate method to have at least one argument indicating what type"; - exit -1; + exit 1; fi; shift; - local arg=$1; + arg=$1; if [ -z "$arg" ]; then echo "Error missing the expected argument for ${type_of_file}"; - exit -1; + exit 1; fi; if [ ! -f "$arg" ]; then echo "Error: file $arg cannot be found, please check the file again"; - exit -1; + exit 1; fi; } @@ -171,26 +183,31 @@ validate(){ # Otherwise it will return non zero to indicate that this method failed resource_version_matches_snapshot_file(){ - local snapshot_file=$1; - local entity_link=$2; - local resource_version=$3; - local action=$4; + snapshot_file=$1; + entity_link=$2; + resource_version=$3; + action=$4; + topic=$5; if [ -z ${resource_version} ]; then echo "Missing the parameter resource version to be passed"; - return -1; + return 1; fi # Modify the entity link passed to remove the /aai/v[0-9]+ - aai_uri=$(echo $entity_link | sed 's/\/aai\/v[0-9][0-9]*//g'); + if [ "${topic}" = "<other-dmaap-topic>" ]; then + aai_uri=$(echo $entity_link | sed 's/\/<other-base>\/v[0-9][0-9]*//g'); + else + aai_uri=$(echo $entity_link | sed 's/\/aai\/v[0-9][0-9]*//g'); + fi - local line=$(grep '"value":"'${aai_uri}'"' ${snapshot_file} 2> /dev/null); + line=$(grep '"value":"'${aai_uri}'"' ${snapshot_file} 2> /dev/null); if [ -z "${line}" ] ; then if [ "${action}" = "DELETE" ]; then return 0; else - return -1; + return 1; fi; fi; @@ -199,51 +216,26 @@ resource_version_matches_snapshot_file(){ if [ $cnt -eq 1 ]; then return 0; else - return -1; + return 1; fi; } -# From a array being passed, it will determine the smallest element -# and return the index of the smallest element -# If the array length is zero, then it will return -1 -retrieve_smallest_index(){ - - local elements=("${@}"); - - if [ ${#elements} -eq 0 ]; then - return -1; - fi; - - local smallest_element=${elements[0]}; - - local index=0; - local smallest_index=0; - - for element in ${elements[@]}; do - if [ $element -lt $smallest_element ]; then - smallest_index=${index}; - fi; - index=$((index+1)); - done; - - return ${smallest_index}; -} - # Send the dmaap event to the host based on # the line that was send to the function send_dmaap(){ - local local_mode=$1; - local line=$2; - local username=$3; - local password=$4; - local baseurl=$5; - local resp_code=0; + local_mode=$1; + line=$2; + username=$3; + password=$4; + baseurl=$5; + topic=$6; + resp_code=0; generated_file=$(uuidgen); - local json_file=/tmp/${generated_file}.json; - local curl_output=/tmp/${generated_file}.txt; + json_file=/tmp/${generated_file}.json; + curl_output=/tmp/${generated_file}.txt; echo ${line} > ${json_file}; > ${curl_output}; @@ -259,13 +251,13 @@ send_dmaap(){ -X POST \ -H "Content-Type: application/json" \ -d "@${json_file}" \ - "${baseurl}/events/AAI-EVENT"\ + "${baseurl}/events/${topic}"\ ); if [ "$response_code" -ne "200" ]; then echo -n "Response failure for dmaap message with id ${id}," >> ${resend_error_log}; echo " code: ${response_code} body: $(cat ${curl_output})" >> ${resend_error_log}; - resp_code=-1; + resp_code=1; fi; fi; @@ -303,25 +295,32 @@ main(){ usage; fi; - while getopts ":e:s:u:lp:b:h" opt; do + versioncheck=true + while getopts ":e:s:u:xlp:b:t:h" opt; do case ${opt} in l ) # Specify that the application will not send messages to dmaap but save it a file - local local_mode=true + local_mode=true ;; e ) # Specify the file for missed events - local missed_events_file=$OPTARG + missed_events_file=$OPTARG ;; s ) # Specify the file for snapshot - local snapshot_file=$OPTARG + snapshot_file=$OPTARG ;; u ) # Specify the username to dmaap - local username=$OPTARG + username=$OPTARG ;; p ) # Specify the password to dmaap - local password=$OPTARG + password=$OPTARG + ;; + t ) # Specify the dmaap topic + topic=$OPTARG + ;; + x ) # Specify whether to skip version check + versioncheck=false ;; b ) # Specify the baseurl to dmaap - local hostname=$OPTARG + hostname=$OPTARG ;; h ) usage; @@ -345,15 +344,25 @@ main(){ id=$(echo $dmaap_event | grep -o '"id":"[^"]*"' | cut -d":" -f2- | sed 's/"//g'); action=$(echo $dmaap_event | grep -o '"action":"[^"]*"' | cut -d":" -f2- | sed 's/"//g'); smallest_resource_version=$(echo $dmaap_event | jq -M '.' | grep 'resource-version' | sort | tail -1 | sed 's/[^0-9]//g'); - resource_version_matches_snapshot_file "${snapshot_file}" "${entity_link}" "${smallest_resource_version}" "${action}" && { - send_dmaap "${local_mode}" "$dmaap_event" "$username" "$password" "$hostname" && { + #smallest_resource_version=$(echo $dmaap_event | python -m json.tool | grep 'resource-version' | sort | tail -1 | sed 's/[^0-9]//g'); + match=0; + if [ "$versioncheck" = true ]; then + resource_version_matches_snapshot_file "${snapshot_file}" "${entity_link}" "${smallest_resource_version}" "${action}" "${topic}" && { + match=0; + } || { + match=1; + } + fi; + + if [ $match -eq 0 ]; then + send_dmaap "${local_mode}" "$dmaap_event" "$username" "$password" "$hostname" "$topic" && { echo "Dmaap Event with Id $id was sent"; } || { echo "Dmaap Event with Id $id was not sent due to dmaap error, please check logs"; } - } || { + else echo "Dmaap Event with Id $id not sent"; - } + fi; done < ${missed_events_file}; diff --git a/src/main/scripts/run_SendDeleteMigrationNotification.sh b/src/main/scripts/run_SendDeleteMigrationNotification.sh index ebd8677..0b2f3e1 100644 --- a/src/main/scripts/run_SendDeleteMigrationNotification.sh +++ b/src/main/scripts/run_SendDeleteMigrationNotification.sh @@ -1,4 +1,4 @@ -#!/bin/ksh +#!/bin/bash ### # ============LICENSE_START======================================================= diff --git a/src/main/scripts/run_SendMigrationNotification.sh b/src/main/scripts/run_SendMigrationNotification.sh index 4bcc0d9..994485e 100644 --- a/src/main/scripts/run_SendMigrationNotification.sh +++ b/src/main/scripts/run_SendMigrationNotification.sh @@ -1,4 +1,4 @@ -#!/bin/ksh +#!/bin/bash ### # ============LICENSE_START======================================================= @@ -43,7 +43,7 @@ fi INPUT_DIR_FOR_JAVA=${INPUT_PATH}/combined mkdir -p "$INPUT_DIR_FOR_JAVA" INPUT_FILE_FOR_JAVA=${INPUT_DIR_FOR_JAVA}/sorted_dmaap_files.txt -sort --numeric-sort -k 1 -t '_' $(find ${INPUT_PATH}/* -maxdepth 0 -type f) | awk -F '_' '{ print $2"_"$3; }' > $INPUT_FILE_FOR_JAVA +sort -g -k 1 -t '_' $(find ${INPUT_PATH}/* -maxdepth 0 -type f) | awk -F '_' '{ print $2"_"$3; }' > $INPUT_FILE_FOR_JAVA shift |