diff options
| author |   Venkata Harish K Kajur <vk250x@att.com> | 2018-01-18 00:47:09 -0500 |
|---|---|---|
| committer | Venkata Harish K Kajur <vk250x@att.com> | 2018-01-18 17:42:01 -0500 |
| commit | 6b98bb3ec93a0bbfbe0e660cabc69978b53bfee8 (patch) | |
| tree | 5e68f93e33737684d63d655ac5a5c6d611f45b5e /aai-core/src/main/java | |
| parent | 89e91557997a4fc11db1b574e13c07b2149a63e3 (diff) | |
Sync the latest code changes
Issue-ID: AAI-493
Change-Id: I5f39ce9bb19cfb9607b659bed82066cae7569ecc
Signed-off-by: Venkata Harish K Kajur <vk250x@att.com>
Diffstat (limited to 'aai-core/src/main/java')
66 files changed, 4753 insertions, 831 deletions
diff --git a/aai-core/src/main/java/org/onap/aai/audit/ListEndpoints.java b/aai-core/src/main/java/org/onap/aai/audit/ListEndpoints.java index 5ee973a3..bfa8748c 100644 --- a/aai-core/src/main/java/org/onap/aai/audit/ListEndpoints.java +++ b/aai-core/src/main/java/org/onap/aai/audit/ListEndpoints.java @@ -40,6 +40,8 @@ import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.introspection.ModelType; import org.onap.aai.introspection.Version; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.logging.LogFormatTools; + import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import com.google.common.base.CaseFormat; @@ -121,7 +123,7 @@ public class ListEndpoints { try { endpoints.add(currentUri + obj.getLoader().introspectorFromName(key).getGenericURI()); } catch (AAIUnknownObjectException e) { - LOGGER.warn("Skipping endpoint for " + key + " (Unknown object)", e); + LOGGER.warn("Skipping endpoint for " + key + " (Unknown object) " + LogFormatTools.getStackTop(e)); } } } @@ -147,7 +149,7 @@ public class ListEndpoints { newVisited ); } catch (AAIUnknownObjectException e) { - LOGGER.warn("Skipping nested endpoint for " + propName + " (Unknown Object)", e); + LOGGER.warn("Skipping nested endpoint for " + propName + " (Unknown Object) " + LogFormatTools.getStackTop(e)); } } } else if (obj.isComplexType(propName)) { @@ -161,7 +163,7 @@ public class ListEndpoints { visited ); } catch (AAIUnknownObjectException e) { - LOGGER.warn("Skipping nested enpoint for " + propName + " (Unknown Object)", e); + LOGGER.warn("Skipping nested enpoint for " + propName + " (Unknown Object) "+ LogFormatTools.getStackTop(e)); } } } diff --git a/aai-core/src/main/java/org/onap/aai/db/props/AAIProperties.java b/aai-core/src/main/java/org/onap/aai/db/props/AAIProperties.java index 76a13363..e26d930e 100644 --- a/aai-core/src/main/java/org/onap/aai/db/props/AAIProperties.java +++ b/aai-core/src/main/java/org/onap/aai/db/props/AAIProperties.java @@ -36,5 +36,6 @@ public class AAIProperties { public static final Integer MAXIMUM_DEPTH = 10000; public static final String LINKED = "linked"; public static final String DB_ALIAS_SUFFIX = "-local"; + public static final String AAI_UUID = "aai-uuid"; } diff --git a/aai-core/src/main/java/org/onap/aai/db/schema/AuditOXM.java b/aai-core/src/main/java/org/onap/aai/db/schema/AuditOXM.java index e64f3ea3..0a3650fa 100644 --- a/aai-core/src/main/java/org/onap/aai/db/schema/AuditOXM.java +++ b/aai-core/src/main/java/org/onap/aai/db/schema/AuditOXM.java @@ -47,6 +47,7 @@ import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.introspection.ModelType; import org.onap.aai.introspection.Version; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.schema.enums.ObjectMetadata; import org.onap.aai.util.AAIConstants; import com.att.eelf.configuration.EELFLogger; @@ -77,7 +78,7 @@ public class AuditOXM extends Auditor { allObjects.add(temp); this.createDBProperties(temp); } catch (AAIUnknownObjectException e) { - LOGGER.warn("Skipping audit for object " + key + " (Unknown Object)", e); + LOGGER.warn("Skipping audit for object " + key + " (Unknown Object) " + LogFormatTools.getStackTop(e)); } } for (Introspector temp : allObjects) { diff --git a/aai-core/src/main/java/org/onap/aai/db/schema/ScriptDriver.java b/aai-core/src/main/java/org/onap/aai/db/schema/ScriptDriver.java index fc494883..968d9ef2 100644 --- a/aai-core/src/main/java/org/onap/aai/db/schema/ScriptDriver.java +++ b/aai-core/src/main/java/org/onap/aai/db/schema/ScriptDriver.java @@ -22,13 +22,17 @@ package org.onap.aai.db.schema; import java.io.IOException; +import java.util.UUID; +import org.apache.commons.configuration.ConfigurationException; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; - +import org.onap.aai.dbmap.AAIGraphConfig; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Version; +import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LoggingContext.StatusCode; import org.onap.aai.util.AAIConfig; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -47,9 +51,19 @@ public class ScriptDriver { * @throws JsonMappingException the json mapping exception * @throws IOException Signals that an I/O exception has occurred. */ - public static void main (String[] args) throws AAIException, IOException { + public static void main (String[] args) throws AAIException, IOException, ConfigurationException { CommandLineArgs cArgs = new CommandLineArgs(); + LoggingContext.init(); + LoggingContext.component("DBSchemaScriptDriver"); + LoggingContext.partnerName("NA"); + LoggingContext.targetEntity("AAI"); + LoggingContext.requestId(UUID.randomUUID().toString()); + LoggingContext.serviceName("AAI"); + LoggingContext.targetServiceName("main"); + LoggingContext.statusCode(StatusCode.COMPLETE); + LoggingContext.responseCode(LoggingContext.SUCCESS); + new JCommander(cArgs, args); if (cArgs.help) { @@ -57,7 +71,7 @@ public class ScriptDriver { } String config = cArgs.config; AAIConfig.init(); - try (TitanGraph graph = TitanFactory.open(config)) { + try (TitanGraph graph = TitanFactory.open(new AAIGraphConfig.Builder(config).forService(ScriptDriver.class.getSimpleName()).withGraphType("NA").buildConfiguration())) { if (!("oxm".equals(cArgs.type) || "graph".equals(cArgs.type))) { System.out.println("type: " + cArgs.type + " not recognized."); System.exit(1); diff --git a/aai-core/src/main/java/org/onap/aai/dbgen/DataGrooming.java b/aai-core/src/main/java/org/onap/aai/dbgen/DataGrooming.java index afc7807b..f7b9d866 100644 --- a/aai-core/src/main/java/org/onap/aai/dbgen/DataGrooming.java +++ b/aai-core/src/main/java/org/onap/aai/dbgen/DataGrooming.java @@ -50,6 +50,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.onap.aai.db.props.AAIProperties; import org.onap.aai.dbmap.AAIGraph; +import org.onap.aai.dbmap.AAIGraphConfig; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Introspector; import org.onap.aai.introspection.Loader; @@ -57,11 +58,13 @@ import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.introspection.ModelType; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; +import org.onap.aai.logging.LoggingContext; import org.onap.aai.serialization.db.AAIDirection; import org.onap.aai.serialization.db.EdgeProperty; -import org.onap.aai.util.AAIConfig; -import org.onap.aai.util.AAIConstants; -import org.onap.aai.util.FormatDate; +import org.onap.aai.util.*; +import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LoggingContext.StatusCode; import com.att.eelf.configuration.Configuration; import com.att.eelf.configuration.EELFLogger; @@ -83,8 +86,9 @@ public class DataGrooming { * @param args the arguments */ public static void main(String[] args) { - + // Set the logging file properties to be used by EELFManager + System.setProperty("aai.service.name", DataGrooming.class.getSimpleName()); Properties props = System.getProperties(); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_DATA_GROOMING_LOGBACK_PROPS); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); @@ -102,9 +106,19 @@ public class DataGrooming { Boolean neverUseCache = false; Boolean skipEdgeCheckFlag = false; + LoggingContext.init(); + LoggingContext.partnerName(FROMAPPID); + LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS); + LoggingContext.component("dataGrooming"); + LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS); + LoggingContext.targetServiceName("main"); + LoggingContext.requestId(TRANSID); + LoggingContext.statusCode(StatusCode.COMPLETE); + LoggingContext.responseCode(LoggingContext.SUCCESS); + int timeWindowMinutes = 0; // A value of 0 means that we will not have a time-window -- we will look // at all nodes of the passed-in nodeType. - long windowStartTime = 0; // Translation of the window into a starting timestamp + int maxRecordsToFix = AAIConstants.AAI_GROOMING_DEFAULT_MAX_FIX; int sleepMinutes = AAIConstants.AAI_GROOMING_DEFAULT_SLEEP_MINUTES; @@ -127,17 +141,9 @@ public class DataGrooming { dupeGrpsDeleted = 0; FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); String dteStr = fd.getDateTime(); - String groomOutFileName = "dataGrooming." + dteStr + ".out"; - String argString = ""; - for( int x = 0; x < args.length; x++ ) { - argString = argString + " " + args[x]; - } - LOGGER.info(" DataGrooming called with these options: [" + argString + "]"); - if (args.length > 0) { // They passed some arguments in that will affect processing - for (int i = 0; i < args.length; i++) { String thisArg = args[i]; if (thisArg.equals("-edgesOnly")) { @@ -165,76 +171,98 @@ public class DataGrooming { } else if (thisArg.equals("-maxFix")) { i++; if (i >= args.length) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error(" No value passed with -maxFix option. "); - System.exit(0); + AAISystemExitUtil.systemExitCloseAAIGraph(0); } String nextArg = args[i]; try { maxRecordsToFix = Integer.parseInt(nextArg); } catch (Exception e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error("Bad value passed with -maxFix option: [" + nextArg + "]"); - System.exit(0); + AAISystemExitUtil.systemExitCloseAAIGraph(0); } } else if (thisArg.equals("-sleepMinutes")) { i++; if (i >= args.length) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error("No value passed with -sleepMinutes option."); - System.exit(0); + AAISystemExitUtil.systemExitCloseAAIGraph(0); } String nextArg = args[i]; try { sleepMinutes = Integer.parseInt(nextArg); } catch (Exception e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error("Bad value passed with -sleepMinutes option: [" + nextArg + "]"); - System.exit(0); + AAISystemExitUtil.systemExitCloseAAIGraph(0); } } else if (thisArg.equals("-timeWindowMinutes")) { i++; if (i >= args.length) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error("No value passed with -timeWindowMinutes option."); - System.exit(0); + AAISystemExitUtil.systemExitCloseAAIGraph(0); } String nextArg = args[i]; try { timeWindowMinutes = Integer.parseInt(nextArg); } catch (Exception e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error("Bad value passed with -timeWindowMinutes option: [" + nextArg + "]"); - System.exit(0); + AAISystemExitUtil.systemExitCloseAAIGraph(0); } - if( timeWindowMinutes > 0 ){ - // Translate the window value (ie. 30 minutes) into a unix timestamp like - // we use in the db - so we can select data created after that time. - windowStartTime = figureWindowStartTime( timeWindowMinutes ); - } + } else if (thisArg.equals("-f")) { i++; if (i >= args.length) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error(" No value passed with -f option. "); - System.exit(0); + AAISystemExitUtil.systemExitCloseAAIGraph(0); } prevFileName = args[i]; } else { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error(" Unrecognized argument passed to DataGrooming: [" + thisArg + "]. "); - LOGGER.error(" Valid values are: -f -autoFix -maxFix -edgesOnly -dupeFixOn -donFixOrphans -sleepMinutes -neverUseCache"); - System.exit(0); + LOGGER.error(" Valid values are: -f -autoFix -maxFix -edgesOnly -skipEdgeChecks -dupeFixOn -donFixOrphans -timeWindowMinutes -sleepMinutes -neverUseCache"); + AAISystemExitUtil.systemExitCloseAAIGraph(0); } } } - + String windowTag = "FULL"; + if( timeWindowMinutes > 0 ){ + windowTag = "PARTIAL"; + } + String groomOutFileName = "dataGrooming." + windowTag + "." + dteStr + ".out"; + try { LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST); } catch (Exception ex){ - LOGGER.error("ERROR - Could not create loader", ex); - System.exit(1); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); + LOGGER.error("ERROR - Could not create loader " + LogFormatTools.getStackTop(ex)); + AAISystemExitUtil.systemExitCloseAAIGraph(1); } + if (skipHostCheck) { + LOGGER.info(" We will skip the HostCheck as requested. "); + } try { if (!prevFileName.equals("")) { @@ -248,7 +276,7 @@ public class DataGrooming { maxRecordsToFix, groomOutFileName, ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn, finalShutdownFlag, cacheDbOkFlag, - skipEdgeCheckFlag, windowStartTime); + skipEdgeCheckFlag, timeWindowMinutes); } else if (doAutoFix) { // They want us to run the processing twice -- first to look for // delete candidates, then after @@ -264,7 +292,7 @@ public class DataGrooming { dontFixOrphansFlag, maxRecordsToFix, groomOutFileName, ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn, finalShutdownFlag, cacheDbOkFlag, - skipEdgeCheckFlag, windowStartTime); + skipEdgeCheckFlag, timeWindowMinutes); if (fixCandCount == 0) { LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. "); } else { @@ -277,7 +305,7 @@ public class DataGrooming { Thread.sleep(sleepMsec); } catch (InterruptedException ie) { LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< "); - System.exit(0); + AAISystemExitUtil.systemExitCloseAAIGraph(0); } dteStr = fd.getDateTime(); @@ -293,7 +321,7 @@ public class DataGrooming { secondGroomOutFileName, ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn, finalShutdownFlag, cacheDbOkFlag, - skipEdgeCheckFlag, windowStartTime); + skipEdgeCheckFlag, timeWindowMinutes); } } else { // Do the grooming - plain vanilla (no fix-it-file, no @@ -309,14 +337,16 @@ public class DataGrooming { maxRecordsToFix, groomOutFileName, ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn, finalShutdownFlag, cacheDbOkFlag, - skipEdgeCheckFlag, windowStartTime); + skipEdgeCheckFlag, timeWindowMinutes); } } catch (Exception ex) { - LOGGER.error("Exception while grooming data", ex); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("Exception while grooming data " + LogFormatTools.getStackTop(ex)); } LOGGER.info(" Done! "); - System.exit(0); + AAISystemExitUtil.systemExitCloseAAIGraph(0); }// End of main() @@ -345,11 +375,12 @@ public class DataGrooming { Boolean dupeCheckOff, Boolean dupeFixOn, Boolean ghost2CheckOff, Boolean ghost2FixOn, Boolean finalShutdownFlag, Boolean cacheDbOkFlag, - Boolean skipEdgeCheckFlag, long windowStartTime) { + Boolean skipEdgeCheckFlag, int timeWindowMinutes) { LOGGER.debug(" Entering doTheGrooming \n"); int cleanupCandidateCount = 0; + long windowStartTime = 0; // Translation of the window into a starting timestamp BufferedWriter bw = null; TitanGraph graph = null; TitanGraph graph2 = null; @@ -360,6 +391,12 @@ public class DataGrooming { Graph g = null; Graph g2 = null; try { + if( timeWindowMinutes > 0 ){ + // Translate the window value (ie. 30 minutes) into a unix timestamp like + // we use in the db - so we can select data created after that time. + windowStartTime = figureWindowStartTime( timeWindowMinutes ); + } + AAIConfig.init(); String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" @@ -375,6 +412,8 @@ public class DataGrooming { } if (deleteCandidateList.size() > maxRecordsToFix) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(" >> WARNING >> Delete candidate list size (" + deleteCandidateList.size() + ") is too big. The maxFix we are using is: " @@ -404,10 +443,10 @@ public class DataGrooming { if( cacheDbOkFlag ){ // Since we're just reading (not deleting/fixing anything), we can use // a cached connection to the DB - graph = TitanFactory.open(AAIConstants.CACHED_DB_CONFIG); + graph = TitanFactory.open(new AAIGraphConfig.Builder(AAIConstants.CACHED_DB_CONFIG).forService(DataGrooming.class.getSimpleName()).withGraphType("cached").buildConfiguration()); } else { - graph = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG); + graph = TitanFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(DataGrooming.class.getSimpleName()).withGraphType("realtime1").buildConfiguration()); } if (graph == null) { String emsg = "null graph object in DataGrooming\n"; @@ -448,12 +487,16 @@ public class DataGrooming { String nType = entry.getKey(); int thisNtCount = 0; int thisNtDeleteCount = 0; + LOGGER.debug(" > Look at : [" + nType + "] ..."); ntList = ntList + "," + nType; // Get a collection of the names of the key properties for this nodeType to use later - // Determine what the key fields are for this nodeType - Collection <String> keyProps = entry.getValue().getKeys(); + // Determine what the key fields are for this nodeType - use an arrayList so they + // can be gotten out in a consistent order. + Set <String> keyPropsSet = entry.getValue().getKeys(); + ArrayList <String> keyProps = new ArrayList <String> (); + keyProps.addAll(keyPropsSet); // Get the types of nodes that this nodetype depends on for uniqueness (if any) Collection <String> depNodeTypes = loader.introspectorFromName(nType).getDependentOn(); @@ -477,12 +520,13 @@ public class DataGrooming { } Vertex thisVtx = iter.next(); if( windowStartTime > 0 ){ - // We only want nodes that are created after a passed-in timestamp - Object objTimeStamp = thisVtx.property("aai-created-ts").orElse(null); - if( objTimeStamp != null ){ - long thisNodeCreateTime = (long)objTimeStamp; - if( thisNodeCreateTime < windowStartTime ){ - // It is NOT in our window, so we can pass over it + // They are using the time-window, so we only want nodes that are updated after a + // passed-in timestamp OR that have no last-modified-timestamp which means they are suspicious. + Object objModTimeStamp = thisVtx.property("aai-last-mod-ts").orElse(null); + if( objModTimeStamp != null ){ + long thisNodeModTime = (long)objModTimeStamp; + if( thisNodeModTime < windowStartTime ){ + // It has a last modified ts and is NOT in our window, so we can pass over it continue; } } @@ -521,7 +565,9 @@ public class DataGrooming { boolean depNodeOk = true; if( depNodeTypes.isEmpty() ){ // This kind of node is not dependent on any other. - // Make sure we can get it back using it's key properties and that we only get one. + // Make sure we can get it back using it's key properties (that is the + // phantom checking) and that we only get one. Note - we also need + // to collect data for a second type of dupe-checking which is done later. secondGetList = getNodeJustUsingKeyParams( TRANSID, FROMAPPID, source1, nType, propHashWithKeys, version ); } @@ -556,7 +602,9 @@ public class DataGrooming { zeroEdges = true; } } catch (Exception ex) { - LOGGER.warn("WARNING from inside the for-each-vid-loop orphan-edges-check ", ex); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.warn("WARNING from inside the for-each-vid-loop orphan-edges-check " + LogFormatTools.getStackTop(ex) ); } if (deleteCandidateList.contains(thisVid)) { @@ -568,7 +616,9 @@ public class DataGrooming { thisNtDeleteCount++; } catch (Exception e) { okFlag = false; - LOGGER.error("ERROR trying to delete missing-dep-node VID = " + thisVid, e); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("ERROR trying to delete missing-dep-node VID = " + thisVid + " " + LogFormatTools.getStackTop(e)); } if (okFlag) { LOGGER.info(" DELETED missing-dep-node VID = " + thisVid); @@ -605,7 +655,7 @@ public class DataGrooming { } } } - } + }// end of -- else this is a dependent node -- piece if( depNodeOk && (secondGetList == null || secondGetList.size() == 0) ){ // We could not get the node back using it's own key info. @@ -618,7 +668,9 @@ public class DataGrooming { thisNtDeleteCount++; } catch (Exception e) { okFlag = false; - LOGGER.error("ERROR trying to delete phantom VID = " + thisVid, e); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("ERROR trying to delete phantom VID = " + thisVid + " " + LogFormatTools.getStackTop(e)); } if (okFlag) { LOGGER.info(" DELETED VID = " + thisVid); @@ -644,21 +696,56 @@ public class DataGrooming { } } catch (AAIException e1) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1); errArr.add(e1.getErrorObject().toString()); } catch (Exception e2) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(" For nodeType = " + nType + " Caught exception", e2); errArr.add(e2.getMessage()); } - }// try block to enclose looping of a single vertex + }// try block to enclose looping over each single vertex catch (Exception exx) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING from inside the while-verts-loop ", exx); } } // while loop for each record of a nodeType + if( depNodeTypes.isEmpty() && !dupeCheckOff ){ + // For this nodeType, we haven't looked at the possibility of a + // non-dependent node where two verts have same key info + ArrayList<ArrayList<Vertex>> nonDependentDupeSets = new ArrayList<ArrayList<Vertex>>(); + nonDependentDupeSets = getDupeSets4NonDepNodes( + TRANSID, FROMAPPID, g, + version, nType, tmpList, + keyProps, loader ); + // For each set found (each set is for a unique instance of key-values), + // process the dupes found + Iterator<ArrayList<Vertex>> dsItr = nonDependentDupeSets.iterator(); + while( dsItr.hasNext() ){ + ArrayList<Vertex> dupeList = dsItr.next(); + LOGGER.info(" - now check Dupes for some non-dependent guys - "); + List<String> tmpDupeGroups = checkAndProcessDupes( + TRANSID, FROMAPPID, g, source1, version, + nType, dupeList, dupeFixOn, + deleteCandidateList, singleCommits, dupeGroups, loader); + Iterator<String> dIter = tmpDupeGroups.iterator(); + while (dIter.hasNext()) { + // Add in any newly found dupes to our running list + String tmpGrp = dIter.next(); + LOGGER.info("Found set of dupes: [" + tmpGrp + "]"); + dupeGroups.add(tmpGrp); + } + } + + }// end of extra dupe check for non-dependent nodes + if ( (thisNtDeleteCount > 0) && singleCommits ) { // NOTE - the singleCommits option is not used in normal processing g.tx().commit(); @@ -669,6 +756,7 @@ public class DataGrooming { LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total overall. " ); }// While-loop for each node type + }// end of check to make sure we weren't only supposed to do edges @@ -686,7 +774,7 @@ public class DataGrooming { LOGGER.debug(" ---- DEBUG --- about to open a SECOND graph (takes a little while)--------\n"); // Note - graph2 just reads - but we want it to use a fresh connection to // the database, so we are NOT using the CACHED DB CONFIG here. - graph2 = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG); + graph2 = TitanFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(DataGrooming.class.getSimpleName()).withGraphType("realtime2").buildConfiguration()); if (graph2 == null) { String emsg = "null graph2 object in DataGrooming\n"; throw new AAIException("AAI_6101", emsg); @@ -718,6 +806,8 @@ public class DataGrooming { try { v = vItor2.next(); } catch (Exception vex) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 "); continue; } @@ -727,6 +817,8 @@ public class DataGrooming { try { thisVertId = v.id().toString(); } catch (Exception ev) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list. "); continue; } @@ -739,12 +831,13 @@ public class DataGrooming { } if( windowStartTime > 0 ){ - // We only want to look at nodes that are created after a passed-in timestamp - Object objTimeStamp = v.property("aai-created-ts").orElse(null); - if( objTimeStamp != null ){ - long thisNodeCreateTime = (long)objTimeStamp; - if( thisNodeCreateTime < windowStartTime ){ - // It is NOT in our window, so we can pass over it + // They are using the time-window, so we only want nodes that are updated after a + // passed-in timestamp OR that have no last-modified-timestamp which means they are suspicious. + Object objModTimeStamp = v.property("aai-last-mod-ts").orElse(null); + if( objModTimeStamp != null ){ + long thisNodeModTime = (long)objModTimeStamp; + if( thisNodeModTime < windowStartTime ){ + // It has a last modified ts and is NOT in our window, so we can pass over it continue; } } @@ -763,6 +856,8 @@ public class DataGrooming { try { e = eItor.next(); } catch (Exception iex) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex); continue; } @@ -770,6 +865,8 @@ public class DataGrooming { try { vIn = e.inVertex(); } catch (Exception err) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err); } String vNtI = ""; @@ -795,6 +892,8 @@ public class DataGrooming { if( ! ghost2CheckOff ){ Vertex connectedVert = g2.traversal().V(vIdLong).next(); if( connectedVert == null ) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong); cantGetUsingVid = true; @@ -805,6 +904,8 @@ public class DataGrooming { ghost2 = g.traversal().V(vIdLong).next(); } catch( Exception ex){ + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex); } if( ghost2 != null ){ @@ -814,6 +915,8 @@ public class DataGrooming { }// end of the ghost2 checking } catch (Exception err) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err); } } @@ -841,6 +944,8 @@ public class DataGrooming { deleteCount++; } catch (Exception e1) { okFlag = false; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = " + vIdI, e1); } @@ -864,6 +969,8 @@ public class DataGrooming { // that this edge has already been // removed okFlag = false; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when trying to delete edge = " + thisEid); } @@ -883,6 +990,8 @@ public class DataGrooming { try { vOut = e.outVertex(); } catch (Exception err) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get edge's Out-vertex "); } String vNtO = ""; @@ -915,6 +1024,8 @@ public class DataGrooming { ghost2 = g.traversal().V(vIdLong).next(); } catch( Exception ex){ + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex); } if( ghost2 != null ){ @@ -923,6 +1034,8 @@ public class DataGrooming { } } } catch (Exception err) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err); } } @@ -950,6 +1063,8 @@ public class DataGrooming { deleteCount++; } catch (Exception e1) { okFlag = false; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = " + vIdO, e1); } @@ -973,6 +1088,8 @@ public class DataGrooming { // that this edge has already been // removed okFlag = false; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when trying to delete edge = " + thisEid, ex); } @@ -990,6 +1107,8 @@ public class DataGrooming { } }// End of while-edges-loop } catch (Exception exx) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING from in the while-verts-loop ", exx); } }// End of while-vertices-loop (the edge-checking) @@ -1004,7 +1123,9 @@ public class DataGrooming { executeFinalCommit = true; LOGGER.info("Commit was successful "); } catch (Exception excom) { - LOGGER.error(" >>>> ERROR <<<< Could not commit changes. ", excom); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error(" >>>> ERROR <<<< Could not commit changes. " + LogFormatTools.getStackTop(excom)); deleteCount = 0; } } @@ -1018,7 +1139,14 @@ public class DataGrooming { deleteCount = deleteCount + dupeGrpsDeleted; bw.write("\n\n ============ Summary ==============\n"); - bw.write("Ran these nodeTypes: " + ntList + "\n\n"); + if( timeWindowMinutes == 0 ){ + bw.write("Ran FULL data grooming (no time-window). \n"); + } + else { + bw.write("Ran PARTIAL data grooming just looking at data added/updated in the last " + timeWindowMinutes + " minutes. \n"); + } + + bw.write("\nRan these nodeTypes: " + ntList + "\n\n"); bw.write("There were this many delete candidates from previous run = " + deleteCandidateList.size() + "\n"); if (dontFixOrphansFlag) { @@ -1084,7 +1212,9 @@ public class DataGrooming { bw.write(info + "\n"); } } catch (Exception dex) { - LOGGER.error("error trying to print detail info for a ghost-node: ", dex); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("error trying to print detail info for a ghost-node: " + LogFormatTools.getStackTop(dex)); } } @@ -1106,7 +1236,9 @@ public class DataGrooming { bw.write(info + "\n"); } } catch (Exception dex) { - LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge", dex); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge " + LogFormatTools.getStackTop(dex)); } } @@ -1129,7 +1261,10 @@ public class DataGrooming { bw.write(info + "\n"); } } catch (Exception dex) { - LOGGER.error("error trying to print detail info for a node missing its dependent edge but not an orphan", dex); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("error trying to print detail info for a node missing its dependent edge but not an orphan " + + LogFormatTools.getStackTop(dex)); } } @@ -1149,7 +1284,9 @@ public class DataGrooming { + propKey.value() + "]\n"); } } catch (Exception pex) { - LOGGER.error("error trying to print empty/bad vertex data: ", pex); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("error trying to print empty/bad vertex data: " + LogFormatTools.getStackTop(pex)); } } @@ -1228,7 +1365,9 @@ public class DataGrooming { }// else last entry }// for each vertex in a group } catch (Exception dex) { - LOGGER.error("error trying to print duplicate vertex data", dex); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("error trying to print duplicate vertex data " + LogFormatTools.getStackTop(dex)); } }// while - work on each group of dupes @@ -1259,10 +1398,14 @@ public class DataGrooming { + "] and investigate delete candidates. "); } } catch (AAIException e) { - LOGGER.error("Caught AAIException while grooming data", e); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("Caught AAIException while grooming data"); ErrorLogHelper.logException(e); } catch (Exception ex) { - LOGGER.error("Caught exception while grooming data", ex); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("Caught exception while grooming data"); ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming"); } finally { @@ -1270,6 +1413,8 @@ public class DataGrooming { try { bw.close(); } catch (IOException iox) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox); } } @@ -1284,6 +1429,8 @@ public class DataGrooming { g.tx().rollback(); } catch (Exception ex) { // Don't throw anything because Titan sometimes is just saying that the graph is already closed + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); LOGGER.warn("WARNING from final graphTransaction.rollback()", ex); } } @@ -1295,6 +1442,8 @@ public class DataGrooming { g2.tx().rollback(); } catch (Exception ex) { // Don't throw anything because Titan sometimes is just saying that the graph is already closed + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex); } } @@ -1307,6 +1456,8 @@ public class DataGrooming { } } catch (Exception ex) { // Don't throw anything because Titan sometimes is just saying that the graph is already closed{ + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); LOGGER.warn("WARNING from final graph.shutdown()", ex); } @@ -1317,6 +1468,8 @@ public class DataGrooming { } } catch (Exception ex) { // Don't throw anything because Titan sometimes is just saying that the graph is already closed{ + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); LOGGER.warn("WARNING from final graph2.shutdown()", ex); } } @@ -1549,6 +1702,7 @@ public class DataGrooming { // (We'll check dep-node later) // Determine what the key fields are for this nodeType Collection <String> keyProps = new ArrayList <>(); + HashMap <String,Object> keyPropValsHash = new HashMap <String,Object>(); try { keyProps = loader.introspectorFromName(vtxANodeType).getKeys(); } catch (AAIUnknownObjectException e) { @@ -1576,6 +1730,11 @@ public class DataGrooming { // data - so don't pick one return nullVtx; } + else { + // Keep these around for (potential) use later + keyPropValsHash.put(propName, vtxAKeyPropVal); + } + } // Collect the vid's and aai-node-types of the vertices that each vertex @@ -1618,18 +1777,20 @@ public class DataGrooming { } // 1 - If this kind of node needs a dependent node for uniqueness, then - // verify that they both nodes - // point to the same dependent node (otherwise they're not really - // duplicates) + // verify that they both nodes point to the same dependent + // node (otherwise they're not really duplicates) // Note - there are sometimes more than one dependent node type since - // one nodeType can be used in - // different ways. But for a particular node, it will only have one - // dependent node that it's - // connected to. - Collection <String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn(); - + // one nodeType can be used in different ways. But for a + // particular node, it will only have one dependent node that + // it's connected to. + String onlyNodeThatIndexPointsToVidStr = ""; + Collection<String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn(); if (depNodeTypes.isEmpty()) { // This kind of node is not dependent on any other. That is ok. + // We need to find out if the unique index info is good or not and + // use that later when deciding if we can delete one. + onlyNodeThatIndexPointsToVidStr = findJustOneUsingIndex( transId, + fromAppId, g, keyPropValsHash, vtxANodeType, vidA, vidB, ver ); } else { String depNodeVtxId4A = ""; String depNodeVtxId4B = ""; @@ -1654,8 +1815,11 @@ public class DataGrooming { } if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) { - // 2 - If they both have edges to all the same vertices, then return - // the one with the lower vertexId. + // 2 - If they both have edges to all the same vertices, + // then return the one that can be reached uniquely via the + // key if that is the case or + // else the one with the lower vertexId + boolean allTheSame = true; Iterator<String> iter = vtxIdsConn2A.iterator(); while (iter.hasNext()) { @@ -1667,7 +1831,19 @@ public class DataGrooming { } if (allTheSame) { - if (vidA < vidB) { + // If everything is the same, but one of the two has a good + // pointer to it, then save that one. Otherwise, take the + // older one. + if( !onlyNodeThatIndexPointsToVidStr.equals("") ){ + // only one is reachable via the index - choose that one. + if( onlyNodeThatIndexPointsToVidStr.equals(vidA.toString()) ){ + preferredVtx = vtxA; + } + else if( onlyNodeThatIndexPointsToVidStr.equals(vidB.toString()) ){ + preferredVtx = vtxB; + } + } + else if (vidA < vidB) { preferredVtx = vtxA; } else { preferredVtx = vtxB; @@ -1675,7 +1851,8 @@ public class DataGrooming { } } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) { // 3 - VertexA is connected to more things than vtxB. - // We'll pick VtxA if its edges are a superset of vtxB's edges. + // We'll pick VtxA if its edges are a superset of vtxB's edges + // and it doesn't contradict the check for the index/key pointer. boolean missingOne = false; Iterator<String> iter = vtxIdsConn2B.iterator(); while (iter.hasNext()) { @@ -1686,11 +1863,15 @@ public class DataGrooming { } } if (!missingOne) { - preferredVtx = vtxA; + if( onlyNodeThatIndexPointsToVidStr.equals("") + || onlyNodeThatIndexPointsToVidStr.equals(vidA.toString()) ){ + preferredVtx = vtxA; + } } } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) { // 4 - VertexB is connected to more things than vtxA. - // We'll pick VtxB if its edges are a superset of vtxA's edges. + // We'll pick VtxB if its edges are a superset of vtxA's edges + // and it doesn't contradict the check for the index/key pointer. boolean missingOne = false; Iterator<String> iter = vtxIdsConn2A.iterator(); while (iter.hasNext()) { @@ -1701,7 +1882,10 @@ public class DataGrooming { } } if (!missingOne) { - preferredVtx = vtxB; + if( onlyNodeThatIndexPointsToVidStr.equals("") + || onlyNodeThatIndexPointsToVidStr.equals(vidB.toString()) ){ + preferredVtx = vtxB; + } } } else { preferredVtx = nullVtx; @@ -1724,7 +1908,6 @@ public class DataGrooming { * @param deleteCandidateList the delete candidate list * @param singleCommits the single commits * @param alreadyFoundDupeGroups the already found dupe groups - * @param dbMaps the db maps * @return the array list */ private static List<String> checkAndProcessDupes(String transId, @@ -1880,6 +2063,8 @@ public class DataGrooming { } } } catch (Exception e) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e); } @@ -1896,7 +2081,6 @@ public class DataGrooming { * @param version the version * @param nType the n type * @param passedVertList the passed vert list - * @param dbMaps the db maps * @return the hash map * @throws AAIException the AAI exception */ @@ -2007,6 +2191,8 @@ public class DataGrooming { // like, "KeepVid=12345" String[] prefArr = prefString.split("="); if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.error("Bad format. Expecting KeepVid=999999"); return false; } else { @@ -2035,7 +2221,9 @@ public class DataGrooming { } } catch (Exception e) { okFlag = false; - LOGGER.error("ERROR trying to delete VID = " + thisVid, e); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error("ERROR trying to delete VID = " + thisVid + " " + LogFormatTools.getStackTop(e)); } if (okFlag) { LOGGER.info(" DELETED VID = " + thisVid); @@ -2044,6 +2232,8 @@ public class DataGrooming { } } } else { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = [" + dupeInfoString + "]"); return false; @@ -2123,7 +2313,9 @@ public class DataGrooming { } } catch( Exception ex ){ - LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]", ex); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); + LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]" + LogFormatTools.getStackTop(ex)); } if( verts != null ){ @@ -2290,7 +2482,7 @@ public class DataGrooming { Vertex parentVtx = null; Iterator <Vertex> vertI = g.V(startVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV()); - + while( vertI != null && vertI.hasNext() ){ // Note - there better only be one! parentVtx = vertI.next(); @@ -2317,4 +2509,146 @@ public class DataGrooming { } // End of figureWindowStartTime() + /** + * Collect Duplicate Sets for nodes that are NOT dependent on parent nodes. + * + * @param transId the trans id + * @param fromAppId the from app id + * @param g the g + * @param version the version + * @param nType the n type + * @param passedVertList the passed vert list + * @return the array list + */ + private static ArrayList<ArrayList<Vertex>> getDupeSets4NonDepNodes( String transId, + String fromAppId, Graph g, String version, String nType, + ArrayList<Vertex> passedVertList, + ArrayList <String> keyPropNamesArr, + Loader loader ) { + + ArrayList<ArrayList<Vertex>> returnList = new ArrayList<ArrayList<Vertex>>(); + + // We've been passed a set of nodes that we want to check. + // They are all NON-DEPENDENT nodes of the same nodeType meaning that they should be + // unique in the DB based on their KEY DATA alone. So, if + // we group them by their key data - if any key has more than one + // vertex mapped to it, those vertices are dupes. + // + // When we find duplicates, we group them in an ArrayList (there can be + // more than one duplicate for one set of key data) + // Then these dupeSets are grouped up and returned. + // + + HashMap <String, ArrayList<String>> keyVals2VidHash = new HashMap <String, ArrayList<String>>(); + HashMap <String,Vertex> vtxHash = new HashMap <String,Vertex>(); + Iterator<Vertex> pItr = passedVertList.iterator(); + while (pItr.hasNext()) { + try { + Vertex tvx = pItr.next(); + String thisVid = tvx.id().toString(); + vtxHash.put(thisVid, tvx); + + // if there are more than one vertexId mapping to the same keyProps -- they are dupes + // we dont check till later since a set can contain more than 2. + String hKey = getNodeKeyValString( tvx, keyPropNamesArr ); + if( keyVals2VidHash.containsKey(hKey) ){ + // We've already seen this key + ArrayList <String> tmpVL = (ArrayList <String>)keyVals2VidHash.get(hKey); + tmpVL.add(thisVid); + keyVals2VidHash.put(hKey, tmpVL); + } + else { + // First time for this key + ArrayList <String> tmpVL = new ArrayList <String>(); + tmpVL.add(thisVid); + keyVals2VidHash.put(hKey, tmpVL); + } + } + catch (Exception e) { + LOGGER.warn(" >>> Threw an error in getDupeSets4NonDepNodes - just absorb this error and move on. ", e); + } + } + + for( Map.Entry<String, ArrayList<String>> entry : keyVals2VidHash.entrySet() ){ + ArrayList <String> vidList = entry.getValue(); + try { + if( !vidList.isEmpty() && vidList.size() > 1 ){ + // There are more than one vertex id's using the same key info + ArrayList <Vertex> vertList = new ArrayList <Vertex> (); + for (int i = 0; i < vidList.size(); i++) { + String tmpVid = vidList.get(i); + vertList.add(vtxHash.get(tmpVid)); + } + returnList.add(vertList); + } + } + catch (Exception e) { + LOGGER.warn(" >>> Threw an error in getDupeSets4NonDepNodes - just absorb this error and move on. ", e); + } + + } + return returnList; + + }// End of getDupeSets4NonDepNodes() + + + /** + * Get values of the key properties for a node as a single string + * + * @param tvx the vertex to pull the properties from + * @param keyPropNamesArr collection of key prop names + * @return a String of concatenated values + */ + private static String getNodeKeyValString( Vertex tvx, + ArrayList <String> keyPropNamesArr ) { + + String retString = ""; + Iterator <String> propItr = keyPropNamesArr.iterator(); + while( propItr.hasNext() ){ + String propName = propItr.next(); + if( tvx != null ){ + Object propValObj = tvx.property(propName).orElse(null); + retString = " " + retString + propValObj.toString(); + } + } + return retString; + + }// End of getNodeKeyValString() + + + static private String findJustOneUsingIndex( String transId, String fromAppId, + GraphTraversalSource gts, HashMap <String,Object> keyPropValsHash, + String nType, Long vidAL, Long vidBL, String apiVer){ + + // See if querying by JUST the key params (which should be indexed) brings back + // ONLY one of the two vertices. Ie. the db still has a pointer to one of them + // and the other one is sort of stranded. + String returnVid = ""; + + try { + List <Vertex> tmpVertList = getNodeJustUsingKeyParams( transId, fromAppId, gts, + nType, keyPropValsHash, apiVer ); + if( tmpVertList != null && tmpVertList.size() == 1 ){ + // We got just one - if it matches one of the ones we're looking + // for, then return that VID + Vertex tmpV = tmpVertList.get(0); + String thisVid = tmpV.id().toString(); + if( thisVid.equals(vidAL.toString()) || thisVid.equals(vidBL.toString()) ){ + String msg = " vid = " + thisVid + " is one of two that the DB can retrieve directly ------"; + //System.out.println(msg); + LOGGER.info(msg); + returnVid = thisVid; + } + } + } + catch ( AAIException ae ){ + String emsg = "Error trying to get node just by key " + ae.getMessage(); + //System.out.println(emsg); + LOGGER.error(emsg); + } + + return returnVid; + + }// End of findJustOneUsingIndex() + } diff --git a/aai-core/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java b/aai-core/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java new file mode 100644 index 00000000..52fc8f19 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java @@ -0,0 +1,498 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.dbgen; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Scanner; +import java.util.Set; +import java.util.UUID; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.dbmap.DBConnectionType; +import org.onap.aai.dbmap.InMemoryGraph; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; +import org.onap.aai.introspection.Version; +import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.logging.LogFormatTools; +import org.onap.aai.parsers.uri.URIToObject; +import org.onap.aai.serialization.engines.InMemoryDBEngine; +import org.onap.aai.serialization.engines.QueryStyle; +import org.onap.aai.serialization.db.DBSerializer; + +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; + +import org.slf4j.MDC; + +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.node.ObjectNode; +import org.codehaus.jackson.type.TypeReference; + +import java.util.Date; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import org.onap.aai.serialization.tinkerpop.TreeBackedVertex; +import org.onap.aai.util.AAIConfig; +import org.onap.aai.util.AAIConstants; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; + +/** + * The Class ListEndpoints. + */ +public class DynamicPayloadGenerator { + + /* + * Create a Dynamic memory graph instance which should not affect the + * AAIGraph + */ + private InMemoryGraph inMemGraph = null; + private InMemoryDBEngine dbEngine; + + /* + * Loader, QueryStyle, ConnectionType for the Serializer + */ + private Loader loader; + private String urlBase; + private BufferedWriter bw = null; + + private CommandLineArgs cArgs; + + private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DynamicPayloadGenerator.class); + + private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL; + private static final DBConnectionType type = DBConnectionType.CACHED; + private static final ModelType introspectorFactoryType = ModelType.MOXY; + + /* + * Version + */ + private static final Version version = Version.getLatest(); + + /** + * The main method. + * + * @param args + * the arguments + * @throws AAIException + * @throws Exception + */ + public static void main(String[] args) { + + MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName()); + DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator(); + try { + payloadgen.init(args); + + payloadgen.generatePayloads(); + } catch (AAIException e) { + LOGGER.error("Exception " + LogFormatTools.getStackTop(e)); + } catch (IOException e) { + LOGGER.error("Exception " + LogFormatTools.getStackTop(e)); + } + System.exit(0); + } + + public void init(String[] args) throws AAIException { + cArgs = new CommandLineArgs(); + JCommander jCommander = new JCommander(cArgs, args); + jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName()); + LOGGER.info("Snapshot file" + cArgs.dataSnapshot); + //TODO- How to add dynamic.properties + + + LOGGER.info("output file" + cArgs.output); + + + LOGGER.info("format file" + cArgs.format); + LOGGER.info("format file" + cArgs.schemaEnabled); + if(cArgs.config.isEmpty()) + cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties"; + + LOGGER.info("config file" + cArgs.config); + if(cArgs.nodePropertyFile.isEmpty()) + cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json"; + LOGGER.info("nodePropertyFile file" + cArgs.nodePropertyFile); + AAIConfig.init(); + + urlBase = AAIConfig.get("aai.server.url.base", ""); + + } + + public void generatePayloads() throws AAIException, IOException{ + + List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile); + LOGGER.info("Load the Graph"); + + this.loadGraph(); + LOGGER.info("Generate payload"); + this.generatePayload(nodeFilters); + LOGGER.info("Close graph"); + this.closeGraph(); + + } + + private List<Map<String, List<String>>> readFile(String inputFile) throws IOException { + + InputStream is = new FileInputStream(inputFile); + Scanner scanner = new Scanner(is); + String jsonFile = scanner.useDelimiter("\\Z").next(); + scanner.close(); + + List<Map<String, List<String>>> allNodes = new ArrayList<>(); + Map<String, List<String>> filterCousins = new HashMap<>(); + Map<String, List<String>> filterParents = new HashMap<>(); + + ObjectMapper mapper = new ObjectMapper(); + + JsonNode rootNode = mapper.readTree(jsonFile); + + Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields(); + + while (nodeFields.hasNext()) { + Entry<String, JsonNode> entry = nodeFields.next(); + String nodeType = entry.getKey(); + JsonNode nodeProperty = entry.getValue(); + + JsonNode cousinFilter = nodeProperty.path("cousins"); + JsonNode parentFilter = nodeProperty.path("parents"); + List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(), + new TypeReference<ArrayList<String>>() { + }); + + List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(), + new TypeReference<ArrayList<String>>() { + }); + for (String cousin : cousins) { + LOGGER.info("Cousins-Filtered" + cousin); + } + for (String parent : parents) { + LOGGER.info("Parents-Filtered" + parent); + } + filterCousins.put(nodeType, cousins); + filterParents.put(nodeType, parents); + + } + + allNodes.add(filterCousins); + allNodes.add(filterParents); + return allNodes; + + } + + private void loadGraph() throws IOException { + + loadGraphIntoMemory(); + buildDbEngine(); + + } + + private void loadGraphIntoMemory() throws IOException { + + inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled); + + } + + private void buildDbEngine() { + // TODO : parametrise version + loader = LoaderFactory.createLoaderForVersion(introspectorFactoryType, version); + + dbEngine = new InMemoryDBEngine(queryStyle, type, loader, inMemGraph.getGraph()); + dbEngine.startTransaction(); + } + + private void generatePayload(List<Map<String, List<String>>> nodeFilters) throws AAIException, IOException { + + Map<String, List<String>> filterCousinsMap = nodeFilters.get(0); + Map<String, List<String>> filterParentsMap = nodeFilters.get(1); + + Set<String> nodeTypes = filterCousinsMap.keySet(); + + for (String nodeType : nodeTypes) { + if ("DMAAP-MR".equals(cArgs.format)) { + bw = createFile(nodeType + ".json"); + } + List<String> filterCousins = filterCousinsMap.get(nodeType); + List<String> filterParents = filterParentsMap.get(nodeType); + readVertices(nodeType, filterCousins, filterParents); + bw.close(); + LOGGER.info("All Done-" + nodeType); + } + + } + + private BufferedWriter createFile(String outfileName) throws IOException { + // FileLocation + String fileName = outfileName; + + File outFile = new File(fileName); + LOGGER.info(" Will write to " + outFile); + FileWriter fw = new FileWriter(outFile.getAbsoluteFile()); + return new BufferedWriter(fw); + + } + + private void createDirectory(String dirName) throws IOException { + // FileLocation + Path pathDir = Paths.get(dirName); + Files.createDirectories(pathDir); + + } + + public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents) + throws AAIException, IOException { + + DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth"); + List<Vertex> nodes = inMemGraph.getGraph().traversal().V().has("aai-node-type", nodeType).toList(); + + LOGGER.info("Number of nodes" + nodes.size()); + String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP; + createDirectory( dirName); + // TODO: Formatter + if ("DMAAP-MR".equals(cArgs.format)) { + for (Vertex node : nodes) { + + Introspector nodeObj = serializer.getLatestVersionView(node); + createPayloadForDmaap(node, nodeObj); + + } + } + int counter = 0; + if ("PAYLOAD".equals(cArgs.format)) { + for (Vertex node : nodes) { + + counter++; + String filename = dirName + counter + "-" + nodeType + ".json"; + bw = createFile(filename); + Introspector obj = loader.introspectorFromName(nodeType); + Set<Vertex> seen = new HashSet<>(); + int depth = AAIProperties.MAXIMUM_DEPTH; + boolean nodeOnly = false; + + Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly); + TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree); + serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins, filterParents); + createPayloadForPut(obj); + bw.close(); + + URI uri = serializer.getURIForVertex(node); + String filenameWithUri = dirName + counter + "-" + nodeType + ".txt"; + bw = createFile(filenameWithUri); + bw.write(uri.toString()); + bw.newLine(); + bw.close(); + } + } + + } + + public void createPayloadForPut(Introspector nodeObj) throws IOException { + + String entityJson = nodeObj.marshal(false); + ObjectMapper mapper = new ObjectMapper(); + + ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson); + rootNode.remove("resource-version"); + + bw.newLine(); + bw.write(rootNode.toString()); + bw.newLine(); + } + + public void createPayloadForDmaap(Vertex node, Introspector nodeObj) + throws AAIException, UnsupportedEncodingException { + + DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth"); + + URI uri = serializer.getURIForVertex(node); + + String sourceOfTruth = ""; + HashMap<String, Introspector> relatedVertices = new HashMap<>(); + List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node); + + for (Vertex vertex : vertexChain) { + try { + + Introspector vertexObj = serializer.getVertexProperties(vertex); + + relatedVertices.put(vertexObj.getObjectId(), vertexObj); + } catch (AAIUnknownObjectException e) { + LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned"); + } + + } + + String transactionId = "TXID"; + createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices); + + } + + public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj, + Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException { + + String action = "CREATE"; + final Introspector notificationEvent = loader.introspectorFromName("notification-event"); + + try { + Introspector eventHeader = loader.introspectorFromName("notification-event-header"); + URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects); + + String entityLink = urlBase + version + uri; + + notificationEvent.setValue("cambria-partition", "AAI"); + + eventHeader.setValue("entity-link", entityLink); + eventHeader.setValue("action", action); + eventHeader.setValue("entity-type", obj.getDbName()); + eventHeader.setValue("top-entity-type", parser.getTopEntityName()); + eventHeader.setValue("source-name", sourceOfTruth); + eventHeader.setValue("version", version.toString()); + eventHeader.setValue("id", transactionId); + eventHeader.setValue("event-type", "AAI-BASELINE"); + if (eventHeader.getValue("domain") == null) { + eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK")); + } + + if (eventHeader.getValue("sequence-number") == null) { + eventHeader.setValue("sequence-number", + AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK")); + } + + if (eventHeader.getValue("severity") == null) { + eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK")); + } + + if (eventHeader.getValue("id") == null) { + eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString()); + + } + + if (eventHeader.getValue("timestamp") == null) { + eventHeader.setValue("timestamp", genDate()); + } + + List<Object> parentList = parser.getParentList(); + parentList.clear(); + + if (!parser.getTopEntity().equals(parser.getEntity())) { + Introspector child; + String json = obj.marshal(false); + child = parser.getLoader().unmarshal(parser.getEntity().getName(), json); + parentList.add(child.getUnderlyingObject()); + } + + final Introspector eventObject; + + String json = ""; + if (parser.getTopEntity().equals(parser.getEntity())) { + json = obj.marshal(false); + eventObject = loader.unmarshal(obj.getName(), json); + } else { + json = parser.getTopEntity().marshal(false); + + eventObject = loader.unmarshal(parser.getTopEntity().getName(), json); + } + notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject()); + notificationEvent.setValue("entity", eventObject.getUnderlyingObject()); + + String entityJson = notificationEvent.marshal(false); + + bw.newLine(); + bw.write(entityJson); + + } catch (AAIUnknownObjectException e) { + LOGGER.error("Fatal error - notification-event-header object not found!"); + } catch (Exception e) { + LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e)); + } + } + + private void closeGraph() { + inMemGraph.getGraph().tx().rollback(); + inMemGraph.getGraph().close(); + } + + public static String genDate() { + Date date = new Date(); + DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS"); + return formatter.format(date); + } + + public static String genDate2() { + Date date = new Date(); + DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss"); + return formatter.format(date); + } + +} + +class CommandLineArgs { + + @Parameter(names = "--help", help = true) + public boolean help; + + @Parameter(names = "-d", description = "snapshot file to be loaded") + public String dataSnapshot; + + @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1) + public boolean schemaEnabled = true; + + @Parameter(names = "-c", description = "location of configuration file") + public String config = ""; + + @Parameter(names = "-o", description = "output location") + public String output = ""; + + @Parameter(names = "-f", description = "format of output") + public String format = "PAYLOAD"; + + @Parameter(names = "-n", description = "Node input file") + public String nodePropertyFile = ""; + +} diff --git a/aai-core/src/main/java/org/onap/aai/dbgen/GenTester.java b/aai-core/src/main/java/org/onap/aai/dbgen/GenTester.java index 7b9eb2fc..81672949 100644 --- a/aai-core/src/main/java/org/onap/aai/dbgen/GenTester.java +++ b/aai-core/src/main/java/org/onap/aai/dbgen/GenTester.java @@ -28,10 +28,13 @@ import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.schema.TitanManagement; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LoggingContext.StatusCode; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; import java.util.Properties; +import java.util.UUID; public class GenTester { @@ -46,7 +49,7 @@ public class GenTester { public static void main(String[] args) { TitanGraph graph = null; - + System.setProperty("aai.service.name", GenTester.class.getSimpleName()); // Set the logging file properties to be used by EELFManager Properties props = System.getProperties(); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_CREATE_DB_SCHEMA_LOGBACK_PROPS); @@ -54,7 +57,16 @@ public class GenTester { LOGGER = EELFManager.getInstance().getLogger(GenTester.class); boolean addDefaultCR = true; - try { + LoggingContext.init(); + LoggingContext.component("DBGenTester"); + LoggingContext.partnerName("AAI-TOOLS"); + LoggingContext.targetEntity("AAI"); + LoggingContext.requestId(UUID.randomUUID().toString()); + LoggingContext.serviceName("AAI"); + LoggingContext.targetServiceName("main"); + LoggingContext.statusCode(StatusCode.COMPLETE); + LoggingContext.responseCode(LoggingContext.SUCCESS); + try { AAIConfig.init(); if (args != null && args.length > 0 ){ if( "genDbRulesOnly".equals(args[0]) ){ @@ -88,6 +100,8 @@ public class GenTester { String emsg = "Unrecognized argument passed to GenTester.java: [" + args[0] + "]. "; System.out.println(emsg); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error(emsg); emsg = "Either pass no argument for normal processing, or use 'GEN_DB_WITH_NO_SCHEMA'."; diff --git a/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java b/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java index 132ad46a..e26ba2c2 100644 --- a/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java +++ b/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java @@ -34,6 +34,7 @@ import org.onap.aai.introspection.Introspector; import org.onap.aai.introspection.Loader; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.introspection.ModelType; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.schema.enums.PropertyMetadata; import org.onap.aai.serialization.db.EdgeRule; import org.onap.aai.serialization.db.EdgeRules; @@ -80,7 +81,7 @@ public class SchemaGenerator{ AAIConfig.init(); } catch (Exception ex){ - LOGGER.error(" ERROR - Could not run AAIConfig.init(). ", ex); + LOGGER.error(" ERROR - Could not run AAIConfig.init(). " + LogFormatTools.getStackTop(ex)); System.out.println(" ERROR - Could not run AAIConfig.init(). "); System.exit(1); } diff --git a/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java b/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java index e62e8e7e..5c7abfe2 100644 --- a/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java +++ b/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.io.IoCore; @@ -58,8 +59,9 @@ public class AAIGraph { private static final EELFLogger logger = EELFManager.getInstance().getLogger(AAIGraph.class); protected static final String COMPONENT = "aaidbmap"; protected Map<String, TitanGraph> graphs = new HashMap<>(); - private final String REALTIME_DB = "realtime"; - private final String CACHED_DB = "cached"; + private static final String REALTIME_DB = "realtime"; + private static final String CACHED_DB = "cached"; + private static boolean isInit = false; @@ -68,6 +70,7 @@ public class AAIGraph { */ private AAIGraph() { try { + String serviceName = System.getProperty("aai.service.name", "NA"); String rtConfig = System.getProperty("realtime.db.config"); String cachedConfig = System.getProperty("cached.db.config"); if (rtConfig == null) { @@ -76,8 +79,8 @@ public class AAIGraph { if (cachedConfig == null) { cachedConfig = AAIConstants.CACHED_DB_CONFIG; } - this.loadGraph(REALTIME_DB, rtConfig); - this.loadGraph(CACHED_DB, cachedConfig); + this.loadGraph(REALTIME_DB, rtConfig, serviceName); + this.loadGraph(CACHED_DB, cachedConfig, serviceName); } catch (Exception e) { throw new RuntimeException("Failed to instantiate graphs", e); } @@ -93,17 +96,23 @@ public class AAIGraph { * @return single instance of AAIGraph */ public static AAIGraph getInstance() { + isInit = true; return Helper.INSTANCE; } + + public static boolean isInit() { + return isInit; + } - private void loadGraph(String name, String configPath) throws AAIException { + private void loadGraph(String name, String configPath, String serviceName) throws Exception { // Graph being opened by TitanFactory is being placed in hashmap to be used later // These graphs shouldn't be closed until the application shutdown - TitanGraph graph = TitanFactory.open(configPath); - try (InputStream is = new FileInputStream(configPath)) { + try { + PropertiesConfiguration propertiesConfiguration = new AAIGraphConfig.Builder(configPath).forService(serviceName).withGraphType(name).buildConfiguration(); + TitanGraph graph = TitanFactory.open(propertiesConfiguration); Properties graphProps = new Properties(); - graphProps.load(is); + propertiesConfiguration.getKeys().forEachRemaining(k -> graphProps.setProperty(k, propertiesConfiguration.getString(k))); if ("inmemory".equals(graphProps.get("storage.backend"))) { // Load the propertyKeys, indexes and edge-Labels into the DB @@ -155,10 +164,10 @@ public class AAIGraph { } /** - * Graph shutdown. + * Close all of the graph connections made in the instance. */ public void graphShutdown() { - graphs.get(REALTIME_DB).close(); + graphs.values().stream().filter(TitanGraph::isOpen).forEach(TitanGraph::close); } /** diff --git a/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraphConfig.java b/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraphConfig.java new file mode 100644 index 00000000..e67051f0 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraphConfig.java @@ -0,0 +1,107 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ + +package org.onap.aai.dbmap; + +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterators; +import com.thinkaurelius.titan.diskstorage.configuration.ConfigElement; +import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.commons.lang.StringUtils; + +import java.io.File; +import java.io.FileNotFoundException; +import java.lang.management.ManagementFactory; +import java.util.Iterator; +import java.util.Objects; +import java.util.regex.Pattern; + +import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.*; + +/** + * For building a config that TitanFactory.open can use with an identifiable graph.unique-instance-id + */ +public class AAIGraphConfig { + + private static final EELFLogger logger = EELFManager.getInstance().getLogger(AAIGraphConfig.class); + + private AAIGraphConfig(){}; + + public PropertiesConfiguration getCc(String configPath, String graphType, String service) throws ConfigurationException, FileNotFoundException { + + PropertiesConfiguration cc = this.loadTitanPropFile(configPath); + + String uid = ManagementFactory.getRuntimeMXBean().getName() + "_" + service + "_" + graphType + "_" + System.currentTimeMillis(); + for (char c : ConfigElement.ILLEGAL_CHARS) { + uid = StringUtils.replaceChars(uid, c,'_'); + } + + cc.addProperty("graph.unique-instance-id", uid); + + return cc; + } + + + private PropertiesConfiguration loadTitanPropFile(String shortcutOrFile) throws ConfigurationException, FileNotFoundException { + File file = new File(shortcutOrFile); + if (file.exists()) { + PropertiesConfiguration propertiesConfiguration = new PropertiesConfiguration(); + propertiesConfiguration.setAutoSave(false); + propertiesConfiguration.load(shortcutOrFile); + return propertiesConfiguration; + } else { + throw new FileNotFoundException(shortcutOrFile); + } + } + + public static class Builder { + private String configPath; + private String graphType; + private String service; + + public Builder(String configPath) { + this.configPath = configPath; + } + + public Builder withGraphType(String graphType) { + this.graphType = Objects.toString(graphType, "NA"); + return this; + } + + public Builder forService(String service) { + this.service = Objects.toString(service, "NA"); + return this; + } + + public PropertiesConfiguration buildConfiguration() throws ConfigurationException, FileNotFoundException { + return new AAIGraphConfig().getCc(this.configPath, this.graphType, this.service); + } + + } + + +} diff --git a/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java b/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java new file mode 100644 index 00000000..61b4e36e --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java @@ -0,0 +1,106 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.dbmap; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +import java.util.Properties; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.tinkerpop.gremlin.structure.io.IoCore; +import org.onap.aai.dbgen.SchemaGenerator; +import org.onap.aai.logging.LogFormatTools; + +import com.thinkaurelius.titan.core.TitanFactory; +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.TitanTransaction; +import com.thinkaurelius.titan.core.schema.TitanManagement; + +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; + +public class InMemoryGraph { + + private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(InMemoryGraph.class); + private TitanGraph graph = null; + + + public InMemoryGraph(Builder builder) throws IOException { + /* + * Create a In-memory graph + */ + InputStream is = new FileInputStream(builder.propertyFile); + try { + graph = TitanFactory.open(builder.propertyFile); + + Properties graphProps = new Properties(); + graphProps.load(is); + TitanManagement graphMgt = graph.openManagement(); + if(builder.isSchemaEnabled){ + LOGGER.info("Schema Enabled"); + SchemaGenerator.loadSchemaIntoTitan(graph, graphMgt); + } + TitanTransaction transaction = graph.newTransaction(); + LOGGER.info("Loading snapshot"); + transaction.io(IoCore.graphson()).readGraph(builder.graphsonLocation); + transaction.commit(); + + } catch (Exception e) { + // TODO : Changesysout to logger + LOGGER.error( + "ERROR: Could not load datasnapshot to in memory graph. \n" + LogFormatTools.getStackTop(e)); + throw new IllegalStateException("Could not load datasnapshot to in memory graph"); + + } + finally{ + is.close(); + } + + } + + public static class Builder { + private String graphsonLocation = ""; + private String propertyFile = ""; + private boolean isSchemaEnabled = false; + + /* + * Builder constructor doesnt do anything + */ + public Builder() { + //Do nothing + } + + public InMemoryGraph build(String graphsonFile, String propertyFile, boolean isSchemaEnabled) throws IOException { + this.graphsonLocation = graphsonFile; + this.propertyFile = propertyFile; + this.isSchemaEnabled = isSchemaEnabled; + return new InMemoryGraph(this); + } + } + + public TitanGraph getGraph() { + return graph; + } + +} diff --git a/aai-core/src/main/java/org/onap/aai/dmaap/AAIDmaapEventJMSConsumer.java b/aai-core/src/main/java/org/onap/aai/dmaap/AAIDmaapEventJMSConsumer.java index d5389981..e52cc3ed 100644 --- a/aai-core/src/main/java/org/onap/aai/dmaap/AAIDmaapEventJMSConsumer.java +++ b/aai-core/src/main/java/org/onap/aai/dmaap/AAIDmaapEventJMSConsumer.java @@ -104,7 +104,16 @@ public class AAIDmaapEventJMSConsumer implements MessageListener { eventName = jo.getString("event-topic"); } + MDC.put ("targetEntity", "DMAAP"); + if (jo.getString("event-topic") != null) { + eventName = jo.getString("event-topic"); + MDC.put ("targetServiceName", eventName); + } + MDC.put ("serviceName", "AAI"); + MDC.put(LoggingField.STATUS_CODE.toString(), StatusCode.COMPLETE.toString()); + MDC.put(LoggingField.RESPONSE_CODE.toString(), "0"); LOGGER.info(eventName + "|" + aaiEvent); + if ("AAI-EVENT".equals(eventName)) { this.sentWithHttp(this.httpClient, this.aaiEventUrl, aaiEvent); } else { @@ -113,13 +122,21 @@ public class AAIDmaapEventJMSConsumer implements MessageListener { } } catch (java.net.SocketException e) { if (!e.getMessage().contains("Connection reset")) { + MDC.put(LoggingField.STATUS_CODE.toString(), StatusCode.ERROR.toString()); + MDC.put(LoggingField.RESPONSE_CODE.toString(), "200"); LOGGER.error("AAI_7304 Error reaching DMaaP to send event. " + aaiEvent, e); } } catch (IOException e) { + MDC.put(LoggingField.STATUS_CODE.toString(), StatusCode.ERROR.toString()); + MDC.put(LoggingField.RESPONSE_CODE.toString(), "200"); LOGGER.error("AAI_7304 Error reaching DMaaP to send event. " + aaiEvent, e); } catch (JMSException | JSONException e) { + MDC.put(LoggingField.STATUS_CODE.toString(), StatusCode.ERROR.toString()); + MDC.put(LoggingField.RESPONSE_CODE.toString(), "200"); LOGGER.error("AAI_7350 Error parsing aaievent jsm message for sending to dmaap. " + jsmMessageTxt, e); } catch (Exception e) { + MDC.put(LoggingField.STATUS_CODE.toString(), StatusCode.ERROR.toString()); + MDC.put(LoggingField.RESPONSE_CODE.toString(), "200"); LOGGER.error("AAI_7350 Error sending message to dmaap. " + jsmMessageTxt, e); } } diff --git a/aai-core/src/main/java/org/onap/aai/dmaap/AAIDmaapEventJMSProducer.java b/aai-core/src/main/java/org/onap/aai/dmaap/AAIDmaapEventJMSProducer.java index d41f4768..914042d4 100644 --- a/aai-core/src/main/java/org/onap/aai/dmaap/AAIDmaapEventJMSProducer.java +++ b/aai-core/src/main/java/org/onap/aai/dmaap/AAIDmaapEventJMSProducer.java @@ -35,7 +35,8 @@ public class AAIDmaapEventJMSProducer { public AAIDmaapEventJMSProducer() { if(AAIConfig.get("aai.jms.enable", "true").equals("true")){ this.jmsTemplate = new JmsTemplate(); - this.jmsTemplate.setConnectionFactory(new CachingConnectionFactory(new ActiveMQConnectionFactory("tcp://localhost:61447"))); + String activeMqTcpUrl = System.getProperty("activemq.tcp.url", "tcp://localhost:61447"); + this.jmsTemplate.setConnectionFactory(new CachingConnectionFactory(new ActiveMQConnectionFactory(activeMqTcpUrl))); this.jmsTemplate.setDefaultDestination(new ActiveMQQueue("IN_QUEUE")); } } diff --git a/aai-core/src/main/java/org/onap/aai/extensions/OrphanLInterfaceHandler.java b/aai-core/src/main/java/org/onap/aai/extensions/OrphanLInterfaceHandler.java new file mode 100644 index 00000000..3da4e227 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/extensions/OrphanLInterfaceHandler.java @@ -0,0 +1,112 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.extensions; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.parsers.query.QueryParser; +import org.onap.aai.query.builder.QueryBuilder; +import org.onap.aai.rest.db.DBRequest; +import org.onap.aai.restcore.HttpMethod; +import org.onap.aai.serialization.db.EdgeType; + +public class OrphanLInterfaceHandler { + + private QueryBuilder<Vertex> createLInterfaceQuery(AAIExtensionMap aaiReqMap, Introspector newvceObj) throws AAIException { + Introspector uplinkLInterfaceTraversalIntro = aaiReqMap.getLoader().introspectorFromName("l-interface"); + + Introspector customerUplinkLInterfaceTraversalIntro = aaiReqMap.getLoader().introspectorFromName("l-interface"); + + Introspector logLinkIntroForTraversal = aaiReqMap.getLoader().introspectorFromName("logical-link"); + + QueryBuilder<Vertex> query = aaiReqMap.getTransactionalGraphEngine().getQueryBuilder() + .exactMatchQuery(newvceObj) + .createEdgeTraversal(EdgeType.TREE, newvceObj, uplinkLInterfaceTraversalIntro) + .getVerticesByProperty("interface-role", "UPLINK") + .createEdgeTraversal(EdgeType.COUSIN, uplinkLInterfaceTraversalIntro, logLinkIntroForTraversal) + .createEdgeTraversal(EdgeType.COUSIN, logLinkIntroForTraversal, customerUplinkLInterfaceTraversalIntro) + .getVerticesByProperty("interface-role", "CUSTOMER-UPLINK").dedup(); + return query; + } + + private URI buildLInterfaceURI(Vertex linterface, AAIExtensionMap aaiReqMap) throws UnsupportedEncodingException, AAIException, URISyntaxException { + Loader loader = aaiReqMap.getLoader(); + Introspector lint = loader.introspectorFromName("l-interface"); + lint.setValue("interface-name", (String)linterface.property("interface-name").value()); + String lintSegment = lint.getURI(); + + Introspector lagInterfaceForTrav = loader.introspectorFromName("lag-interface"); + QueryBuilder<Vertex> lagIntQuery = aaiReqMap.getTransactionalGraphEngine().getQueryBuilder() + .exactMatchQuery(lint) + .createEdgeTraversal(EdgeType.TREE, linterface, lagInterfaceForTrav).dedup(); + List<Vertex> lagInterfaces = lagIntQuery.toList(); + if (lagInterfaces.isEmpty()) { + throw new AAIException("AAI_6114"); + } else if (lagInterfaces.size() > 1) { + throw new AAIException("AAI_6140"); + } + Vertex lagInt = lagInterfaces.get(0); + lagInterfaceForTrav.setValue("interface-name", (String)lagInt.property("interface-name").value()); + String lagSegment = lagInterfaceForTrav.getURI(); + + Introspector gvVPEforTrav = loader.introspectorFromName("generic-vnf"); + QueryBuilder<Vertex> gvVPEquery = aaiReqMap.getTransactionalGraphEngine().getQueryBuilder() + .exactMatchQuery(lagInterfaceForTrav) + .createEdgeTraversal(EdgeType.TREE, lagInterfaceForTrav, gvVPEforTrav).dedup(); + List<Vertex> genvnfs = gvVPEquery.toList(); + if (genvnfs.isEmpty()) { + throw new AAIException("AAI_6114"); + } else if (genvnfs.size() > 1) { + throw new AAIException("AAI_6140"); + } + Vertex genvnf = genvnfs.get(0); + gvVPEforTrav.setValue("vnf-id", (String)genvnf.property("vnf-id").value()); + String gvSegment = gvVPEforTrav.getURI(); + + return new URI(gvSegment + lagSegment + lintSegment); + } + + public List<DBRequest> createOrphanLInterfaceDelRequests(AAIExtensionMap aaiReqMap, Introspector newvce) throws AAIException, UnsupportedEncodingException, URISyntaxException{ + List<DBRequest> requests = new ArrayList<>(); + QueryBuilder<Vertex> query = createLInterfaceQuery(aaiReqMap, newvce); + List<Vertex> linterfaces = query.toList(); + + for (Vertex lint : linterfaces) { + URI lintURI = buildLInterfaceURI(lint, aaiReqMap); + QueryParser parser = createLInterfaceQuery(aaiReqMap, newvce).createQueryFromObjectName("l-interface"); + DBRequest originalDbRequest = aaiReqMap.getDbRequest(); + DBRequest request = new DBRequest.Builder(HttpMethod.DELETE, lintURI, parser, newvce, originalDbRequest.getHeaders(), originalDbRequest.getInfo(), originalDbRequest.getTransactionId()).build(); + requests.add(request); + } + + return requests; + } +} diff --git a/aai-core/src/main/java/org/onap/aai/introspection/Introspector.java b/aai-core/src/main/java/org/onap/aai/introspection/Introspector.java index f6609bff..fa8be863 100644 --- a/aai-core/src/main/java/org/onap/aai/introspection/Introspector.java +++ b/aai-core/src/main/java/org/onap/aai/introspection/Introspector.java @@ -28,6 +28,7 @@ import org.apache.commons.lang.ClassUtils; import org.eclipse.persistence.exceptions.DynamicException; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.restcore.MediaType; import org.onap.aai.schema.enums.ObjectMetadata; import org.onap.aai.schema.enums.PropertyMetadata; @@ -465,7 +466,7 @@ public abstract class Introspector implements Cloneable { break; } } catch (AAIUnknownObjectException e) { - LOGGER.warn("Skipping inheritor " + inheritor + " (Unknown Object)", e); + LOGGER.warn("Skipping inheritor " + inheritor + " (Unknown Object) " + LogFormatTools.getStackTop(e)); } } } else { diff --git a/aai-core/src/main/java/org/onap/aai/introspection/IntrospectorWalker.java b/aai-core/src/main/java/org/onap/aai/introspection/IntrospectorWalker.java index 09dcec64..e939b6d9 100644 --- a/aai-core/src/main/java/org/onap/aai/introspection/IntrospectorWalker.java +++ b/aai-core/src/main/java/org/onap/aai/introspection/IntrospectorWalker.java @@ -25,6 +25,7 @@ import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.logging.LogFormatTools; import java.util.HashSet; import java.util.LinkedHashSet; @@ -141,7 +142,7 @@ public class IntrospectorWalker { walk(child, parent, localVisited); } } catch (AAIUnknownObjectException e) { - LOGGER.warn("Skipping property " + prop + " (Unknown Object)", e); + LOGGER.warn("Skipping property " + prop + " (Unknown Object) " + LogFormatTools.getStackTop(e)); } } else { w.processPrimitiveList(prop, obj); @@ -159,7 +160,7 @@ public class IntrospectorWalker { obj.setValue(prop, listReference); } catch (AAIUnknownObjectException e) { - LOGGER.warn("Skipping property " + prop + " (Unknown Object)", e); + LOGGER.warn("Skipping property " + prop + " (Unknown Object) " + LogFormatTools.getStackTop(e)); } } else if (!isComplexType){ w.processPrimitiveList(prop, obj); @@ -176,7 +177,7 @@ public class IntrospectorWalker { child = obj.newIntrospectorInstanceOfProperty(prop); obj.setValue(prop, child.getUnderlyingObject()); } catch (AAIUnknownObjectException e) { - LOGGER.warn("Skipping property " + prop + " (Unknown Object)", e); + LOGGER.warn("Skipping property " + prop + " (Unknown Object) " + LogFormatTools.getStackTop(e)); } } } diff --git a/aai-core/src/main/java/org/onap/aai/introspection/ModelInjestor.java b/aai-core/src/main/java/org/onap/aai/introspection/ModelInjestor.java index 4cdfc634..c670e946 100644 --- a/aai-core/src/main/java/org/onap/aai/introspection/ModelInjestor.java +++ b/aai-core/src/main/java/org/onap/aai/introspection/ModelInjestor.java @@ -105,7 +105,7 @@ public class ModelInjestor { */ public Version getVersionFromClassName (String classname) { Matcher m = classNamePattern.matcher(classname); - String version = "v2"; //for the OXM, only the v2 ones don't include a model name, hence this default + String version = "v12"; if (m.find()) { version = m.group(1); } diff --git a/aai-core/src/main/java/org/onap/aai/introspection/MoxyLoader.java b/aai-core/src/main/java/org/onap/aai/introspection/MoxyLoader.java index cf81349c..56ad2dc2 100644 --- a/aai-core/src/main/java/org/onap/aai/introspection/MoxyLoader.java +++ b/aai-core/src/main/java/org/onap/aai/introspection/MoxyLoader.java @@ -32,6 +32,7 @@ import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; import org.onap.aai.introspection.exceptions.AAIUnmarshallingException; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.restcore.MediaType; import org.onap.aai.workarounds.NamingExceptions; import org.w3c.dom.Document; @@ -157,7 +158,7 @@ public class MoxyLoader extends Loader { Introspector introspector = this.introspectorFromName(objName); map.put(introspector.getDbName(), introspector); } catch (AAIUnknownObjectException e) { - LOGGER.warn("Unexpected AAIUnknownObjectException while running getAllObjects()", e); + LOGGER.warn("Unexpected AAIUnknownObjectException while running getAllObjects() " + LogFormatTools.getStackTop(e)); } } allObjs = map.build(); @@ -182,7 +183,7 @@ public class MoxyLoader extends Loader { result.add(list.item(i).getAttributes().getNamedItem("name").getNodeValue()); } } catch (ParserConfigurationException | SAXException | IOException e) { - LOGGER.warn("Exception while enumerating objects for API version " + getVersion() + " (returning partial results)", e); + LOGGER.warn("Exception while enumerating objects for API version " + getVersion() + " (returning partial results) " + LogFormatTools.getStackTop(e)); } //result.remove("EdgePropNames"); diff --git a/aai-core/src/main/java/org/onap/aai/introspection/PojoLoader.java b/aai-core/src/main/java/org/onap/aai/introspection/PojoLoader.java index 62a7dc0e..88411476 100644 --- a/aai-core/src/main/java/org/onap/aai/introspection/PojoLoader.java +++ b/aai-core/src/main/java/org/onap/aai/introspection/PojoLoader.java @@ -30,6 +30,7 @@ import org.onap.aai.db.props.AAIProperties; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; import org.onap.aai.introspection.exceptions.AAIUnmarshallingException; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.restcore.MediaType; import org.onap.aai.workarounds.NamingExceptions; @@ -60,7 +61,7 @@ public class PojoLoader extends Loader { try { context = JAXBContextFactory.createContext(pojoPackageName, this.getClass().getClassLoader()); } catch (JAXBException e) { - LOGGER.error("JAXBException while instantiation contect for PojoLoader", e); + LOGGER.error("JAXBException while instantiation contect for PojoLoader " + LogFormatTools.getStackTop(e)); } } diff --git a/aai-core/src/main/java/org/onap/aai/introspection/tools/CreateUUID.java b/aai-core/src/main/java/org/onap/aai/introspection/tools/CreateUUID.java index b049c5c5..a061ad6f 100644 --- a/aai-core/src/main/java/org/onap/aai/introspection/tools/CreateUUID.java +++ b/aai-core/src/main/java/org/onap/aai/introspection/tools/CreateUUID.java @@ -27,10 +27,37 @@ import org.onap.aai.schema.enums.PropertyMetadata; import java.util.Map; import java.util.UUID; +/** + * <b>CreateUUID</b> is an issue resolver that is responsible + * for looking to check if the property that is missing has + * the metadata autoGenerateUuid associated to it in the oxm + * As if that is found, then it will automatically resolve the + * issue by generating a uuid and setting that value to that property + * + * If this is needed for a specific property that you need + * then you need to add the following xml code in the oxm + * + * <pre> + * {@code + * <xml-element java-attribute="myElementProp" name="my-element-prop" type="java.lang.String"> + * <xml-properties> + * <xml-property name="autoGenerateUuid" value="true" /> + * </xml-properties> + * } + * </pre> + */ public class CreateUUID implements IssueResolver { /** - * {@inheritDoc} + * Resolves the issue by checking if the issue type is missing key prop + * and if it is it will retrieve the introspector associated with the issue + * then gets the metadata associated to that specific property + * and if it contains the auto generate meta property and if it does + * then it will fix it by setting that property value to generated uuid + * + * @param issue the issue with the details associated to the problem + * @return true if the issue has been successfully resolved + * false otherwise */ @Override public boolean resolveIssue(Issue issue) { diff --git a/aai-core/src/main/java/org/onap/aai/logging/EcompErrorCategory.java b/aai-core/src/main/java/org/onap/aai/logging/EcompErrorCategory.java new file mode 100644 index 00000000..9bf4056e --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/logging/EcompErrorCategory.java @@ -0,0 +1,44 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.logging; + +import ch.qos.logback.classic.pattern.ClassicConverter; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.Level; + +public class EcompErrorCategory extends ClassicConverter { + + @Override + public String convert(ILoggingEvent event) { + + final Level lev = event.getLevel(); + final String defaultCategory = "WARN"; + + if ((Level.WARN).equals(lev)) { + return (defaultCategory); + } + else if ((Level.ERROR).equals(lev)) { + return ("ERROR"); + } + return (defaultCategory); + } +} diff --git a/aai-core/src/main/java/org/onap/aai/logging/EcompPatternLayout.java b/aai-core/src/main/java/org/onap/aai/logging/EcompPatternLayout.java index a08c021f..b0eba743 100644 --- a/aai-core/src/main/java/org/onap/aai/logging/EcompPatternLayout.java +++ b/aai-core/src/main/java/org/onap/aai/logging/EcompPatternLayout.java @@ -28,5 +28,8 @@ public class EcompPatternLayout extends PatternLayout { PatternLayout.defaultConverterMap.put("ecompStartTime", EcompStartTime.class.getName()); PatternLayout.defaultConverterMap.put("ecompElapsedTime", EcompElapsedTime.class.getName()); PatternLayout.defaultConverterMap.put("eelfClassOfCaller", EelfClassOfCaller.class.getName()); + PatternLayout.defaultConverterMap.put("ecompErrorCategory", EcompErrorCategory.class.getName()); + PatternLayout.defaultConverterMap.put("ecompResponseCode", EcompResponseCode.class.getName()); + PatternLayout.defaultConverterMap.put("ecompResponseDescription", EcompResponseDescription.class.getName()); } } diff --git a/aai-core/src/main/java/org/onap/aai/logging/EcompResponseCode.java b/aai-core/src/main/java/org/onap/aai/logging/EcompResponseCode.java new file mode 100644 index 00000000..af3fdb71 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/logging/EcompResponseCode.java @@ -0,0 +1,40 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.logging; + +import org.onap.aai.logging.LoggingContext.LoggingField; + +import ch.qos.logback.classic.pattern.ClassicConverter; +import ch.qos.logback.classic.spi.ILoggingEvent; + +public class EcompResponseCode extends ClassicConverter { + + @Override + public String convert(ILoggingEvent event) { + + if (!event.getMDCPropertyMap().containsKey(LoggingField.RESPONSE_CODE.toString())) { + // if response code is not set, return "unknown" (900) + return LoggingContext.UNKNOWN_ERROR; + } + return event.getMDCPropertyMap().get(LoggingField.RESPONSE_CODE.toString()); + } +} diff --git a/aai-core/src/main/java/org/onap/aai/logging/EcompResponseDescription.java b/aai-core/src/main/java/org/onap/aai/logging/EcompResponseDescription.java new file mode 100644 index 00000000..6ef4b758 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/logging/EcompResponseDescription.java @@ -0,0 +1,46 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.logging; + +import org.onap.aai.logging.LoggingContext.LoggingField; + +import ch.qos.logback.classic.pattern.ClassicConverter; +import ch.qos.logback.classic.spi.ILoggingEvent; + +public class EcompResponseDescription extends ClassicConverter { + public final static String DefaultDescription = "Unknown response/error description"; + @Override + public String convert(ILoggingEvent event) { + + if (!event.getMDCPropertyMap().containsKey(LoggingField.RESPONSE_DESCRIPTION.toString())) { + return (DefaultDescription); + } + // Replace pipes and new lines + String currentDesc = event.getMDCPropertyMap().get(LoggingField.RESPONSE_DESCRIPTION.toString()); + if ( (currentDesc == null) || (currentDesc.length() == 0) ) { + return (DefaultDescription); + } + currentDesc = currentDesc.replaceAll("|", "!"); + currentDesc = currentDesc.replaceAll("[\\r\\n]+", "^"); + return event.getMDCPropertyMap().get(LoggingField.RESPONSE_DESCRIPTION.toString()); + } +} diff --git a/aai-core/src/main/java/org/onap/aai/logging/ErrorLogHelper.java b/aai-core/src/main/java/org/onap/aai/logging/ErrorLogHelper.java index 1c3ab853..a585a4a9 100644 --- a/aai-core/src/main/java/org/onap/aai/logging/ErrorLogHelper.java +++ b/aai-core/src/main/java/org/onap/aai/logging/ErrorLogHelper.java @@ -291,7 +291,7 @@ public class ErrorLogHelper { } } catch (Exception ex) { - LOGGER.error("We were unable to create a rest exception to return on an API because of a parsing error", ex); + LOGGER.error("We were unable to create a rest exception to return on an API because of a parsing error " + ex.getMessage()); } } else { @@ -319,7 +319,7 @@ public class ErrorLogHelper { response = (MapperUtil.writeAsJSONString((Object) restresp)); } } catch (AAIException ex) { - LOGGER.error("We were unable to create a rest exception to return on an API because of a parsing error", ex); + LOGGER.error("We were unable to create a rest exception to return on an API because of a parsing error " + ex.getMessage()); } } } @@ -343,7 +343,7 @@ public class ErrorLogHelper { public static String getRESTAPIErrorResponseWithLogging(List<MediaType> acceptHeadersOrig, AAIException are, ArrayList<String> variables) { String response = ErrorLogHelper.getRESTAPIErrorResponse(acceptHeadersOrig, are, variables); - LOGGER.error(are.getMessage(), are); + LOGGER.error(are.getMessage() + " " + LogFormatTools.getStackTop(are)); return response; } @@ -427,7 +427,7 @@ public class ErrorLogHelper { responseMessages.getResponseMessage().add(responseMessage); } catch (Exception ex) { - LOGGER.error("We were unable to create a rest exception to return on an API because of a parsing error", ex); + LOGGER.error("We were unable to create a rest exception to return on an API because of a parsing error " + ex.getMessage()); } } @@ -553,7 +553,7 @@ public class ErrorLogHelper { } } catch (Exception ex) { - LOGGER.error("We were unable to create a rest exception to return on an API because of a parsing error", ex); + LOGGER.error("We were unable to create a rest exception to return on an API because of a parsing error "+ ex.getMessage()); } return response; } @@ -571,7 +571,13 @@ public class ErrorLogHelper { LoggingContext.severity(sevCode); } } - + String stackTrace = ""; + try { + stackTrace = LogFormatTools.getStackTop(e); + } + catch (Exception a) { + //ignore + } final String errorMessage = new StringBuilder() .append(errorObject.getErrorText()) .append(":") @@ -586,14 +592,19 @@ public class ErrorLogHelper { LoggingContext.responseDescription(errorMessage); LoggingContext.statusCode(StatusCode.ERROR); + final String details = new StringBuilder().append(errorObject.getErrorCodeString()) + .append(" ") + .append(stackTrace) + .toString(); + if (errorObject.getSeverity().equalsIgnoreCase("WARN")) - LOGGER.warn(errorMessage, e); + LOGGER.warn(details); else if (errorObject.getSeverity().equalsIgnoreCase("ERROR")) - LOGGER.error(errorMessage, e); + LOGGER.error(details); else if (errorObject.getSeverity().equalsIgnoreCase("FATAL")) - LOGGER.error(errorMessage, e); + LOGGER.error(details); else if (errorObject.getSeverity().equals("INFO")) - LOGGER.info(errorMessage + ", " + e.getMessage()); + LOGGER.info(details); } public static void logError(String code) { diff --git a/aai-core/src/main/java/org/onap/aai/logging/LogFormatTools.java b/aai-core/src/main/java/org/onap/aai/logging/LogFormatTools.java index 702741b1..19650b65 100644 --- a/aai-core/src/main/java/org/onap/aai/logging/LogFormatTools.java +++ b/aai-core/src/main/java/org/onap/aai/logging/LogFormatTools.java @@ -25,6 +25,10 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import org.onap.aai.util.AAIConfig; +import org.onap.aai.util.AAIConstants; +import org.onap.aai.exceptions.AAIException; +import org.apache.commons.lang.exception.ExceptionUtils; public class LogFormatTools { @@ -43,4 +47,76 @@ public class LogFormatTools { public static long toTimestamp(String date) { return ZonedDateTime.parse(date, DTF).toInstant().toEpochMilli(); } + /** + * Gets the stack top. + * + * @param e the e + * @return the stack top + * @throws NumberFormatException the number format exception + * @throws AAIException the AAI exception + */ + public static String getStackTop(Throwable e) { + StringBuffer stackMessage = new StringBuffer(); + int maxStackTraceEntries = 10; + try { + maxStackTraceEntries = Integer.valueOf(AAIConfig.get(AAIConstants.LOGGING_MAX_STACK_TRACE_ENTRIES)); + } + catch (AAIException a) { + //ignore, use default + } + catch (NumberFormatException n) { + //ignore, use default + } + if (e != null) { + Throwable rootCause = ExceptionUtils.getRootCause(e); + if (rootCause != null) { + stackMessage.append("root cause=" + ExceptionUtils.getRootCause(e)); + StackTraceElement[] elements = rootCause.getStackTrace(); + int i = 0; + for (StackTraceElement element : elements) { + if (i < maxStackTraceEntries) { + stackMessage.append(" ClassName- "); + stackMessage.append(element.getClassName()); + stackMessage.append(" :LineNumber- "); + stackMessage.append(element.getLineNumber()); + stackMessage.append(" :MethodName- "); + stackMessage.append(element.getMethodName()); + } + i++; + } + } else if (e.getCause() != null) { + stackMessage.append("cause=" + e.getCause()); + StackTraceElement[] elements = e.getCause().getStackTrace(); + int i = 0; + for (StackTraceElement element : elements) { + if (i < maxStackTraceEntries) { + stackMessage.append(" ClassName- "); + stackMessage.append(element.getClassName()); + stackMessage.append(" :LineNumber- "); + stackMessage.append(element.getLineNumber()); + stackMessage.append(" :MethodName- "); + stackMessage.append(element.getMethodName()); + } + i++; + } + } else if (e.getStackTrace() != null) { + stackMessage.append("ex=" + e.toString()); + StackTraceElement[] elements = e.getStackTrace(); + int i = 0; + for (StackTraceElement element : elements) { + if (i < maxStackTraceEntries) { + stackMessage.append(" ClassName- "); + stackMessage.append(element.getClassName()); + stackMessage.append(" :LineNumber- "); + stackMessage.append(element.getLineNumber()); + stackMessage.append(" :MethodName- "); + stackMessage.append(element.getMethodName()); + } + i++; + } + } + } + return stackMessage.toString(); + } + } diff --git a/aai-core/src/main/java/org/onap/aai/logging/LoggingContext.java b/aai-core/src/main/java/org/onap/aai/logging/LoggingContext.java index 7d9dcce5..cb31bdde 100644 --- a/aai-core/src/main/java/org/onap/aai/logging/LoggingContext.java +++ b/aai-core/src/main/java/org/onap/aai/logging/LoggingContext.java @@ -58,6 +58,12 @@ public class LoggingContext { public static final String BUSINESS_PROCESS_ERROR = "500"; public static final String UNKNOWN_ERROR = "900"; + public static final Map<String, String> responseMap = new HashMap(); + + static { + responseMap.put(SUCCESS, "Success"); + responseMap.put(UNKNOWN_ERROR, "Unknown error"); + } //ECOMP Specific Log Event Fields public static enum LoggingField { START_TIME("startTime"), @@ -126,9 +132,6 @@ public class LoggingContext { public static void requestId(String requestId) { try { - if(requestId == null){ - throw new IllegalArgumentException(); - } if (requestId.contains(":")) { String[] uuidParts = requestId.split(":"); requestId = uuidParts[0]; @@ -138,12 +141,13 @@ public class LoggingContext { final UUID generatedRequestUuid = UUID.randomUUID(); MDC.put(LoggingField.REQUEST_ID.toString(), generatedRequestUuid.toString()); LoggingContext.save(); - AAIException ex = new AAIException("AAI_7405", e); - String responseCode = Integer.toString(ex.getErrorObject().getHTTPResponseCode().getStatusCode()); + // set response code to 0 since we don't know what the outcome of this request is yet + String responseCode = LoggingContext.DATA_ERROR; LoggingContext.responseCode(responseCode); - - LOGGER.warn("Unable to use UUID " + requestId + " (Not formatted properly). Using generated UUID=" - + generatedRequestUuid); + LoggingContext.responseDescription("Unable to use UUID " + requestId + " (Not formatted properly) "); + LoggingContext.statusCode(StatusCode.ERROR); + + LOGGER.warn("Using generated UUID=" + generatedRequestUuid); LoggingContext.restore(); } @@ -196,6 +200,7 @@ public class LoggingContext { public static void successStatusFields() { responseCode(SUCCESS); statusCode(LoggingContext.StatusCode.COMPLETE); + responseDescription("Success"); } private static void serverIpAddress() { try { diff --git a/aai-core/src/main/java/org/onap/aai/logging/StopWatch.java b/aai-core/src/main/java/org/onap/aai/logging/StopWatch.java index b57387d5..4b05f1a6 100644 --- a/aai-core/src/main/java/org/onap/aai/logging/StopWatch.java +++ b/aai-core/src/main/java/org/onap/aai/logging/StopWatch.java @@ -34,7 +34,18 @@ public final class StopWatch { public static double stop() { return LoggingContext.stopWatchStop(); } - + public static void conditionalStart() { + if ( LoggingContext.isStopWatchStarted() ) { + return; + } + start(); + } + public static double stopIfStarted() { + if ( LoggingContext.isStopWatchStarted() ) { + return (stop()); + } + return (0); + } public static void clear() { LoggingContext.remove(LoggingField.STOP_WATCH_START.toString()); LoggingContext.remove(LoggingField.ELAPSED_TIME.toString()); diff --git a/aai-core/src/main/java/org/onap/aai/parsers/query/LegacyQueryParser.java b/aai-core/src/main/java/org/onap/aai/parsers/query/LegacyQueryParser.java index 9251be6b..10530d19 100644 --- a/aai-core/src/main/java/org/onap/aai/parsers/query/LegacyQueryParser.java +++ b/aai-core/src/main/java/org/onap/aai/parsers/query/LegacyQueryParser.java @@ -27,6 +27,7 @@ import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Introspector; import org.onap.aai.introspection.Loader; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.parsers.uri.Parsable; import org.onap.aai.parsers.uri.URIParser; import org.onap.aai.parsers.uri.URIToObject; @@ -134,7 +135,8 @@ public class LegacyQueryParser extends QueryParser implements Parsable { Introspector child = obj.newIntrospectorInstanceOfNestedProperty(obj.getChildName()); this.handleUriKeys(child, uriKeys); } catch (AAIUnknownObjectException e) { - LOGGER.warn("Skipping container child " + obj.getChildName() + " (Unknown Object)", e); + LOGGER.warn("Skipping container child " + obj.getChildName() + " (Unknown Object) " + + LogFormatTools.getStackTop(e)); } } diff --git a/aai-core/src/main/java/org/onap/aai/parsers/uri/URIToExtensionInformation.java b/aai-core/src/main/java/org/onap/aai/parsers/uri/URIToExtensionInformation.java index 0b34a45f..868a7427 100644 --- a/aai-core/src/main/java/org/onap/aai/parsers/uri/URIToExtensionInformation.java +++ b/aai-core/src/main/java/org/onap/aai/parsers/uri/URIToExtensionInformation.java @@ -118,8 +118,15 @@ public class URIToExtensionInformation implements Parsable { */ public String getMethodName(HttpMethod httpMethod, boolean isPreprocess) { String result = "Dynamic"; - if (httpMethod.equals(HttpMethod.PUT)) { + /* + if (httpMethod.equals(HttpMethod.PUT) || httpMethod.equals(HttpMethod.PUT_EDGE)) { result += "Add"; + } + */ + if (httpMethod.equals(HttpMethod.PUT) ) { + result += "Add"; + } else if (httpMethod.equals(HttpMethod.PUT_EDGE)) { + result += "AddEdge"; } else if (httpMethod.equals(HttpMethod.DELETE)) { result += "Del"; } else { diff --git a/aai-core/src/main/java/org/onap/aai/parsers/uri/URIToRelationshipObject.java b/aai-core/src/main/java/org/onap/aai/parsers/uri/URIToRelationshipObject.java index cec55bc3..48a6324e 100644 --- a/aai-core/src/main/java/org/onap/aai/parsers/uri/URIToRelationshipObject.java +++ b/aai-core/src/main/java/org/onap/aai/parsers/uri/URIToRelationshipObject.java @@ -134,7 +134,6 @@ public class URIToRelationshipObject implements Parsable { URI originalUri = parser.getOriginalURI(); URI relatedLink = new URI(this.baseURL + this.originalVersion + "/" + originalUri); - this.relationship.setValue("related-link", relatedLink); if (this.originalVersion.compareTo(Version.v10) >= 0) { //only return the path section of the URI past v10 relatedLink = new URI(relatedLink.getRawPath()); diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java b/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java index 82079ad7..8156dd09 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java @@ -141,6 +141,37 @@ public abstract class GraphTraversalBuilder<E> extends QueryBuilder<E> { stepIndex++; return (QueryBuilder<Vertex>) this; } + + /** + * @{inheritDoc} + */ + @Override + public QueryBuilder<Vertex> getVerticesExcludeByProperty(String key, Object value) { + + // correct value call because the index is registered as an Integer + traversal.has(key, P.neq(this.correctObjectType(value))); + + stepIndex++; + return (QueryBuilder<Vertex>) this; + } + + /** + * @{inheritDoc} + */ + @Override + public QueryBuilder<Vertex> getVerticesExcludeByProperty(final String key, final List<?> values) { + + //this is because the index is registered as an Integer + List<Object> correctedValues = new ArrayList<>(); + for (Object item : values) { + correctedValues.add(this.correctObjectType(item)); + } + + traversal.has(key, P.without(correctedValues)); + + stepIndex++; + return (QueryBuilder<Vertex>) this; + } /** * @{inheritDoc} @@ -414,6 +445,30 @@ public abstract class GraphTraversalBuilder<E> extends QueryBuilder<E> { return this; } + @Override + public QueryBuilder<E> groupCount() { + this.traversal.groupCount(); + stepIndex++; + + return this; + } + + @Override + public QueryBuilder<E> both() { + this.traversal.both(); + stepIndex++; + + return this; + } + + @Override + public QueryBuilder<E> by(String name) { + this.traversal.by(name); + stepIndex++; + + return this; + } + /** * {@inheritDoc} */ diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/GremlinQueryBuilder.java b/aai-core/src/main/java/org/onap/aai/query/builder/GremlinQueryBuilder.java index f2eaa91f..d87dac48 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/GremlinQueryBuilder.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/GremlinQueryBuilder.java @@ -153,6 +153,47 @@ public abstract class GremlinQueryBuilder<E> extends QueryBuilder<E> { return (QueryBuilder<Vertex>) this; } + /** + * @{inheritDoc} + */ + @Override + public QueryBuilder<Vertex> getVerticesExcludeByProperty(String key, Object value) { + + String term = ""; + String predicate = "P.neq(#!#argument#!#)"; + if (value != null && !(value instanceof String) ) { + term = value.toString(); + } else { + term = "'" + value + "'"; + } + predicate = predicate.replace("#!#argument#!#", term); + list.add(".has('" + key + "', " + predicate + ")"); + stepIndex++; + return (QueryBuilder<Vertex>) this; + } + + /** + * @{inheritDoc} + */ + @Override + public QueryBuilder<Vertex> getVerticesExcludeByProperty(String key, List<?> values) { + + String term = ""; + String predicate = "P.without(#!#argument#!#)"; + List<String> arguments = new ArrayList<>(); + for (Object item : values) { + if (item != null && !(item instanceof String)) { + arguments.add(item.toString()); + } else { + arguments.add("'" + item + "'"); + } + } + String argument = Joiner.on(",").join(arguments); + predicate = predicate.replace("#!#argument#!#", argument); + list.add(".has('" + key + "', " + predicate + ")"); + stepIndex++; + return (QueryBuilder<Vertex>) this; + } /** * @{inheritDoc} @@ -274,7 +315,9 @@ public abstract class GremlinQueryBuilder<E> extends QueryBuilder<E> { } }); - if (inLabels.isEmpty() && !outLabels.isEmpty()) { + if(inLabels.isEmpty() && outLabels.isEmpty()) { + throw new NoEdgeRuleFoundException("no " + type.toString() + " edge rule between " + outType + " and " + inType ); + } else if (inLabels.isEmpty() && !outLabels.isEmpty()) { list.add(".out('" + String.join("','", outLabels) + "')"); } else if (outLabels.isEmpty() && !inLabels.isEmpty()) { list.add(".in('" + String.join("','", inLabels) + "')"); @@ -319,7 +362,9 @@ public abstract class GremlinQueryBuilder<E> extends QueryBuilder<E> { } }); - if (inLabels.isEmpty() && !outLabels.isEmpty()) { + if(inLabels.isEmpty() && outLabels.isEmpty()) { + throw new NoEdgeRuleFoundException("no " + type.toString() + " edge rule between " + outType + " and " + inType ); + } else if (inLabels.isEmpty() && !outLabels.isEmpty()) { list.add(".outE('" + String.join("','", outLabels) + "')"); } else if (outLabels.isEmpty() && !inLabels.isEmpty()) { list.add(".inE('" + String.join("','", inLabels) + "')"); @@ -449,6 +494,30 @@ public abstract class GremlinQueryBuilder<E> extends QueryBuilder<E> { return this; } + @Override + public QueryBuilder<E> groupCount() { + this.list.add(".groupCount()"); + stepIndex++; + + return this; + } + + @Override + public QueryBuilder<E> both() { + this.list.add(".both()"); + stepIndex++; + + return this; + } + + @Override + public QueryBuilder<E> by(String name) { + this.list.add(".by('"+ name + "')"); + stepIndex++; + + return this; + } + /** * {@inheritDoc} */ diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java b/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java index f5d47871..a51f592a 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java @@ -112,6 +112,33 @@ public abstract class QueryBuilder<E> implements Iterator<E> { public abstract QueryBuilder<Vertex> getVerticesByProperty(String key, List<?> values); /** + * Gets the vertices that are excluded by property. + * + * @param key the key + * @param value the value + * @return the vertices by property + */ + public abstract QueryBuilder<Vertex> getVerticesExcludeByProperty(String key, Object value); + + /** + * filters by all the values for this property and excludes the vertices + * @param key + * @param values + * @return vertices that match these values + */ + public QueryBuilder<Vertex> getVerticesExcludeByIndexedProperty(String key, List<?> values) { + return this.getVerticesExcludeByProperty(key, values); + } + + /** + * filters by all the values for this property and excludes the vertices + * @param key + * @param values + * @return vertices that match these values + */ + public abstract QueryBuilder<Vertex> getVerticesExcludeByProperty(String key, List<?> values); + + /** * Gets the child vertices from parent. * * @param parentKey the parent key @@ -369,6 +396,9 @@ public abstract class QueryBuilder<E> implements Iterator<E> { public abstract QueryBuilder<E> as(String name); public abstract QueryBuilder<E> select(String name); public abstract QueryBuilder<E> until(QueryBuilder<E> builder); + public abstract QueryBuilder<E> groupCount(); + public abstract QueryBuilder<E> by(String name); + public abstract QueryBuilder<E> both(); /** * Used to prevent the traversal from repeating its path through the graph. diff --git a/aai-core/src/main/java/org/onap/aai/rest/RestHandlerService.java b/aai-core/src/main/java/org/onap/aai/rest/RestHandlerService.java new file mode 100644 index 00000000..9c037b30 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/rest/RestHandlerService.java @@ -0,0 +1,46 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.rest; + +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; + +public class RestHandlerService { + private static RestHandlerService single_instance = null; + public ThreadPoolExecutor executor; + // private constructor restricted to this class itself + private RestHandlerService() + { + executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(50); + } + /** + * Gets the single instance of RestHandlerService. + * + * @return single instance of RestHandlerService + */ + public static RestHandlerService getInstance() { + if (single_instance == null) { + single_instance = new RestHandlerService(); + } + return single_instance; + } +} diff --git a/aai-core/src/main/java/org/onap/aai/rest/db/HttpEntry.java b/aai-core/src/main/java/org/onap/aai/rest/db/HttpEntry.java index 0d89a74c..d6bf3625 100644 --- a/aai-core/src/main/java/org/onap/aai/rest/db/HttpEntry.java +++ b/aai-core/src/main/java/org/onap/aai/rest/db/HttpEntry.java @@ -30,7 +30,10 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; @@ -48,6 +51,8 @@ import org.onap.aai.dbmap.DBConnectionType; import org.onap.aai.domain.responseMessage.AAIResponseMessage; import org.onap.aai.domain.responseMessage.AAIResponseMessageDatum; import org.onap.aai.exceptions.AAIException; +import org.onap.aai.extensions.AAIExtensionMap; +import org.onap.aai.extensions.ExtensionController; import org.onap.aai.introspection.Introspector; import org.onap.aai.introspection.Loader; import org.onap.aai.introspection.LoaderFactory; @@ -57,6 +62,7 @@ import org.onap.aai.introspection.ModelType; import org.onap.aai.introspection.Version; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LoggingContext; import org.onap.aai.parsers.query.QueryParser; import org.onap.aai.parsers.uri.URIToExtensionInformation; import org.onap.aai.rest.ueb.UEBNotification; @@ -82,6 +88,7 @@ import com.thinkaurelius.titan.core.TitanException; public class HttpEntry { private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(HttpEntry.class); + private static final String TARGET_ENTITY = "DB"; private final ModelType introspectorFactoryType; @@ -175,6 +182,7 @@ public class HttpEntry { */ public Pair<Boolean, List<Pair<URI, Response>>> process (List<DBRequest> requests, String sourceOfTruth, boolean enableResourceVersion) throws AAIException { DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, sourceOfTruth); + String methodName = "process"; Response response = null; Status status = Status.NOT_FOUND; Introspector obj = null; @@ -191,16 +199,25 @@ public class HttpEntry { QueryEngine queryEngine = dbEngine.getQueryEngine(); int maxRetries = 10; int retry = 0; + + LoggingContext.save(); for (DBRequest request : requests) { try { for (retry = 0; retry < maxRetries; ++retry) { try { method = request.getMethod(); + + LoggingContext.targetEntity(TARGET_ENTITY); + LoggingContext.targetServiceName(methodName + " " + method); + obj = request.getIntrospector(); query = request.getParser(); transactionId = request.getTransactionId(); uriTemp = request.getUri().getRawPath().replaceFirst("^v\\d+/", ""); uri = UriBuilder.fromPath(uriTemp).build(); + + LoggingContext.startTime(); + List<Vertex> vertices = query.getQueryBuilder().toList(); boolean isNewVertex = false; String outputMediaType = getMediaType(request.getHeaders().getAcceptableMediaTypes()); @@ -219,8 +236,10 @@ public class HttpEntry { } if (vertices.size() > 1 && processSingle && !method.equals(HttpMethod.GET)) { if (method.equals(HttpMethod.DELETE)) { + LoggingContext.restoreIfPossible(); throw new AAIException("AAI_6138"); } else { + LoggingContext.restoreIfPossible(); throw new AAIException("AAI_6137"); } } @@ -254,8 +273,12 @@ public class HttpEntry { case GET: String nodeOnly = params.getFirst("nodes-only"); boolean isNodeOnly = nodeOnly != null; - obj = this.getObjectFromDb(vertices, serializer, query, obj, request.getUri(), depth, isNodeOnly, cleanUp); + + LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs(),TimeUnit.MILLISECONDS); + LOGGER.info ("Completed"); + LoggingContext.restoreIfPossible(); + if (obj != null) { status = Status.OK; MarshallerProperties properties; @@ -282,11 +305,20 @@ public class HttpEntry { if (query.isDependent()) { relatedObjects = this.getRelatedObjects(serializer, queryEngine, v); } + LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs() + + (long)queryEngine.getDBTimeMsecs(), TimeUnit.MILLISECONDS); + LOGGER.info ("Completed"); + LoggingContext.restoreIfPossible(); notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, obj, relatedObjects); + break; case PUT_EDGE: serializer.touchStandardVertexProperties(v, false); serializer.createEdge(obj, v); + + LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs(),TimeUnit.MILLISECONDS); + LOGGER.info ("Completed"); + LoggingContext.restoreIfPossible(); status = Status.OK; break; case MERGE_PATCH: @@ -319,8 +351,15 @@ public class HttpEntry { if (query.isDependent()) { relatedObjects = this.getRelatedObjects(serializer, queryEngine, v); } + LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs() + + (long)queryEngine.getDBTimeMsecs(), TimeUnit.MILLISECONDS); + LOGGER.info ("Completed"); + LoggingContext.restoreIfPossible(); notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, patchedObj, relatedObjects); } catch (IOException | JsonPatchException e) { + + LOGGER.info ("Caught exception: " + e.getMessage()); + LoggingContext.restoreIfPossible(); throw new AAIException("AAI_3000", "could not perform patch operation"); } break; @@ -330,13 +369,58 @@ public class HttpEntry { if (query.isDependent()) { relatedObjects = this.getRelatedObjects(serializer, queryEngine, v); } + /* + * Find all Delete-other-vertex vertices and create structure for notify + * findDeleatble also returns the startVertex v and we dont want to create + * duplicate notification events for the same + * So remove the startvertex first + */ + + List<Vertex> deletableVertices = dbEngine.getQueryEngine().findDeletable(v); + Long vId = (Long) v.id(); + + /* + * I am assuming vertexId cant be null + */ + deletableVertices.removeIf(s -> vId.equals(s.id())); + boolean isDelVerticesPresent = !deletableVertices.isEmpty(); + Map<Vertex, Introspector> deleteObjects = new HashMap<>(); + Map<String, URI> uriMap = new HashMap<>(); + Map<String, HashMap<String, Introspector>> deleteRelatedObjects = new HashMap<>(); + + if(isDelVerticesPresent){ + deleteObjects = this.buildIntrospectorObjects(serializer, deletableVertices); + + uriMap = this.buildURIMap(serializer, deleteObjects); + deleteRelatedObjects = this.buildRelatedObjects(serializer, queryEngine, deleteObjects); + } + serializer.delete(v, resourceVersion, enableResourceVersion); + + LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs() + + (long)queryEngine.getDBTimeMsecs(), TimeUnit.MILLISECONDS); + LOGGER.info ("Completed"); + LoggingContext.restoreIfPossible(); status = Status.NO_CONTENT; notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, obj, relatedObjects); + + /* + * Notify delete-other-v candidates + */ + + if(isDelVerticesPresent){ + this.buildNotificationEvent(sourceOfTruth, status, transactionId, notification, deleteObjects, + uriMap, deleteRelatedObjects); + } + break; case DELETE_EDGE: serializer.touchStandardVertexProperties(v, false); serializer.deleteEdge(obj, v); + + LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs(),TimeUnit.MILLISECONDS); + LOGGER.info ("Completed"); + LoggingContext.restoreIfPossible(); status = Status.NO_CONTENT; break; default: @@ -370,25 +454,27 @@ public class HttpEntry { break; } catch (TitanException e) { this.dbEngine.rollback(); + + LOGGER.info ("Caught exception: " + e.getMessage()); + LoggingContext.restoreIfPossible(); AAIException ex = new AAIException("AAI_6142", e); ErrorLogHelper.logException(ex); - Thread.sleep((retry + 1) * 20); + Thread.sleep((retry + 1) * 20L); this.dbEngine.startTransaction(); queryEngine = dbEngine.getQueryEngine(); serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, sourceOfTruth); } - if (retry == maxRetries) { throw new AAIException("AAI_6134"); } } } catch (AAIException e) { success = false; - ArrayList<String> templateVars = new ArrayList<String>(); + ArrayList<String> templateVars = new ArrayList<>(); templateVars.add(request.getMethod().toString()); //GET, PUT, etc - templateVars.add(request.getUri().getPath().toString()); + templateVars.add(request.getUri().getPath()); templateVars.addAll(e.getTemplateVars()); - + ErrorLogHelper.logException(e); response = Response .status(e.getErrorObject().getHTTPResponseCode()) .entity(ErrorLogHelper.getRESTAPIErrorResponse(request.getHeaders().getAcceptableMediaTypes(), e, templateVars)) @@ -403,7 +489,7 @@ public class HttpEntry { ArrayList<String> templateVars = new ArrayList<String>(); templateVars.add(request.getMethod().toString()); //GET, PUT, etc templateVars.add(request.getUri().getPath().toString()); - + ErrorLogHelper.logException(ex); response = Response .status(ex.getErrorObject().getHTTPResponseCode()) .entity(ErrorLogHelper.getRESTAPIErrorResponse(request.getHeaders().getAcceptableMediaTypes(), ex, templateVars)) @@ -415,10 +501,10 @@ public class HttpEntry { } notification.triggerEvents(); - Pair<Boolean, List<Pair<URI, Response>>> tuple = Pair.with(success, responses); - return tuple; + return Pair.with(success, responses); } + /** * Gets the media type. * @@ -562,5 +648,88 @@ public class HttpEntry { return relatedVertices; } + + private Map<Vertex, Introspector> buildIntrospectorObjects(DBSerializer serializer, Iterable<Vertex> vertices) { + Map<Vertex, Introspector> deleteObjectMap = new HashMap<>(); + for (Vertex vertex : vertices) { + try { + // deleteObjectMap.computeIfAbsent(vertex, s -> + // serializer.getLatestVersionView(vertex)); + Introspector deleteObj = serializer.getLatestVersionView(vertex); + deleteObjectMap.put(vertex, deleteObj); + } catch (UnsupportedEncodingException | AAIException e) { + LOGGER.warn("Unable to get Introspctor Objects, Just continue"); + continue; + } + + } + + return deleteObjectMap; + + } + + private Map<String, URI> buildURIMap(DBSerializer serializer, Map<Vertex, Introspector> introSpector) { + Map<String, URI> uriMap = new HashMap<>(); + for (Map.Entry<Vertex, Introspector> entry : introSpector.entrySet()) { + URI uri; + try { + uri = serializer.getURIForVertex(entry.getKey()); + if (null != entry.getValue()) + uriMap.put(entry.getValue().getObjectId(), uri); + } catch (UnsupportedEncodingException e) { + LOGGER.warn("Unable to get URIs, Just continue"); + continue; + } + + } + + return uriMap; + + } + + private Map<String, HashMap<String, Introspector>> buildRelatedObjects(DBSerializer serializer, + QueryEngine queryEngine, Map<Vertex, Introspector> introSpector) { + + Map<String, HashMap<String, Introspector>> relatedObjectsMap = new HashMap<>(); + for (Map.Entry<Vertex, Introspector> entry : introSpector.entrySet()) { + try { + HashMap<String, Introspector> relatedObjects = this.getRelatedObjects(serializer, queryEngine, + entry.getKey()); + if (null != entry.getValue()) + relatedObjectsMap.put(entry.getValue().getObjectId(), relatedObjects); + } catch (IllegalAccessException | IllegalArgumentException + | InvocationTargetException | SecurityException | InstantiationException | NoSuchMethodException + | UnsupportedEncodingException | AAIException | URISyntaxException e) { + LOGGER.warn("Unable to get realted Objects, Just continue"); + continue; + } + + } + + return relatedObjectsMap; + + } + + private void buildNotificationEvent(String sourceOfTruth, Status status, String transactionId, + UEBNotification notification, Map<Vertex, Introspector> deleteObjects, Map<String, URI> uriMap, + Map<String, HashMap<String, Introspector>> deleteRelatedObjects) { + for (Map.Entry<Vertex, Introspector> entry : deleteObjects.entrySet()) { + try { + String vertexObjectId = ""; + + if (null != entry.getValue()) { + vertexObjectId = entry.getValue().getObjectId(); + + if (uriMap.containsKey(vertexObjectId) && deleteRelatedObjects.containsKey(vertexObjectId)) + notification.createNotificationEvent(transactionId, sourceOfTruth, status, + uriMap.get(vertexObjectId), entry.getValue(), deleteRelatedObjects.get(vertexObjectId)); + } + } catch (UnsupportedEncodingException | AAIException e) { + + LOGGER.warn("Error in sending otification"); + } + } + } + } diff --git a/aai-core/src/main/java/org/onap/aai/restcore/RESTAPI.java b/aai-core/src/main/java/org/onap/aai/restcore/RESTAPI.java index f26ac197..9f13096b 100644 --- a/aai-core/src/main/java/org/onap/aai/restcore/RESTAPI.java +++ b/aai-core/src/main/java/org/onap/aai/restcore/RESTAPI.java @@ -25,6 +25,14 @@ import java.io.UnsupportedEncodingException; import java.net.URI; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeoutException; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; @@ -77,7 +85,6 @@ public class RESTAPI { * Gets the from app id. * * @param headers the headers - * @param logline the logline * @return the from app id * @throws AAIException the AAI exception */ @@ -105,7 +112,6 @@ public class RESTAPI { * Gets the trans id. * * @param headers the headers - * @param logline the logline * @return the trans id * @throws AAIException the AAI exception */ @@ -194,6 +200,7 @@ public class RESTAPI { templateVars.add(info.getPath().toString()); templateVars.addAll(e.getTemplateVars()); + ErrorLogHelper.logException(e); return Response .status(e.getErrorObject().getHTTPResponseCode()) .entity(ErrorLogHelper.getRESTAPIErrorResponseWithLogging(headers.getAcceptableMediaTypes(), e, templateVars)) @@ -206,7 +213,6 @@ public class RESTAPI { * @param obj the obj * @param loader the loader * @param uri the uri - * @param validateRequired the validate required * @throws AAIException the AAI exception * @throws UnsupportedEncodingException the unsupported encoding exception */ @@ -255,12 +261,15 @@ public class RESTAPI { throw new AAIException("AAI_4009", "X-FromAppId is not set"); } + DBConnectionType type = DBConnectionType.REALTIME; boolean isRealTimeClient = AAIConfig.get("aai.realtime.clients", "").contains(fromAppId); if (isRealTimeClient || realTime != null) { - return DBConnectionType.REALTIME; + type = DBConnectionType.REALTIME; } else { - return DBConnectionType.CACHED; + type = DBConnectionType.CACHED; } + + return type; } /** @@ -276,4 +285,116 @@ public class RESTAPI { } + /** + * Returns the app specific timeout in milliseconds, -1 overrides the timeout for an app + * + * @param sot + * @param appTimeouts + * @param defaultTimeout + * @return integer timeout in or -1 to bypass + * @throws AAIException + */ + + public int getTimeoutLimit(String sot, String appTimeouts, String defaultTimeout) throws AAIException{ + String[] ignoreAppIds = (appTimeouts).split("\\|"); + int appLimit = Integer.parseInt(defaultTimeout); + final Map<String, Integer> m = new HashMap<String, Integer>(); + if(ignoreAppIds != null) { + for (int i = 0; i < ignoreAppIds.length; i++) { + String[] vals = ignoreAppIds[i].split(","); + m.put(vals[0], Integer.parseInt(vals[1])); + } + if (m.get(sot) != null) { + appLimit = m.get(sot); + } + } + return appLimit; + } + + /** + * Returns whether time out is enabled + * @param sot + * @param isEnabled + * @param appTimeouts + * @param defaultTimeout + * @return boolean of whether the timeout is enabled + * @throws AAIException + */ + public boolean isTimeoutEnabled(String sot, String isEnabled, String appTimeouts, String defaultTimeout) throws AAIException{ + Boolean isTimeoutEnabled = Boolean.parseBoolean(isEnabled); + int ata = -1; + if(isTimeoutEnabled) { + ata = getTimeoutLimit(sot, appTimeouts, defaultTimeout); + } + return isTimeoutEnabled && (ata > -1); + } + + /** + * Executes the process thread and watches the future for the timeout + * @param handler + * @param sourceOfTruth + * @param appTimeoutLimit + * @param defaultTimeoutLimit + * @param method + * @param headers + * @param info + * @return the response + */ + + public Response executeProcess(Future<Response> handler, String sourceOfTruth, String appTimeoutLimit, String defaultTimeoutLimit, HttpMethod method, HttpHeaders headers, UriInfo info){ + Response response = null; + int timeoutLimit = 0; + try { + timeoutLimit = getTimeoutLimit(sourceOfTruth, appTimeoutLimit, defaultTimeoutLimit); + response = handler.get(timeoutLimit, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + AAIException ex = new AAIException("AAI_7406", String.format("Timeout limit of %s seconds reached.", timeoutLimit/1000)); + response = consumerExceptionResponseGenerator(headers, info, method, ex); + handler.cancel(true); + } catch (Exception e) { + AAIException ex = new AAIException("AAI_4000", e); + response = consumerExceptionResponseGenerator(headers, info, method, ex); + } + return response; + } + + /** + * runner sets up the timer logic and invokes it + * @param toe + * @param tba + * @param tdl + * @param headers + * @param info + * @param httpMethod + * @param c + * @return the response + */ + public Response runner(String toe, String tba, String tdl, HttpHeaders headers, UriInfo info, HttpMethod httpMethod, Callable c){ + Response response = null; + Future<Response> handler = null; + ExecutorService executor = null; + try { + String timeoutEnabled = AAIConfig.get(toe); + String timeoutByApp = AAIConfig.get(tba); + String timeoutDefaultLimit = AAIConfig.get(tdl); + String sourceOfTruth = headers.getRequestHeaders().getFirst("X-FromAppId"); + if (isTimeoutEnabled(sourceOfTruth, timeoutEnabled, timeoutByApp, timeoutDefaultLimit)) { + executor = Executors.newSingleThreadExecutor(); + handler = executor.submit(c); + response = executeProcess(handler, sourceOfTruth, timeoutByApp, timeoutDefaultLimit, httpMethod, headers, info); + } else { + response = (Response) c.call(); + } + }catch(Exception e){ + AAIException ex = new AAIException("AAI_4000", e); + response = consumerExceptionResponseGenerator(headers, info, httpMethod, ex); + }finally{ + if(executor != null && handler != null){ + executor.shutdownNow(); + } + } + return response; + } + } + diff --git a/aai-core/src/main/java/org/onap/aai/restcore/util/EdgeRuleBean.java b/aai-core/src/main/java/org/onap/aai/restcore/util/EdgeRuleBean.java new file mode 100644 index 00000000..cd175e38 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/restcore/util/EdgeRuleBean.java @@ -0,0 +1,95 @@ +/*- + * ============LICENSE_START======================================================= + * org.openecomp.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.restcore.util; + +public class EdgeRuleBean { + private String from; + private String to; + private String label; + private String direction; + private String multiplicity; + private String lineage; + private String preventDelete; + private String deleteOtherV; + private String svcInfra; + private String defaultVal; + + public String getFrom() { + return from; + } + public void setFrom(String from) { + this.from = from; + } + public String getTo() { + return to; + } + public void setTo(String to) { + this.to = to; + } + public String getLabel() { + return label; + } + public void setLabel(String label) { + this.label = label; + } + public String getDirection() { + return direction; + } + public void setDirection(String direction) { + this.direction = direction; + } + public String getMultiplicity() { + return multiplicity; + } + public void setMultiplicity(String multiplicity) { + this.multiplicity = multiplicity; + } + public String getDeleteOtherV() { + return deleteOtherV; + } + public void setDeleteOtherV(String deleteOtherV) { + this.deleteOtherV = deleteOtherV; + } + public String getPreventDelete() { + return preventDelete; + } + public void setPreventDelete(String preventDelete) { + this.preventDelete = preventDelete; + } + public String getSvcInfra() { + return svcInfra; + } + public void setSvcInfra(String svcInfra) { + this.svcInfra = svcInfra; + } + public String getLineage() { + return lineage; + } + public void setLineage(String lineage) { + this.lineage = lineage; + } + public String getDefault() { + return defaultVal; + } + public void setDefault(String defaultVal) { + this.defaultVal = defaultVal; + } +} diff --git a/aai-core/src/main/java/org/onap/aai/restcore/util/GenerateEdgeRules.java b/aai-core/src/main/java/org/onap/aai/restcore/util/GenerateEdgeRules.java index 1b854667..e1660ab3 100644 --- a/aai-core/src/main/java/org/onap/aai/restcore/util/GenerateEdgeRules.java +++ b/aai-core/src/main/java/org/onap/aai/restcore/util/GenerateEdgeRules.java @@ -21,8 +21,6 @@ */ package org.onap.aai.restcore.util; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; @@ -30,108 +28,162 @@ import freemarker.template.TemplateException; import java.io.*; import java.util.*; -public class GenerateEdgeRules { +import org.onap.aai.serialization.db.EdgeRules; +import org.onap.aai.introspection.Version; - private static final EELFLogger LOG = EELFManager.getInstance().getLogger(GenerateEdgeRules.class); +public class GenerateEdgeRules { public static void main(String[] args) throws IOException, TemplateException { - String filename = "/AAI8032.csv"; + String filename = "/edgeLabelMigration.csv"; InputStream inputStream = GenerateEdgeRules.class.getResourceAsStream(filename); Map<String, Integer> headers = new HashMap<>(); - Map<String, Object> edgeRulesMap = new TreeMap<String, Object>(); - List<Map<String, String>> edgeRules = new ArrayList<>(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + List<EdgeRuleBean> rulesToWriteV12 = new ArrayList<>(); + List<EdgeRuleBean> rulesToWriteV7 = new ArrayList<>(); + List<EdgeRuleBean> rulesToWriteV8 = new ArrayList<>(); + List<EdgeRuleBean> rulesToWriteV9 = new ArrayList<>(); + List<EdgeRuleBean> rulesToWriteV10 = new ArrayList<>(); + List<EdgeRuleBean> rulesToWriteV11 = new ArrayList<>(); - String line = null; + ArrayList <String> rulesWeAlreadyHave = new ArrayList <String>(); - int rowNum = 0; + EdgeRules rulesV8 = EdgeRules.getInstance(Version.v8); + EdgeRules rulesV9 = EdgeRules.getInstance(Version.v9); + EdgeRules rulesV10 = EdgeRules.getInstance(Version.v10); + EdgeRules rulesV11 = EdgeRules.getInstance(Version.v11); - // Retrieve the header line to map the indexes to their column names + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + String line = null; + int rowNum = 0; while ((line = reader.readLine()) != null) { - - if (rowNum == 0) { + if (rowNum == 0) { headers = retrieveHeaderMap(line); - } else { - String[] columns = line.split(","); - - String originalNode = columns[headers.get("Orig NodeA|NodeB")]; - String finalNode = columns[headers.get("Final NodeA|NodeB")]; - String originalEdge = columns[headers.get("Orig EdgeLabel")]; - String finalEdge = columns[headers.get("Final EdgeLabel")]; - - String lineage = columns[headers.get("Final Lineage")]; - String originalParent = columns[headers.get("Orig ParentOf")]; - String usesResource = columns[headers.get("Revised UsesResource")]; - String hasDelTarget = columns[headers.get("Revised hasDelTarget")]; - String svcInfra = columns[headers.get("Final SVC-INFRA")]; - String svcInfraRev = ""; - - if(usesResource.equals("T")) - usesResource = "true"; - else if(usesResource.equals("F")) - usesResource = "false"; - - if (hasDelTarget.equals("T") || hasDelTarget.equals("AB")) { - hasDelTarget = "true"; - } else if (hasDelTarget.equals("F")) { - hasDelTarget = "false"; + } + else { + EdgeRuleBean data = new EdgeRuleBean(); + String[] columns = line.split(","); + String oldNodeA = columns[headers.get("from")]; + String oldNodeB = columns[headers.get("to")]; + String oldEdgeLabel = columns[headers.get("label")]; + + String nodeA = columns[headers.get("new from")]; + data.setFrom(nodeA); + String nodeB = columns[headers.get("new to")]; + data.setTo(nodeB); + + String edgeLabel = columns[headers.get("new label")]; + data.setLabel( edgeLabel ); + + + // Note: it is assumed that if we know the two NodeTypes and the edgeLabel, we can + // uniquely identify an edgeRule -- so if that key is found twice, it is a + // problem with our CSV file. Note -we check with the nodeTypes in both directions. + String key1 = nodeA + "|" + nodeB + "|" + edgeLabel; + String key2 = nodeB + "|" + nodeA + "|" + edgeLabel; + if( rulesWeAlreadyHave.contains(key1) ){ + throw new Exception ("Duplicate rule found for [" + key1 + "] -- please fix the CSV file. "); } - - if (svcInfra.equals("T")) { - svcInfra = "true"; - } else if (svcInfra.equals("F")) { - svcInfra = "false"; - } else if (svcInfra.equals("R")) { - svcInfra = "reverse"; + else if( rulesWeAlreadyHave.contains(key2) ){ + throw new Exception ("Duplicate rule found for [" + key2 + "] -- please fix the CSV file. "); } + else { + rulesWeAlreadyHave.add(key1); + rulesWeAlreadyHave.add(key2); + } + + String direction = columns[headers.get("new direction")]; + data.setDirection(direction); + + String multiplicity = columns[headers.get("new multiplicity")]; + data.setMultiplicity(multiplicity); - if (originalParent.equals("T")) { - if (lineage.trim().equalsIgnoreCase("CHILD")) { - lineage = "true"; - } else if (lineage.trim().equalsIgnoreCase("PARENT")) { - lineage = "reverse"; - } - } else { - lineage = "false"; + String lineage = columns[headers.get("new contains-other-v")]; + data.setLineage(lineage); + + String deleteOtherV = columns[headers.get("new delete-other-v")]; + data.setDeleteOtherV(deleteOtherV); + + String svcInfra = columns[headers.get("new SVC-INFRA")]; + data.setSvcInfra(svcInfra); + + String prevDel = columns[headers.get("new prevent-delete")]; + data.setPreventDelete(prevDel); + + String defaultVal = columns[headers.get("new default")]; + if( defaultVal.equals("T") ){ + data.setDefault("true"); } + else if( defaultVal.equals("F") ){ + data.setDefault("false"); + } + + rulesToWriteV12.add(data); - Map<String, String> edgeMap = new HashMap<String, String>(); + if( rulesV8.hasEdgeRule(oldNodeA, oldNodeB, oldEdgeLabel) ){ + rulesToWriteV8.add(data); + } - edgeMap.put("lineage", lineage); - edgeMap.put("usesResource", usesResource); - edgeMap.put("hasDelTarget", hasDelTarget); - edgeMap.put("SVC-INFRA", svcInfra); - edgeMap.put("SVC-INFRA-REV", svcInfraRev); - edgeMap.put("nodes", finalNode); - edgeMap.put("edge", finalEdge); - edgeMap.put("direction", columns[headers.get("Orig Direction")]); - edgeMap.put("multiplicity", columns[headers.get("Revised Multiplicity")]); + if( rulesV9.hasEdgeRule(oldNodeA, oldNodeB, oldEdgeLabel) ){ + rulesToWriteV9.add(data); + } - edgeRules.add(edgeMap); + if( rulesV10.hasEdgeRule(oldNodeA, oldNodeB, oldEdgeLabel) ){ + rulesToWriteV10.add(data); + } + if( rulesV11.hasEdgeRule(oldNodeA, oldNodeB, oldEdgeLabel) ){ + rulesToWriteV11.add(data); + } } ++rowNum; } + + Configuration configuration = new Configuration(); + Template template = configuration.getTemplate("src/main/resources/edgerulesTemplate.ftlh"); + Writer file = new FileWriter(new File("src/main/resources/EdgeRulesWithNewLabels_v12.json")); + Map<String, List<EdgeRuleBean>> wrappedRules = new HashMap<>(); + wrappedRules.put("wrappedRules", rulesToWriteV12); + template.process(wrappedRules, file); + file.close(); + + file = new FileWriter(new File("src/main/resources/EdgeRulesWithNewLabels_v7.json")); + wrappedRules = new HashMap<>(); + wrappedRules.put("wrappedRules", rulesToWriteV7); + template.process(wrappedRules, file); + file.close(); + + file = new FileWriter(new File("src/main/resources/EdgeRulesWithNewLabels_v8.json")); + wrappedRules = new HashMap<>(); + wrappedRules.put("wrappedRules", rulesToWriteV8); + template.process(wrappedRules, file); + file.close(); + + + file = new FileWriter(new File("src/main/resources/EdgeRulesWithNewLabels_v9.json")); + wrappedRules = new HashMap<>(); + wrappedRules.put("wrappedRules", rulesToWriteV9); + template.process(wrappedRules, file); + file.close(); + + file = new FileWriter(new File("src/main/resources/EdgeRulesWithNewLabels_v10.json")); + wrappedRules = new HashMap<>(); + wrappedRules.put("wrappedRules", rulesToWriteV10); + template.process(wrappedRules, file); + file.close(); + + file = new FileWriter(new File("src/main/resources/EdgeRulesWithNewLabels_v11.json")); + wrappedRules = new HashMap<>(); + wrappedRules.put("wrappedRules", rulesToWriteV11); + template.process(wrappedRules, file); + file.close(); + } catch(Exception ex){ ex.printStackTrace(); } - edgeRulesMap.put("edgeRules", edgeRules); - - Collections.sort(edgeRules, new Comparator<Map<String, String>>() { - @Override - public int compare(Map<String, String> o1, Map<String, String> o2) { - return o1.get("nodes").compareTo(o2.get("nodes")); - } - }); - Configuration configuration = new Configuration(); - Template template = configuration.getTemplate("ajsc-aai/src/main/resources/EdgeRules.ftl"); - Writer file = new FileWriter(new File("ajsc-aai/src/main/resources" + "/" + "EdgeRules.txt")); - template.process(edgeRulesMap, file); } private static Map<String, Integer> retrieveHeaderMap(String line){ @@ -146,7 +198,7 @@ public class GenerateEdgeRules { int index = 0; for(String columnName : columnNames){ - map.put(columnName, index++); + map.put(columnName, index++); } return map; diff --git a/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java b/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java index cd48fc6a..8e1bf968 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java @@ -1,73 +1,48 @@ -/*- +/** * ============LICENSE_START======================================================= * org.onap.aai * ================================================================================ - * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. */ - package org.onap.aai.serialization.db; -import java.io.UnsupportedEncodingException; -import java.lang.reflect.Array; -import java.lang.reflect.InvocationTargetException; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; - -import javax.ws.rs.core.UriBuilder; - +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; +import com.google.common.base.CaseFormat; +import com.thinkaurelius.titan.core.SchemaViolationException; import org.apache.commons.collections.IteratorUtils; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree; -import org.apache.tinkerpop.gremlin.structure.Direction; -import org.apache.tinkerpop.gremlin.structure.Edge; -import org.apache.tinkerpop.gremlin.structure.Element; -import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.apache.tinkerpop.gremlin.structure.*; import org.javatuples.Pair; import org.javatuples.Triplet; import org.onap.aai.db.props.AAIProperties; import org.onap.aai.exceptions.AAIException; -import org.onap.aai.introspection.Introspector; -import org.onap.aai.introspection.IntrospectorFactory; -import org.onap.aai.introspection.Loader; -import org.onap.aai.introspection.LoaderFactory; -import org.onap.aai.introspection.ModelType; -import org.onap.aai.introspection.PropertyPredicates; -import org.onap.aai.introspection.Version; +import org.onap.aai.introspection.*; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; import org.onap.aai.introspection.sideeffect.DataCopy; import org.onap.aai.introspection.sideeffect.DataLinkReader; import org.onap.aai.introspection.sideeffect.DataLinkWriter; import org.onap.aai.introspection.sideeffect.SideEffectRunner; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.parsers.query.QueryParser; import org.onap.aai.parsers.uri.URIParser; import org.onap.aai.parsers.uri.URIToRelationshipObject; @@ -82,11 +57,22 @@ import org.onap.aai.util.AAIApiServerURLBase; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; import org.onap.aai.workarounds.NamingExceptions; +import org.onap.aai.logging.StopWatch; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; -import com.google.common.base.CaseFormat; -import com.thinkaurelius.titan.core.SchemaViolationException; +import javax.ws.rs.core.UriBuilder; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Array; +import java.lang.reflect.InvocationTargetException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class DBSerializer { @@ -100,6 +86,7 @@ public class DBSerializer { private final EdgeRules edgeRules = EdgeRules.getInstance(); private final Loader loader; private final String baseURL; + private double dbTimeMsecs = 0; /** * Instantiates a new DB serializer. * @@ -133,6 +120,7 @@ public class DBSerializer { if (isNewVertex) { v.property(AAIProperties.SOURCE_OF_TRUTH, this.sourceOfTruth); v.property(AAIProperties.CREATED_TS, timeNowInSec); + v.property(AAIProperties.AAI_UUID, UUID.randomUUID().toString()); } v.property(AAIProperties.RESOURCE_VERSION, timeNowInSec ); v.property(AAIProperties.LAST_MOD_TS, timeNowInSec); @@ -158,11 +146,15 @@ public class DBSerializer { * @throws AAIException the AAI exception */ public Vertex createNewVertex(Introspector wrappedObject) { - - - Vertex v = this.engine.tx().addVertex(); - touchStandardVertexProperties(wrappedObject.getDbName(), v, true); - + Vertex v; + try { + StopWatch.conditionalStart(); + v = this.engine.tx().addVertex(); + touchStandardVertexProperties(wrappedObject.getDbName(), v, true); + } + finally { + dbTimeMsecs += StopWatch.stopIfStarted(); + } return v; } @@ -205,7 +197,7 @@ public class DBSerializer { * @throws AAIUnknownObjectException */ public void serializeToDb(Introspector obj, Vertex v, QueryParser uriQuery, String identifier, String requestContext) throws AAIException, UnsupportedEncodingException { - + StopWatch.conditionalStart(); try { if (uriQuery.isDependent()) { //try to find the parent @@ -214,6 +206,7 @@ public class DBSerializer { Vertex parent = vertices.get(0); this.reflectDependentVertex(parent, v, obj, requestContext); } else { + dbTimeMsecs += StopWatch.stopIfStarted(); throw new AAIException("AAI_6114", "No parent Node of type " + uriQuery.getParentResultType() + " for " + identifier); } } else { @@ -221,17 +214,20 @@ public class DBSerializer { } } catch (SchemaViolationException e) { + dbTimeMsecs += StopWatch.stopIfStarted(); throw new AAIException("AAI_6117", e); } - + dbTimeMsecs += StopWatch.stopIfStarted(); } public void serializeSingleVertex(Vertex v, Introspector obj, String requestContext) throws UnsupportedEncodingException, AAIException { + StopWatch.conditionalStart(); try { boolean isTopLevel = obj.isTopLevel(); if (isTopLevel) { v.property(AAIProperties.AAI_URI, obj.getURI()); } + processObject(obj, v, requestContext); if (!isTopLevel) { URI uri = this.getURIForVertex(v); @@ -246,6 +242,9 @@ public class DBSerializer { } catch (SchemaViolationException e) { throw new AAIException("AAI_6117", e); } + finally { + dbTimeMsecs += StopWatch.stopIfStarted(); + } } /** @@ -555,7 +554,8 @@ public class DBSerializer { String parentUri = parent.<String>property(AAIProperties.AAI_URI).orElse(null); if (parentUri != null) { - String uri = obj.getURI(); + String uri; + uri = obj.getURI(); child.property(AAIProperties.AAI_URI, parentUri + uri); } processObject(obj, child, requestContext); @@ -565,7 +565,8 @@ public class DBSerializer { if (e == null) { String canBeLinked = obj.getMetadata(ObjectMetadata.CAN_BE_LINKED); if (canBeLinked != null && canBeLinked.equals("true")) { - boolean isFirst = !this.engine.getQueryBuilder(parent).createEdgeTraversal(EdgeType.TREE, parent, obj).hasNext(); + Loader ldrForCntxt = LoaderFactory.createLoaderForVersion(introspectionType, getVerForContext(requestContext)); + boolean isFirst = !this.engine.getQueryBuilder(ldrForCntxt, parent).createEdgeTraversal(EdgeType.TREE, parent, obj).hasNext(); if (isFirst) { child.property(AAIProperties.LINKED, true); } @@ -575,6 +576,16 @@ public class DBSerializer { return child; } + + private Version getVerForContext(String requestContext) { + Pattern pattern = Pattern.compile("v[0-9]+"); + Matcher m = pattern.matcher(requestContext); + if (!m.find()) { + return this.version; + } else { + return Version.valueOf(requestContext); + } + } /** * Db to object. @@ -597,14 +608,15 @@ public class DBSerializer { * @throws URISyntaxException */ public Introspector dbToObject(List<Vertex> vertices, final Introspector obj, int depth, boolean nodeOnly, String cleanUp) throws UnsupportedEncodingException, AAIException { - final int internalDepth; if (depth == Integer.MAX_VALUE) { internalDepth = depth--; } else { internalDepth = depth; } + StopWatch.conditionalStart(); if (vertices.size() > 1 && !obj.isContainer()) { + dbTimeMsecs += StopWatch.stopIfStarted(); throw new AAIException("AAI_6136", "query object mismatch: this object cannot hold multiple items." + obj.getDbName()); } else if (obj.isContainer()) { final List getList; @@ -624,6 +636,8 @@ public class DBSerializer { ExecutorService pool = GetAllPool.getInstance().getPool(); List<Future<Object>> futures = new ArrayList<>(); + + for (Vertex v : vertices) { Callable<Object> task = () -> { Set<Vertex> seen = new HashSet<>(); @@ -640,7 +654,11 @@ public class DBSerializer { for (Future<Object> future : futures) { try { getList.add(future.get()); - } catch (ExecutionException | InterruptedException e) { + } catch (ExecutionException e) { + dbTimeMsecs += StopWatch.stopIfStarted(); + throw new AAIException("AAI_4000", e); + } catch (InterruptedException e) { + dbTimeMsecs += StopWatch.stopIfStarted(); throw new AAIException("AAI_4000", e); } } @@ -653,7 +671,7 @@ public class DBSerializer { //obj = null; } - + dbTimeMsecs += StopWatch.stopIfStarted(); return obj; } @@ -747,7 +765,7 @@ public class DBSerializer { Introspector argumentObject = obj.newIntrospectorInstanceOfNestedProperty(property); Object result = dbToObject(argumentObject, childVertex, seen, depth, nodeOnly, cleanUp); - if (result != null) { + if (result != null) { getList.add(argumentObject.getUnderlyingObject()); } @@ -795,13 +813,15 @@ public class DBSerializer { if (nodeType == null) { throw new AAIException("AAI_6143"); } + Introspector obj = this.latestLoader.introspectorFromName(nodeType); Set<Vertex> seen = new HashSet<>(); int depth = 0; String cleanUp = "false"; boolean nodeOnly = true; + StopWatch.conditionalStart(); this.dbToObject(obj, v, seen, depth, nodeOnly, cleanUp); - + dbTimeMsecs += StopWatch.stopIfStarted(); return obj; } @@ -815,10 +835,11 @@ public class DBSerializer { int depth = AAIProperties.MAXIMUM_DEPTH; String cleanUp = "false"; boolean nodeOnly = false; + StopWatch.conditionalStart(); Tree<Element> tree = this.engine.getQueryEngine().findSubGraph(v, depth, nodeOnly); TreeBackedVertex treeVertex = new TreeBackedVertex(v, tree); this.dbToObject(obj, treeVertex, seen, depth, nodeOnly, cleanUp); - + dbTimeMsecs += StopWatch.stopIfStarted(); return obj; } /** @@ -959,7 +980,8 @@ public class DBSerializer { uriParser = new URIToRelationshipObject(relationshipObj.getLoader(), uri, this.baseURL); result = uriParser.getResult(); } catch (AAIException | URISyntaxException e) { - LOGGER.error("Error while processing edge relationship in version " + relationshipObj.getVersion() + " (bad vertex ID=" + tuple.get().getValue0().id().toString() + ": " + e.getMessage(), e); + LOGGER.error("Error while processing edge relationship in version " + relationshipObj.getVersion() + " (bad vertex ID=" + tuple.get().getValue0().id().toString() + ": " + + e.getMessage() + " " + LogFormatTools.getStackTop(e)); if ("true".equals(cleanUp)) { this.deleteWithTraversal(tuple.get().getValue0()); } @@ -1003,11 +1025,15 @@ public class DBSerializer { if (aaiUri != null && !overwrite) { uri = UriBuilder.fromPath(aaiUri).build(); } else { + StopWatch.conditionalStart(); Optional<Pair<Vertex, List<Introspector>>> tuple = this.getParents(this.loader, v, false); + dbTimeMsecs += StopWatch.stopIfStarted(); if (tuple.isPresent()) { List<Introspector> list = tuple.get().getValue1(); uri = this.getURIFromList(list); } + + } return uri; } @@ -1159,7 +1185,7 @@ public class DBSerializer { public boolean createEdge(Introspector relationship, Vertex inputVertex) throws UnsupportedEncodingException, AAIException { Vertex relatedVertex = null; - + StopWatch.conditionalStart(); QueryParser parser = engine.getQueryBuilder().createQueryFromRelationship(relationship); String label = null; @@ -1169,6 +1195,7 @@ public class DBSerializer { List<Vertex> results = parser.getQueryBuilder().toList(); if (results.isEmpty()) { + dbTimeMsecs += StopWatch.stopIfStarted(); AAIException e = new AAIException("AAI_6129", "Node of type " + parser.getResultType() + ". Could not find object at: " + parser.getUri()); e.getTemplateVars().add(parser.getResultType()); e.getTemplateVars().add(parser.getUri().toString()); @@ -1180,14 +1207,20 @@ public class DBSerializer { if (relatedVertex != null) { - Edge e = this.getEdgeBetween(EdgeType.COUSIN, inputVertex, relatedVertex, label); - if (e == null) { - edgeRules.addEdge(this.engine.asAdmin().getTraversalSource(), inputVertex, relatedVertex, label); - } else { - //attempted to link two vertexes already linked + Edge e; + try { + e = this.getEdgeBetween(EdgeType.COUSIN, inputVertex, relatedVertex, label); + if (e == null) { + edgeRules.addEdge(this.engine.asAdmin().getTraversalSource(), inputVertex, relatedVertex, label); + } else { + //attempted to link two vertexes already linked + } + } finally { + dbTimeMsecs += StopWatch.stopIfStarted(); } } + dbTimeMsecs += StopWatch.stopIfStarted(); return true; } @@ -1256,16 +1289,18 @@ public class DBSerializer { */ public Edge getEdgeBetween(EdgeType type, Vertex aVertex, Vertex bVertex, String label) throws AAIException { + StopWatch.conditionalStart(); if (bVertex != null) { List<Edge> edges = this.getEdgesBetween(type, aVertex, bVertex, label); if (!edges.isEmpty()) { + dbTimeMsecs += StopWatch.stopIfStarted(); return edges.get(0); } } - + dbTimeMsecs += StopWatch.stopIfStarted(); return null; } public Edge getEdgeBetween(EdgeType type, Vertex aVertex, Vertex bVertex) throws AAIException { @@ -1285,7 +1320,7 @@ public class DBSerializer { public boolean deleteEdge(Introspector relationship, Vertex inputVertex) throws UnsupportedEncodingException, AAIException { Vertex relatedVertex = null; - + StopWatch.conditionalStart(); QueryParser parser = engine.getQueryBuilder().createQueryFromRelationship(relationship); List<Vertex> results = parser.getQueryBuilder().toList(); @@ -1296,6 +1331,7 @@ public class DBSerializer { } if (results.isEmpty()) { + dbTimeMsecs += StopWatch.stopIfStarted(); return false; } @@ -1304,12 +1340,15 @@ public class DBSerializer { try { edge = this.getEdgeBetween(EdgeType.COUSIN, inputVertex, relatedVertex, label); } catch (NoEdgeRuleFoundException e) { + dbTimeMsecs += StopWatch.stopIfStarted(); throw new AAIException("AAI_6129", e); } if (edge != null) { edge.remove(); + dbTimeMsecs += StopWatch.stopIfStarted(); return true; } else { + dbTimeMsecs += StopWatch.stopIfStarted(); return false; } @@ -1322,10 +1361,12 @@ public class DBSerializer { * @throws IllegalStateException the illegal state exception */ public void deleteItemsWithTraversal(List<Vertex> vertexes) throws IllegalStateException { + for (Vertex v : vertexes) { - LOGGER.debug("About to delete the vertex with id: " + v.id()); + LOGGER.debug("About to delete the vertex with id: " + v.id()); deleteWithTraversal(v); } + } /** @@ -1334,7 +1375,7 @@ public class DBSerializer { * @param startVertex the start vertex */ public void deleteWithTraversal(Vertex startVertex) { - + StopWatch.conditionalStart(); List<Vertex> results = this.engine.getQueryEngine().findDeletable(startVertex); for (Vertex v : results) { @@ -1342,7 +1383,7 @@ public class DBSerializer { v.remove(); } - + dbTimeMsecs += StopWatch.stopIfStarted(); } /** @@ -1385,8 +1426,10 @@ public class DBSerializer { nodeType = vertex.<String>property(AAIProperties.NODE_TYPE).orElse(null); if (enableResourceVersion && !this.verifyResourceVersion("delete", nodeType, vertex.<String>property(AAIProperties.RESOURCE_VERSION).orElse(null), resourceVersion, nodeType)) { } + StopWatch.conditionalStart(); List<Object> preventDeleteVertices = this.engine.asAdmin().getReadOnlyTraversalSource().V(vertex).union(__.inE().has(EdgeProperty.PREVENT_DELETE.toString(), AAIDirection.IN.toString()).outV().values(AAIProperties.NODE_TYPE), __.outE().has(EdgeProperty.PREVENT_DELETE.toString(), AAIDirection.OUT.toString()).inV().values(AAIProperties.NODE_TYPE)).dedup().toList(); + dbTimeMsecs += StopWatch.stopIfStarted(); if (!preventDeleteVertices.isEmpty()) { aaiExceptionCode = "AAI_6110"; errorDetail = String.format("Object is being reference by additional objects preventing it from being deleted. Please clean up references from the following types %s", preventDeleteVertices); @@ -1499,4 +1542,194 @@ public class DBSerializer { runner.execute(obj, self); } + public double getDBTimeMsecs() { + return (dbTimeMsecs); + } + + /** + * Db to object With Filters + * This is for a one-time run with Tenant Isloation to only filter relationships + * TODO: Chnage the original dbToObject to take filter parent/cousins + * + * @param vertices the vertices + * @param obj the obj + * @param depth the depth + * @param cleanUp the clean up + * @return the introspector + * @throws AAIException the AAI exception + * @throws IllegalAccessException the illegal access exception + * @throws IllegalArgumentException the illegal argument exception + * @throws InvocationTargetException the invocation target exception + * @throws SecurityException the security exception + * @throws InstantiationException the instantiation exception + * @throws NoSuchMethodException the no such method exception + * @throws UnsupportedEncodingException the unsupported encoding exception + * @throws MalformedURLException the malformed URL exception + * @throws AAIUnknownObjectException + * @throws URISyntaxException + */ + //TODO - See if you can merge the 2 dbToObjectWithFilters + public Introspector dbToObjectWithFilters(Introspector obj, Vertex v, Set<Vertex> seen, int depth, boolean nodeOnly, List<String> filterCousinNodes, List<String> filterParentNodes) throws AAIException, UnsupportedEncodingException { + String cleanUp = "false"; + if (depth < 0) { + return null; + } + depth--; + seen.add(v); + boolean modified = false; + for (String property : obj.getProperties(PropertyPredicates.isVisible())) { + List<Object> getList = null; + Vertex[] vertices = null; + + if (!(obj.isComplexType(property) || obj.isListType(property))) { + this.copySimpleProperty(property, obj, v); + modified = true; + } else { + if (obj.isComplexType(property)) { + /* container case */ + + if (!property.equals("relationship-list") && depth >= 0) { + Introspector argumentObject = obj.newIntrospectorInstanceOfProperty(property); + Object result = dbToObjectWithFilters(argumentObject, v, seen, depth+1, nodeOnly, filterCousinNodes, filterParentNodes); + if (result != null) { + obj.setValue(property, argumentObject.getUnderlyingObject()); + modified = true; + } + } else if (property.equals("relationship-list") && !nodeOnly){ + /* relationships need to be handled correctly */ + Introspector relationshipList = obj.newIntrospectorInstanceOfProperty(property); + relationshipList = createFilteredRelationshipList(v, relationshipList, cleanUp, filterCousinNodes); + if (relationshipList != null) { + modified = true; + obj.setValue(property, relationshipList.getUnderlyingObject()); + modified = true; + } + + } + } else if (obj.isListType(property)) { + + if (property.equals("any")) { + continue; + } + String genericType = obj.getGenericTypeClass(property).getSimpleName(); + if (obj.isComplexGenericType(property) && depth >= 0) { + final String childDbName = convertFromCamelCase(genericType); + String vType = v.<String>property(AAIProperties.NODE_TYPE).orElse(null); + EdgeRule rule; + + boolean isthisParentRequired = filterParentNodes.parallelStream().anyMatch(childDbName::contains); + + + + rule = edgeRules.getEdgeRule(EdgeType.TREE, vType, childDbName); + if (!rule.getContains().equals(AAIDirection.NONE.toString()) && isthisParentRequired) { + //vertices = this.queryEngine.findRelatedVertices(v, Direction.OUT, rule.getLabel(), childDbName); + Direction ruleDirection = rule.getDirection(); + Iterator<Vertex> itr = v.vertices(ruleDirection, rule.getLabel()); + List<Vertex> verticesList = (List<Vertex>)IteratorUtils.toList(itr); + itr = verticesList.stream().filter(item -> { + return item.property(AAIProperties.NODE_TYPE).orElse("").equals(childDbName); + }).iterator(); + if (itr.hasNext()) { + getList = (List<Object>)obj.getValue(property); + } + int processed = 0; + int removed = 0; + while (itr.hasNext()) { + Vertex childVertex = itr.next(); + if (!seen.contains(childVertex)) { + Introspector argumentObject = obj.newIntrospectorInstanceOfNestedProperty(property); + + Object result = dbToObjectWithFilters(argumentObject, childVertex, seen, depth, nodeOnly, filterCousinNodes, filterParentNodes); + if (result != null) { + getList.add(argumentObject.getUnderlyingObject()); + } + + processed++; + } else { + removed++; + LOGGER.warn("Cycle found while serializing vertex id={}", childVertex.id().toString()); + } + } + if (processed == 0) { + //vertices were all seen, reset the list + getList = null; + } + if (processed > 0) { + modified = true; + } + } + } else if (obj.isSimpleGenericType(property)) { + List<Object> temp = this.engine.getListProperty(v, property); + if (temp != null) { + getList = (List<Object>)obj.getValue(property); + getList.addAll(temp); + modified = true; + } + + } + + } + + } + } + + //no changes were made to this obj, discard the instance + if (!modified) { + return null; + } + this.enrichData(obj, v); + return obj; + + } + + /** + * Creates the relationship list with the filtered node types. + * + * @param v the v + * @param obj the obj + * @param cleanUp the clean up + * @return the object + * @throws InstantiationException the instantiation exception + * @throws IllegalAccessException the illegal access exception + * @throws IllegalArgumentException the illegal argument exception + * @throws InvocationTargetException the invocation target exception + * @throws NoSuchMethodException the no such method exception + * @throws SecurityException the security exception + * @throws UnsupportedEncodingException the unsupported encoding exception + * @throws AAIException the AAI exception + * @throws MalformedURLException the malformed URL exception + * @throws URISyntaxException + */ + private Introspector createFilteredRelationshipList(Vertex v, Introspector obj, String cleanUp, List<String> filterNodes) throws UnsupportedEncodingException, AAIException { + List<Vertex> allCousins = this.engine.getQueryEngine().findCousinVertices(v); + + Iterator<Vertex> cousinVertices = allCousins.stream().filter(item -> { + String node = (String)item.property(AAIProperties.NODE_TYPE).orElse(""); + return filterNodes.parallelStream().anyMatch(node::contains); + }).iterator(); + + + List<Vertex> cousins = (List<Vertex>)IteratorUtils.toList(cousinVertices); + + //items.parallelStream().anyMatch(inputStr::contains) + List<Object> relationshipObjList = obj.getValue("relationship"); + for (Vertex cousin : cousins) { + + Introspector relationshipObj = obj.newIntrospectorInstanceOfNestedProperty("relationship"); + Object result = processEdgeRelationship(relationshipObj, cousin, cleanUp, null); + if (result != null) { + relationshipObjList.add(result); + } + + + } + + if (relationshipObjList.isEmpty()) { + return null; + } else { + return obj; + } + } + } diff --git a/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeProperty.java b/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeProperty.java index ecd749c7..65753a26 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeProperty.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeProperty.java @@ -1,21 +1,23 @@ -/*- +/** * ============LICENSE_START======================================================= * org.onap.aai * ================================================================================ - * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. */ package org.onap.aai.serialization.db; @@ -24,7 +26,8 @@ public enum EdgeProperty { CONTAINS("contains-other-v"), DELETE_OTHER_V("delete-other-v"), SVC_INFRA("SVC-INFRA"), - PREVENT_DELETE("prevent-delete"); + PREVENT_DELETE("prevent-delete"), + DESCRIPTION("description"); private final String name; private EdgeProperty(String name) { diff --git a/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeRule.java b/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeRule.java index ae9e96c0..0a32db7c 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeRule.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeRule.java @@ -1,30 +1,31 @@ -/*- +/** * ============LICENSE_START======================================================= * org.onap.aai * ================================================================================ - * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. */ - package org.onap.aai.serialization.db; +import org.apache.tinkerpop.gremlin.structure.Direction; + import java.util.EnumMap; import java.util.Map; -import org.apache.tinkerpop.gremlin.structure.Direction; - public class EdgeRule { private String label = ""; @@ -32,14 +33,32 @@ public class EdgeRule { private Direction direction = null; private Map<EdgeProperty, String> edgeProperties = null; private boolean isDefaultEdge = false; - + private String from; + private String to; + /** * Instantiates a new edge rule. */ public EdgeRule() { edgeProperties = new EnumMap<>(EdgeProperty.class); } - + + public String getFrom() { + return from; + } + + public void setFrom(String from) { + this.from = from; + } + + public String getTo() { + return to; + } + + public void setTo(String to) { + this.to = to; + } + /** * Gets the label. * @@ -210,14 +229,14 @@ public class EdgeRule { public boolean isDefault() { return isDefaultEdge; } - + public void setIsDefault(boolean isDefault) { this.isDefaultEdge = isDefault; } - + public void setIsDefault(String isDefault) { this.isDefaultEdge = "true".equals(isDefault); } - - + + } diff --git a/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeRules.java b/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeRules.java index 349f9e6b..622c3067 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeRules.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/db/EdgeRules.java @@ -25,14 +25,8 @@ import static com.jayway.jsonpath.Criteria.where; import static com.jayway.jsonpath.Filter.filter; import java.io.InputStream; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Optional; -import java.util.Scanner; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; @@ -243,6 +237,8 @@ public class EdgeRules { for (Entry<EdgeProperty, String> entry : propMap.entrySet()) { edge.property(entry.getKey().toString(), entry.getValue()); } + + edge.property(AAIProperties.AAI_UUID, UUID.randomUUID().toString()); } /** @@ -652,11 +648,17 @@ public class EdgeRules { rule.setDeleteOtherV(edge.get(EdgeProperty.DELETE_OTHER_V.toString())); rule.setServiceInfrastructure(edge.get(EdgeProperty.SVC_INFRA.toString())); rule.setPreventDelete(edge.get(EdgeProperty.PREVENT_DELETE.toString())); + rule.setTo(edge.get("to")); + rule.setFrom(edge.get("from")); if (edge.containsKey("default")) { rule.setIsDefault(edge.get("default")); } - return rule; + if(rule.getFrom().equals(rule.getTo())){ + return this.flipDirection(rule); + } else { + return rule; + } } /** @@ -777,6 +779,11 @@ public class EdgeRules { */ private void verifyRule(Map<String, String> rule) { for (EdgeProperty prop : EdgeProperty.values()) { + + // Description is not required as it is only set for v12 versions + if("description".equals(prop.toString())){ + continue; + } if (!rule.containsKey(prop.toString())) { /* Throws RuntimeException as rule definition errors * cannot be recovered from, and should never happen anyway diff --git a/aai-core/src/main/java/org/onap/aai/serialization/db/InMemoryGraphSingleton.java b/aai-core/src/main/java/org/onap/aai/serialization/db/InMemoryGraphSingleton.java new file mode 100644 index 00000000..091ab457 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/serialization/db/InMemoryGraphSingleton.java @@ -0,0 +1,60 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.serialization.db; + +import org.onap.aai.dbmap.DBConnectionType; + +import com.thinkaurelius.titan.core.TitanGraph; + +public class InMemoryGraphSingleton extends GraphSingleton { + + private static TitanGraph inMemgraph; + + private static class Helper { + private static final InMemoryGraphSingleton INSTANCE = new InMemoryGraphSingleton(); + } + + /** + * Gets the single instance of TitanGraphSingleton. + * + * @return single instance of TitanGraphSingleton + */ + public static InMemoryGraphSingleton getInstance(TitanGraph graph) { + inMemgraph = graph; + return Helper.INSTANCE; + } + + /** + * Gets the tx graph. + * + * @return the tx graph + */ + @Override + public TitanGraph getTxGraph() { + return inMemgraph; + } + + @Override + public TitanGraph getTxGraph(DBConnectionType connectionType) { + return inMemgraph; + } +} diff --git a/aai-core/src/main/java/org/onap/aai/serialization/db/exceptions/MultipleEdgeRuleFoundException.java b/aai-core/src/main/java/org/onap/aai/serialization/db/exceptions/MultipleEdgeRuleFoundException.java index bb4ba4df..f94eb407 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/db/exceptions/MultipleEdgeRuleFoundException.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/db/exceptions/MultipleEdgeRuleFoundException.java @@ -1,21 +1,23 @@ -/*- +/** * ============LICENSE_START======================================================= * org.onap.aai * ================================================================================ - * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. */ package org.onap.aai.serialization.db.exceptions; @@ -25,7 +27,7 @@ import org.onap.aai.exceptions.AAIException; public class MultipleEdgeRuleFoundException extends AAIException { private static final long serialVersionUID = -906843868234976763L; - + public MultipleEdgeRuleFoundException(String message) { super("AAI_6107", message); } diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/InMemoryDBEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/InMemoryDBEngine.java new file mode 100644 index 00000000..197aff94 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/InMemoryDBEngine.java @@ -0,0 +1,207 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.serialization.engines; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ReadOnlyStrategy; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; + +import org.onap.aai.dbmap.DBConnectionType; +import org.onap.aai.introspection.Loader; +import org.onap.aai.query.builder.GremlinTraversal; +import org.onap.aai.query.builder.GremlinUnique; +import org.onap.aai.query.builder.QueryBuilder; +import org.onap.aai.query.builder.TraversalQuery; +import org.onap.aai.serialization.db.InMemoryGraphSingleton; +import org.onap.aai.serialization.engines.query.GraphTraversalQueryEngine; +import org.onap.aai.serialization.engines.query.QueryEngine; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; + +import com.thinkaurelius.titan.core.TitanGraph; + +public class InMemoryDBEngine extends TransactionalGraphEngine { + + /** + * Instantiates a new titan DB engine. + * + * @param style + * the style + * @param loader + * the loader + */ + private TitanGraph graph = null; + + private static final TransactionalGraphEngine.Admin admin = null; + + public InMemoryDBEngine(QueryStyle style, DBConnectionType connectionType, Loader loader, TitanGraph graph) { + super(style, loader, connectionType, InMemoryGraphSingleton.getInstance(graph)); + this.graph = graph; + } + + /** + * Instantiates a new titan DB engine. + * + * @param style + * the style + * @param loader + * the loader + * @param connect + * the connect + */ + public InMemoryDBEngine(QueryStyle style, Loader loader, boolean connect, TitanGraph graph) { + super(style, loader); + if (connect) { + this.singleton = InMemoryGraphSingleton.getInstance(graph); + } + this.graph = graph; + } + + @Override + public QueryEngine getQueryEngine() { + + if (style.equals(QueryStyle.TRAVERSAL)) { + + GraphTraversalSource traversalSource = graph.traversal(); + return new GraphTraversalQueryEngine(traversalSource); + + } else { + throw new IllegalArgumentException("Query Engine type not recognized"); + } + + } + + @Override + public QueryBuilder<Vertex> getQueryBuilder(QueryStyle style, Loader loader) { + if (style.equals(QueryStyle.GREMLIN_TRAVERSAL)) { + return new GremlinTraversal<>(loader, graph.traversal()); + } else if (style.equals(QueryStyle.TRAVERSAL)) { + return new TraversalQuery<>(loader, graph.traversal()); + + } else { + throw new IllegalArgumentException("Query Builder type is Not recognized"); + } + + } + + /** + * {@inheritDoc} + */ + @Override + public boolean setListProperty(Vertex v, String name, List<?> objs) { + + // clear out list full replace style + + Iterator<VertexProperty<Object>> iterator = v.properties(name); + while (iterator.hasNext()) { + iterator.next().remove(); + } + if (objs != null) { + for (Object obj : objs) { + v.property(name, obj); + } + } + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public List<Object> getListProperty(Vertex v, String name) { + + List<Object> result = new ArrayList<>(); + + Iterator<VertexProperty<Object>> iterator = v.properties(name); + + while (iterator.hasNext()) { + result.add(iterator.next().value()); + } + + if (result.isEmpty()) { + result = null; + } + + return result; + + } + + @Override + public QueryBuilder<Vertex> getQueryBuilder() { + return getQueryBuilder(this.loader); + } + + @Override + public QueryBuilder<Vertex> getQueryBuilder(Loader loader) { + if (style.equals(QueryStyle.GREMLIN_TRAVERSAL)) { + return new GremlinTraversal<>(loader, this.asAdmin().getTraversalSource()); + } else if (style.equals(QueryStyle.GREMLIN_UNIQUE)) { + return new GremlinUnique<>(loader, this.asAdmin().getTraversalSource()); + } else if (style.equals(QueryStyle.TRAVERSAL)) { + return new TraversalQuery<>(loader, graph.traversal()); + } else { + throw new IllegalArgumentException("Query Builder type not recognized"); + } + + } + + @Override + public QueryBuilder<Vertex> getQueryBuilder(Vertex start) { + return getQueryBuilder(this.loader, start); + } + + public GraphTraversalSource getTraversalSource() { + return graph.traversal(); + } + + @Override + public QueryBuilder<Vertex> getQueryBuilder(Loader loader, Vertex start) { + if (style.equals(QueryStyle.GREMLIN_TRAVERSAL)) { + return new GremlinTraversal<>(loader, graph.traversal(), start); + } else if (style.equals(QueryStyle.GREMLIN_UNIQUE)) { + return new GremlinUnique<>(loader, this.asAdmin().getTraversalSource(), start); + } else if (style.equals(QueryStyle.TRAVERSAL)) { + return new TraversalQuery<>(loader, graph.traversal(), start); + } else { + throw new IllegalArgumentException("Query Builder type not recognized"); + } + + } + + @Override + public Graph startTransaction() { + if (this.tx() == null) { + this.currentTx = graph.newTransaction(); + this.currentTraversal = this.tx().traversal(); + this.readOnlyTraversal = this.tx() + .traversal(GraphTraversalSource.build().with(ReadOnlyStrategy.instance())); + } + return currentTx; + } + + + +} diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java index 7efe2ea9..27733039 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java @@ -42,6 +42,7 @@ import org.apache.tinkerpop.gremlin.structure.Element; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.onap.aai.db.props.AAIProperties; import org.onap.aai.introspection.Loader; +import org.onap.aai.logging.StopWatch; /* * This class needs some big explanation despite its compact size. @@ -68,10 +69,15 @@ public class GraphTraversalQueryEngine extends QueryEngine { */ @Override public List<Vertex> findParents(Vertex start) { - - @SuppressWarnings("unchecked") - final GraphTraversal<Vertex, Vertex> pipe = this.g.V(start).emit(v -> true).repeat(__.union(__.inE().has(CONTAINS.toString(), OUT.toString()).outV(), __.outE().has(CONTAINS.toString(), IN.toString()).inV())); - return pipe.toList(); + try { + StopWatch.conditionalStart(); + @SuppressWarnings("unchecked") + final GraphTraversal<Vertex, Vertex> pipe = this.g.V(start).emit(v -> true).repeat(__.union(__.inE().has(CONTAINS.toString(), OUT.toString()).outV(), __.outE().has(CONTAINS.toString(), IN.toString()).inV())); + return pipe.toList(); + } + finally { + dbTimeMsecs += StopWatch.stopIfStarted(); + } } /** @@ -129,7 +135,7 @@ public class GraphTraversalQueryEngine extends QueryEngine { __.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(), __.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV() ) - ); + ).dedup(); return pipe.toList(); } @@ -201,5 +207,8 @@ public class GraphTraversalQueryEngine extends QueryEngine { return pipeline.toList(); } + public double getDBTimeMsecs() { + return (dbTimeMsecs); + } } diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java index 5a468b79..0e000948 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java @@ -34,8 +34,8 @@ import java.util.List; public abstract class QueryEngine { - protected final GraphTraversalSource g; - + final protected GraphTraversalSource g; + protected double dbTimeMsecs = 0; /** * Instantiates a new query engine. * @@ -145,4 +145,6 @@ public abstract class QueryEngine { */ public abstract List<Vertex> findCousinVertices(Vertex start); + public abstract double getDBTimeMsecs(); + } diff --git a/aai-core/src/main/java/org/onap/aai/serialization/queryformats/FormatFactory.java b/aai-core/src/main/java/org/onap/aai/serialization/queryformats/FormatFactory.java index 21a7e0fe..aec0be27 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/queryformats/FormatFactory.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/queryformats/FormatFactory.java @@ -31,9 +31,6 @@ import org.onap.aai.serialization.queryformats.exceptions.QueryParamInjectionExc import org.onap.aai.serialization.queryformats.utils.QueryParamInjector; import org.onap.aai.serialization.queryformats.utils.UrlBuilder; -import javax.ws.rs.core.MultivaluedHashMap; -import javax.ws.rs.core.MultivaluedMap; - public class FormatFactory { private final Loader loader; diff --git a/aai-core/src/main/java/org/onap/aai/serialization/queryformats/FormatMapper.java b/aai-core/src/main/java/org/onap/aai/serialization/queryformats/FormatMapper.java index 27d204a6..44018354 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/queryformats/FormatMapper.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/queryformats/FormatMapper.java @@ -21,9 +21,9 @@ */ package org.onap.aai.serialization.queryformats; +import com.google.gson.JsonObject; import org.onap.aai.serialization.queryformats.exceptions.AAIFormatQueryResultFormatNotSupported; import org.onap.aai.serialization.queryformats.exceptions.AAIFormatVertexException; -import com.google.gson.JsonObject; public interface FormatMapper { diff --git a/aai-core/src/main/java/org/onap/aai/serialization/queryformats/Formatter.java b/aai-core/src/main/java/org/onap/aai/serialization/queryformats/Formatter.java index b145d423..698aef6e 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/queryformats/Formatter.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/queryformats/Formatter.java @@ -21,19 +21,20 @@ */ package org.onap.aai.serialization.queryformats; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import org.onap.aai.serialization.queryformats.exceptions.AAIFormatQueryResultFormatNotSupported; -import org.onap.aai.serialization.queryformats.exceptions.AAIFormatVertexException; - import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import org.onap.aai.logging.LogFormatTools; +import org.onap.aai.serialization.queryformats.exceptions.AAIFormatQueryResultFormatNotSupported; +import org.onap.aai.serialization.queryformats.exceptions.AAIFormatVertexException; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + public class Formatter { private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(Formatter.class); @@ -50,14 +51,14 @@ public class Formatter { Stream<Object> stream = null; JsonObject result = new JsonObject(); JsonArray body = new JsonArray(); - + if (this.format instanceof Count) { JsonObject countResult; try { countResult = format.formatObject(queryResults); - body.add(countResult); + body.add(countResult); } catch (Exception e) { - LOGGER.warn("Failed to format result type of the query", e); + LOGGER.warn("Failed to format result type of the query " + LogFormatTools.getStackTop(e)); } } else { if (queryResults.size() >= format.parallelThreshold()) { @@ -65,16 +66,16 @@ public class Formatter { } else { stream = queryResults.stream(); } - + final boolean isParallel = stream.isParallel(); - + stream.map(o -> { try { return Optional.<JsonObject>of(format.formatObject(o)); } catch (AAIFormatVertexException e) { - LOGGER.warn("Failed to format vertex, returning a partial list", e); + LOGGER.warn("Failed to format vertex, returning a partial list " + LogFormatTools.getStackTop(e)); } catch (AAIFormatQueryResultFormatNotSupported e) { - LOGGER.warn("Failed to format result type of the query", e); + LOGGER.warn("Failed to format result type of the query " + LogFormatTools.getStackTop(e)); } return Optional.<JsonObject>empty(); diff --git a/aai-core/src/main/java/org/onap/aai/tasks/ScheduledTasks.java b/aai-core/src/main/java/org/onap/aai/tasks/ScheduledTasks.java index 9d2e8b68..e1f554ad 100644 --- a/aai-core/src/main/java/org/onap/aai/tasks/ScheduledTasks.java +++ b/aai-core/src/main/java/org/onap/aai/tasks/ScheduledTasks.java @@ -32,6 +32,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LoggingContext.StatusCode; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; import com.att.eelf.configuration.EELFLogger; @@ -57,9 +58,15 @@ public class ScheduledTasks { public void loadAAIProperties() { final UUID transId = UUID.randomUUID(); + LoggingContext.init(); LoggingContext.requestId(transId); LoggingContext.partnerName(FROM_APP_ID); LoggingContext.component(COMPONENT); + LoggingContext.targetEntity("AAI"); + LoggingContext.targetServiceName("loadAAIProperties"); + LoggingContext.serviceName("AAI"); + LoggingContext.statusCode(StatusCode.COMPLETE); + LoggingContext.responseCode(LoggingContext.SUCCESS); String dir = FilenameUtils.getFullPathNoEndSeparator(GlobalPropFileName); if (dir == null || dir.length() < 3) { @@ -82,7 +89,7 @@ public class ScheduledTasks { long curTSTm = curTS.getTime(); if (curTSTm - lastModTm < PROPERTY_READ_INTERVAL + 1000) { AAIConfig.reloadConfig(); - LOGGER.info("reloaded from aaiconfig.properties"); + LOGGER.debug("reloaded from aaiconfig.properties"); } break; } diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIApiServerURLBase.java b/aai-core/src/main/java/org/onap/aai/util/AAIApiServerURLBase.java index 45f671ef..e458c4f9 100644 --- a/aai-core/src/main/java/org/onap/aai/util/AAIApiServerURLBase.java +++ b/aai-core/src/main/java/org/onap/aai/util/AAIApiServerURLBase.java @@ -73,8 +73,7 @@ public class AAIApiServerURLBase { */ public static String get(Version v) throws AAIException { String hostName = null; - hostName = AAIApiServerURLBase.get(); - + hostName = AAIApiServerURLBase.get(); return hostName; } diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java b/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java index 445fa0dd..57bf2cc7 100644 --- a/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java +++ b/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java @@ -30,6 +30,10 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Properties; import java.util.Timer; +import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LoggingContext.StatusCode; + +import java.util.UUID; import org.eclipse.jetty.util.security.Password; @@ -62,6 +66,16 @@ public class AAIConfig { * @throws AAIException the AAI exception */ public synchronized static void init() throws AAIException{ + + LoggingContext.save(); + LoggingContext.component("config"); + LoggingContext.partnerName("NA"); + LoggingContext.targetEntity("AAI"); + LoggingContext.requestId(UUID.randomUUID().toString()); + LoggingContext.serviceName("AAI"); + LoggingContext.targetServiceName("init"); + LoggingContext.statusCode(StatusCode.COMPLETE); + LOGGER.info("Initializing AAIConfig"); ArrayList<String> genericVnfBools = new ArrayList<String>(); @@ -96,6 +110,7 @@ public class AAIConfig { } else { LOGGER.info("A&AI Server Node Name = " + AAIConstants.AAI_NODENAME); } + LoggingContext.restore(); } /** @@ -137,7 +152,7 @@ public class AAIConfig { String propFileName = GLOBAL_PROP_FILE_NAME; Properties newServerProps = null; - LOGGER.info("Reloading config from " + propFileName); + LOGGER.debug("Reloading config from " + propFileName); try { InputStream is = new FileInputStream(propFileName); diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java b/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java index e201b6d8..9c875ec8 100644 --- a/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java +++ b/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java @@ -47,6 +47,8 @@ public final class AAIConstants { public static final String CACHED_DB_CONFIG = AAI_HOME_ETC_APP_PROPERTIES + "titan-cached.properties"; public static final String AAI_HOME_ETC_OXM = AAI_HOME_ETC + "oxm" + AAI_FILESEP; public static final String AAI_EVENT_DMAAP_PROPS = AAI_HOME_ETC_APP_PROPERTIES + "aaiEventDMaaPPublisher.properties"; + public static final String AAI_HOME_ETC_SCRIPT = AAI_HOME_ETC + AAI_FILESEP + "scriptdata" + AAI_FILESEP; + public static final String AAI_PROV_LOGBACK_PROPS = "prov-logback.xml"; public static final String AAI_GETRES_LOGBACK_PROPS = "getres-logback.xml"; @@ -96,6 +98,14 @@ public final class AAIConstants { public static final String AAI_BULKCONSUMER_LIMIT = "aai.bulkconsumer.payloadlimit"; public static final String AAI_BULKCONSUMER_OVERRIDE_LIMIT = "aai.bulkconsumer.payloadoverride"; + public static final String AAI_TRAVERSAL_TIMEOUT_LIMIT = "aai.traversal.timeoutlimit"; + public static final String AAI_TRAVERSAL_TIMEOUT_ENABLED = "aai.traversal.timeoutenabled"; + public static final String AAI_TRAVERSAL_TIMEOUT_APP = "aai.traversal.timeout.appspecific"; + + public static final String AAI_CRUD_TIMEOUT_LIMIT = "aai.crud.timeoutlimit"; + public static final String AAI_CRUD_TIMEOUT_ENABLED = "aai.crud.timeoutenabled"; + public static final String AAI_CRUD_TIMEOUT_APP = "aai.crud.timeout.appspecific"; + public static final String AAI_LOGGING_HBASE_INTERCEPTOR = "aai.logging.hbase.interceptor"; public static final String AAI_LOGGING_HBASE_ENABLED = "aai.logging.hbase.enabled"; public static final String AAI_LOGGING_HBASE_LOGREQUEST = "aai.logging.hbase.logrequest"; @@ -124,7 +134,7 @@ public final class AAIConstants { public static final String AAI_SKIPREALTIME_GROOMING = (System.getProperty("aai.skiprealtime.grooming") == null) ? "true" : System.getProperty("aai.skiprealtime.grooming"); - /*** UEB ***/ + /*** UEB ***/ public static final String UEB_PUB_PARTITION_AAI = "AAI"; public static final String UEB_PUB_AAI_VCE_INTERFACE_DATA_TOPIC = "ueb.pub.aai.vce.interface.data.topic"; diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIRSyncUtility.java b/aai-core/src/main/java/org/onap/aai/util/AAIRSyncUtility.java index f20346eb..448d067a 100644 --- a/aai-core/src/main/java/org/onap/aai/util/AAIRSyncUtility.java +++ b/aai-core/src/main/java/org/onap/aai/util/AAIRSyncUtility.java @@ -35,7 +35,7 @@ import java.util.StringTokenizer; import org.onap.aai.exceptions.AAIException; import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; - +import org.onap.aai.logging.ErrorLogHelper; public class AAIRSyncUtility { @@ -145,7 +145,7 @@ public class AAIRSyncUtility { } } } catch ( Exception e) { - LOGGER.error("no server found processing serverList for host " + host + ": " + e.getMessage() + " (AAI_4000)"); + ErrorLogHelper.logError("AAI_4000", "no server found processing serverList for host " + host + ": " + e.getMessage()); } } diff --git a/aai-core/src/main/java/org/onap/aai/util/AAISystemExitUtil.java b/aai-core/src/main/java/org/onap/aai/util/AAISystemExitUtil.java new file mode 100644 index 00000000..9149d0b4 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/util/AAISystemExitUtil.java @@ -0,0 +1,34 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.util; + +import org.onap.aai.dbmap.AAIGraph; + +public class AAISystemExitUtil { + + public static void systemExitCloseAAIGraph(int code) { + if (AAIGraph.isInit()) { + AAIGraph.getInstance().graphShutdown(); + } + System.exit(code); + } +} diff --git a/aai-core/src/main/java/org/onap/aai/util/AutoGenerateHtml.java b/aai-core/src/main/java/org/onap/aai/util/AutoGenerateHtml.java new file mode 100644 index 00000000..f0489da2 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/util/AutoGenerateHtml.java @@ -0,0 +1,67 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +package org.onap.aai.util; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.ListIterator; + +import org.onap.aai.introspection.Version; +import org.onap.aai.util.swagger.GenerateSwagger; + +import freemarker.template.TemplateException; + +public class AutoGenerateHtml { + + public static final String DEFAULT_SCHEMA_DIR = "../aai-schema"; + //if the program is run from aai-common, use this directory as default" + public static final String ALT_SCHEMA_DIR = "aai-schema"; + //used to check to see if program is run from aai-core + public static final String DEFAULT_RUN_DIR = "aai-core"; + + public static void main(String[] args) throws IOException, TemplateException { + String savedProperty = System.getProperty("aai.generate.version"); + List<Version> versionsToGen = Arrays.asList(Version.values()); + Collections.sort(versionsToGen); + Collections.reverse(versionsToGen); + ListIterator<Version> versionIterator = versionsToGen.listIterator(); + String schemaDir; + if(System.getProperty("user.dir") != null && !System.getProperty("user.dir").contains(DEFAULT_RUN_DIR)) { + schemaDir = ALT_SCHEMA_DIR; + } + else { + schemaDir = DEFAULT_SCHEMA_DIR; + } + while (versionIterator.hasNext()) { + System.setProperty("aai.generate.version", versionIterator.next().toString()); + String yamlFile = schemaDir + "/src/main/resources/aai_swagger_yaml/aai_swagger_" + System.getProperty("aai.generate.version")+ ".yaml"; + File swaggerYamlFile = new File(yamlFile); + if(swaggerYamlFile.exists()) { + GenerateSwagger.main(args); + } + } + String versionToGenerate = System.setProperty("aai.generate.version", savedProperty); + } +} diff --git a/aai-core/src/main/java/org/onap/aai/util/GenerateXsd.java b/aai-core/src/main/java/org/onap/aai/util/GenerateXsd.java index d2dc8b76..af9171dd 100644 --- a/aai-core/src/main/java/org/onap/aai/util/GenerateXsd.java +++ b/aai-core/src/main/java/org/onap/aai/util/GenerateXsd.java @@ -7,9 +7,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,13 +19,18 @@ * * ECOMP is a trademark and service mark of AT&T Intellectual Property. */ + package org.onap.aai.util; import com.google.common.base.Joiner; import com.jayway.jsonpath.JsonPath; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.similarity.LevenshteinDistance; + import org.onap.aai.introspection.Version; import org.onap.aai.serialization.db.EdgeProperty; import org.w3c.dom.*; +import org.xml.sax.InputSource; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; @@ -38,10 +43,13 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GenerateXsd { + private static final Logger logger = LoggerFactory.getLogger("GenerateXsd.class"); + static String apiVersion = null; static String apiVersionFmt = null; static boolean useAnnotationsInXsd = false; @@ -51,9 +59,24 @@ public class GenerateXsd { static Map<String, String> generatedJavaType; static Map<String, String> appliedPaths; + static Map<String, String> deletePaths; + static Map<String, String> putRelationPaths; static NodeList javaTypeNodes; + static Map<String,String> javaTypeDefinitions = createJavaTypeDefinitions(); + private static Map<String, String> createJavaTypeDefinitions() + { + StringBuffer aaiInternal = new StringBuffer(); + Map<String,String> javaTypeDefinitions = new HashMap<String, String>(); + aaiInternal.append(" aai-internal:\n"); + aaiInternal.append(" properties:\n"); + aaiInternal.append(" property-name:\n"); + aaiInternal.append(" type: string\n"); + aaiInternal.append(" property-value:\n"); + aaiInternal.append(" type: string\n"); + javaTypeDefinitions.put("aai-internal", aaiInternal.toString()); + return javaTypeDefinitions; + } - public static final int VALUE_NONE = 0; public static final int VALUE_DESCRIPTION = 1; public static final int VALUE_INDEXED_PROPS = 2; @@ -64,12 +87,17 @@ public class GenerateXsd { private static final String root = "../aai-schema/src/main/resources"; private static final String xsd_dir = root + "/aai_schema"; private static final String yaml_dir = root + "/aai_swagger_yaml"; + + /* These three strings are for yaml auto-generation from aai-common class*/ + private static final String normalStartDir = "aai-core"; + private static final String autoGenRoot = "aai-schema/src/main/resources"; + private static final String alt_yaml_dir = autoGenRoot + "/aai_swagger_yaml"; private static int annotationsStartVersion = 9; // minimum version to support annotations in xsd private static int swaggerSupportStartsVersion = 7; // minimum version to support swagger documentation private static XPath xpath = XPathFactory.newInstance().newXPath(); - + private enum LineageType { PARENT, CHILD, UNRELATED; @@ -77,10 +105,91 @@ public class GenerateXsd { private class EdgeDescription { private String ruleKey; + private String to; + private String from; private LineageType type = LineageType.UNRELATED; private String direction; private String multiplicity; + private String preventDelete; + private String deleteOtherV; private boolean hasDelTarget = false; + private String label; + private String description; + /** + * @return the deleteOtherV + */ + public String getDeleteOtherV() { + return deleteOtherV; + } + /** + * @param deleteOtherV the deleteOtherV to set + */ + public void setDeleteOtherV(String deleteOtherV) { + logger.debug("Edge: "+this.getRuleKey()); + logger.debug("Truth: "+(("${direction}".equals(deleteOtherV)) ? "true" : "false")); + logger.debug("Truth: "+(("!${direction}".equals(deleteOtherV)) ? "true" : "false")); + + if("${direction}".equals(deleteOtherV) ) { + this.deleteOtherV = this.direction; + } else if("!${direction}".equals(deleteOtherV) ) { + this.deleteOtherV = this.direction.equals("IN") ? "OUT" : ((this.direction.equals("OUT")) ? "IN" : deleteOtherV); + } else { + this.deleteOtherV = deleteOtherV; + } + logger.debug("DeleteOtherV="+deleteOtherV+"/"+this.direction+"="+this.deleteOtherV); + } + /** + * @return the preventDelete + */ + public String getPreventDelete() { + return preventDelete; + } + /** + * @param preventDelete the preventDelete to set + */ + public void setPreventDelete(String preventDelete) { + if(this.getTo().equals("flavor") || this.getFrom().equals("flavor") ){ + logger.debug("Edge: "+this.getRuleKey()); + logger.debug("Truth: "+(("${direction}".equals(preventDelete)) ? "true" : "false")); + logger.debug("Truth: "+(("!${direction}".equals(preventDelete)) ? "true" : "false")); + } + + if("${direction}".equals(preventDelete) ) { + this.preventDelete = this.direction; + } else if("!${direction}".equals(preventDelete) ) { + this.preventDelete = this.direction.equals("IN") ? "OUT" : ((this.direction.equals("OUT")) ? "IN" : preventDelete); + } else { + this.preventDelete = preventDelete; + } + if(this.getTo().equals("flavor") || this.getFrom().equals("flavor")) { + logger.debug("PreventDelete="+preventDelete+"/"+this.direction+"="+this.preventDelete); + } + } + /** + * @return the to + */ + public String getTo() { + return to; + } + /** + * @param to the to to set + */ + public void setTo(String to) { + this.to = to; + } + /** + * @return the from + */ + public String getFrom() { + return from; + } + /** + * @param from the from to set + */ + public void setFrom(String from) { + this.from = from; + } + public String getRuleKey() { return ruleKey; @@ -91,6 +200,9 @@ public class GenerateXsd { public String getDirection() { return direction; } + public String getDescription() { + return this.description; + } public void setRuleKey(String val) { this.ruleKey=val; } @@ -106,6 +218,9 @@ public class GenerateXsd { public void setHasDelTarget(String val) { hasDelTarget = Boolean.parseBoolean(val); } + public void setDescription(String val) { + this.description = val; + } public String getRelationshipDescription(String fromTo, String otherNodeName) { @@ -114,95 +229,487 @@ public class GenerateXsd { if ("FROM".equals(fromTo)) { if ("OUT".equals(direction)) { if (LineageType.PARENT == type) { - result = " (is composed of "+otherNodeName; + result = " (PARENT of "+otherNodeName; + result = String.join(" ", result+",", this.from, this.getLabel(), this.to); } } else { if (LineageType.CHILD == type) { - result = " (comprises "+otherNodeName; + result = " (CHILD of "+otherNodeName; + result = String.join(" ", result+",", this.from, this.getLabel(), this.to); } else if (LineageType.PARENT == type) { - result = " (comprises "+otherNodeName; + result = " (PARENT of "+otherNodeName; + result = String.join(" ", result+",", this.from, this.getLabel(), this.to); } } + if (result.length() == 0) result = String.join(" ", "(", this.from, this.getLabel(), this.to+",", this.getMultiplicity()); } else { + //if ("TO".equals(fromTo) if ("OUT".equals(direction)) { if (LineageType.PARENT == type) { - result = " (comprises "+otherNodeName; + result = " (CHILD of "+otherNodeName; + result = String.join(" ", result+",", this.from, this.getLabel(), this.to+",", this.getMultiplicity()); } } else { if (LineageType.PARENT == type) { - result = " (is composed of "+otherNodeName; + result = " (PARENT of "+otherNodeName; + result = String.join(" ", result+",", this.from, this.getLabel(), this.to+",", this.getMultiplicity()); } } - } + if (result.length() == 0) result = String.join(" ", "(", this.from, this.getLabel(), this.to+",", this.getMultiplicity()); -// if (type != null) { -// if (LineageType.PARENT.equals(type) && "FROM".equals(fromTo)) { -// if ("OUT".equals(direction)) { -// result = " (is composed of "+otherNodeName; -// } else { -// result = " (comprises "+otherNodeName; -// } -// } else { -// result = " (comprises " + otherNodeName; -// // if (!(multiplicity.startsWith("One"))) { -// // System.err.println("Surprised to find multiplicity "+multiplicity+" with comprises for "+ruleKey); -// // } -// } -// } - if ("TO".equals(fromTo)) { - if (result.length() == 0) result = result + " ("; - else result = result + ", "; - - result = result + mapMultiplicity(fromTo); - if (hasDelTarget) result = result + ", will delete target node"; } + if (hasDelTarget) result = result + ", will delete target node"; + if (result.length() > 0) result = result + ")"; + if (description != null && description.length() > 0) result = result + "\n "+ description; // 6 spaces is important for yaml + return result; } - private String mapMultiplicity(String fromTo) { - String result = multiplicity; -// Below logic works if an IN switches multiplicity, which it doesn't today. -// if ("TO".equals(fromTo)) { -// if (direction.equals("OUT")) { -// result = multiplicity; -// } else { -// result = switchMultiplicity(multiplicity); -// } -// } else { -// if (direction.equals("OUT")) { -// result = multiplicity; -// } else { -// result = switchMultiplicity(multiplicity); -// } -// } - return result; + /** + * @return the hasDelTarget + */ + @SuppressWarnings("unused") + public boolean isHasDelTarget() { + return hasDelTarget; + } + /** + * @param hasDelTarget the hasDelTarget to set + */ + @SuppressWarnings("unused") + public void setHasDelTarget(boolean hasDelTarget) { + this.hasDelTarget = hasDelTarget; + } + /** + * @return the type + */ + @SuppressWarnings("unused") + public LineageType getType() { + return type; } + /** + * @return the label + */ + public String getLabel() { + return label; + } + public void setLabel(String string) { + this.label=string; + } + } + + private static class PutRelationPathSet { + String apiPath; + String opId; + ArrayList<String> relations = new ArrayList<String>(); + String objectName = ""; + String currentAPIVersion = ""; + public PutRelationPathSet(String opId, String path) { + super(); + this.apiPath = path.replace("/relationship-list/relationship", ""); + this.opId = opId; + objectName = GenerateXsd.deletePaths.get(apiPath); + currentAPIVersion = GenerateXsd.apiVersion; + } + public void process() { + this.toRelations(); + this.fromRelations(); + this.writeRelationsFile(); -// private String switchMultiplicity(String val) throws IllegalArgumentException -// { -// String result = null; -// switch (val) { -// case "Many2Many": -// case "One2One": -// result = val; -// break; -// case "Many2One": -// result = "One2Many"; -// break; -// case "One2Many": -// result = "Many2One"; -// break; -// default: -// throw new IllegalArgumentException("Multiplicity cannot be "+val); -// } -// System.out.println("Switched Multiplicity from "+val+" to "+result); -// return result; -// } + } + public void toRelations() { + logger.debug("{“comment”: “Valid TO Relations that can be added”},"); + logger.debug("apiPath: "+apiPath+"\nopId="+opId+"\nobjectName="+objectName); + Collection<EdgeDescription> toEdges = GenerateXsd.getEdgeRulesTO(objectName); + + if(toEdges.size() > 0) { + relations.add("{\"comment\": \"Valid TO Relations that can be added\"}\n"); + } + for (EdgeDescription ed : toEdges) { + logger.debug(ed.getRuleKey()+"Type="+ed.type); + String obj = ed.getRuleKey().replace(objectName,"").replace("|",""); + String selectedRelation = ""; + if(ed.type == LineageType.UNRELATED) { + String selectObj = getUnrelatedObjectPaths(obj, apiPath); + logger.debug("SelectedObj:"+selectObj); + selectedRelation = formatObjectRelationSet(obj,selectObj); + logger.trace("ObjectRelationSet"+selectedRelation); + } else { + String selectObj = getKinObjectPath(obj, apiPath); + logger.debug("SelectedObj:"+selectObj); + selectedRelation = formatObjectRelation(obj,selectObj); + logger.trace("ObjectRelationSet"+selectedRelation); + } + relations.add(selectedRelation); + logger.trace(selectedRelation); + } + } + + public void fromRelations() { + logger.debug("“comment”: “Valid FROM Relations that can be added”"); + Collection<EdgeDescription> fromEdges = getEdgeRulesFROM(objectName); + if(fromEdges.size() > 0) { + relations.add("{\"comment\": \"Valid FROM Relations that can be added\"}\n"); + } + for (EdgeDescription ed : fromEdges) { + logger.debug(ed.getRuleKey()+"Type="+ed.type); + String obj = ed.getRuleKey().replace(objectName,"").replace("|",""); + String selectedRelation = ""; + if(ed.type == LineageType.UNRELATED) { + String selectObj = getUnrelatedObjectPaths(obj, apiPath); + logger.debug("SelectedObj"+selectObj); + selectedRelation = formatObjectRelationSet(obj,selectObj); + logger.trace("ObjectRelationSet"+selectedRelation); + } else { + String selectObj = getKinObjectPath(obj, apiPath); + logger.debug("SelectedObj"+selectObj); + selectedRelation = formatObjectRelation(obj,selectObj); + logger.trace("ObjectRelationSet"+selectedRelation); + } + relations.add(selectedRelation); + logger.trace(selectedRelation); + } + } + public void writeRelationsFile() { + File examplefilePath = new File(yaml_dir + "/relations/" + currentAPIVersion+"/"+opId.replace("RelationshipListRelationship", "") + ".json"); + + logger.debug(String.join("exampleFilePath: ", examplefilePath.toString())); + FileOutputStream fop = null; + try { + if (!examplefilePath.exists()) { + examplefilePath.getParentFile().mkdirs(); + examplefilePath.createNewFile(); + } + fop = new FileOutputStream(examplefilePath); + } catch(Exception e) { + e.printStackTrace(); + return; + } + try { + if(relations.size() > 0) {fop.write("[\n".getBytes());} + fop.write(String.join(",\n", relations).getBytes()); + if(relations.size() > 0) {fop.write("\n]\n".getBytes());} + fop.flush(); + fop.close(); + } catch (Exception e) { + e.printStackTrace(); + return; + } + logger.debug(String.join(",\n", relations)); + return; + } + + private static String formatObjectRelationSet(String obj, String selectObj) { + StringBuffer pathSb = new StringBuffer(); + String[] paths = selectObj.split("[|]"); + for (String s: paths) { + logger.trace("SelectOBJ"+s); + pathSb.append(formatObjectRelation(obj, s)+",\n"); + } + pathSb.deleteCharAt(pathSb.length()-2); + return pathSb.toString(); + } + + private static String formatObjectRelation(String obj, String selectObj) { + StringBuffer pathSb = new StringBuffer(); + pathSb.append("{\n"); + pathSb.append("\"related-to\" : \""+obj+"\",\n"); + pathSb.append("\"related-link\" : \""+selectObj+"\"\n"); + pathSb.append("}"); + return pathSb.toString(); + } + + private static String getKinObjectPath(String obj, String apiPath) { + LevenshteinDistance proximity = new LevenshteinDistance(); + String targetPath = ""; + int targetScore = Integer.MAX_VALUE; + int targetMaxScore = 0; + for (Map.Entry<String, String> p : deletePaths.entrySet()) { + if(p.getValue().equals(obj)) { + targetScore = (targetScore >= proximity.apply(apiPath, p.getKey())) ? proximity.apply(apiPath, p.getKey()) : targetScore; + targetPath = (targetScore >= proximity.apply(apiPath, p.getKey())) ? p.getKey() : targetPath; + targetMaxScore = (targetMaxScore <= proximity.apply(apiPath, p.getKey())) ? proximity.apply(apiPath, p.getKey()) : targetScore; + logger.trace(proximity.apply(apiPath, p.getKey())+":"+p.getKey()); + logger.trace(proximity.apply(apiPath, p.getKey())+":"+apiPath); + } + } + return targetPath; + } + + private static String getUnrelatedObjectPaths(String obj, String apiPath) { + String targetPath = ""; + logger.trace("Obj:"+obj +"\n" + apiPath); + for (Map.Entry<String, String> p : deletePaths.entrySet()) { + if(p.getValue().equals(obj)) { + logger.trace("p.getvalue:"+p.getValue()+"p.getkey:"+p.getKey()); + targetPath += ((targetPath.length() == 0 ? "" : "|") + p.getKey()); + logger.trace("Match:"+apiPath +"\n" + targetPath); + } + } + return targetPath; + } + } + + private static class PatchOperation { + String useOpId; + String xmlRootElementName; + String tag; + String path; + String pathParams; + + public PatchOperation(String useOpId, String xmlRootElementName, String tag, String path, String pathParams) { + super(); + this.useOpId = useOpId; + this.xmlRootElementName = xmlRootElementName; + this.tag = tag; + this.path = path; + this.pathParams = pathParams; + } + + @Override + public String toString() { + StringBuffer pathSb = new StringBuffer(); + pathSb.append(" patch:\n"); + pathSb.append(" tags:\n"); + pathSb.append(" - " + tag + "\n"); + + pathSb.append(" summary: update an existing " + xmlRootElementName + "\n"); + pathSb.append(" description: update an existing " + xmlRootElementName + "\n"); + pathSb.append(" operationId: Update" + useOpId + "\n"); + pathSb.append(" consumes:\n"); + pathSb.append(" - application/json\n"); + pathSb.append(" - application/xml\n"); + pathSb.append(" produces:\n"); + pathSb.append(" - application/json\n"); + pathSb.append(" - application/xml\n"); + pathSb.append(" responses:\n"); + pathSb.append(" \"default\":\n"); + pathSb.append(" " + responsesUrl); + + pathSb.append(" parameters:\n"); + pathSb.append(pathParams); // for nesting + pathSb.append(" - name: body\n"); + pathSb.append(" in: body\n"); + pathSb.append(" description: " + xmlRootElementName + " object that needs to be created or updated\n"); + pathSb.append(" required: true\n"); + pathSb.append(" schema:\n"); + pathSb.append(" $ref: \"patchSchema.yaml#/definitions/" + xmlRootElementName + "\"\n"); + + return pathSb.toString(); + } + public String toString1() { + StringBuffer pathSb = new StringBuffer(); + StringBuffer relationshipExamplesSb = new StringBuffer(); + if ( path.endsWith("/relationship") ) { + pathSb.append(" " + path + ":\n" ); + } + pathSb.append(" patch:\n"); + pathSb.append(" tags:\n"); + pathSb.append(" - " + tag + "\n"); + + if ( path.endsWith("/relationship") ) { + pathSb.append(" summary: see node definition for valid relationships\n"); + relationshipExamplesSb.append("[See Examples](apidocs/relations/"+GenerateXsd.apiVersion+"/"+useOpId+".json)"); + } else { + pathSb.append(" summary: update an existing " + xmlRootElementName + "\n"); + pathSb.append(" description: |\n"); + pathSb.append(" Update an existing " + xmlRootElementName + "\n"); + pathSb.append(" #\n"); + pathSb.append(" Note: Endpoints that are not devoted to object relationships support both PUT and PATCH operations.\n"); + pathSb.append(" The PUT operation will entirely replace an existing object.\n"); + pathSb.append(" The PATCH operation sends a \"description of changes\" for an existing object. The entire set of changes must be applied. An error result means no change occurs.\n"); + pathSb.append(" #\n"); + pathSb.append(" Other differences between PUT and PATCH are:\n"); + pathSb.append(" #\n"); + pathSb.append(" - For PATCH, you can send any of the values shown in sample REQUEST body. There are no required values.\n"); + pathSb.append(" - For PATCH, resource-id which is a required REQUEST body element for PUT, must not be sent.\n"); + pathSb.append(" - PATCH cannot be used to update relationship elements; there are dedicated PUT operations for this.\n"); + } + pathSb.append(" operationId: Update" + useOpId + "\n"); + pathSb.append(" consumes:\n"); + pathSb.append(" - application/json\n"); + pathSb.append(" - application/xml\n"); + pathSb.append(" produces:\n"); + pathSb.append(" - application/json\n"); + pathSb.append(" - application/xml\n"); + pathSb.append(" responses:\n"); + pathSb.append(" \"default\":\n"); + pathSb.append(" " + responsesUrl); + pathSb.append(" parameters:\n"); + pathSb.append(pathParams); // for nesting + pathSb.append(" - name: body\n"); + pathSb.append(" in: body\n"); + pathSb.append(" description: " + xmlRootElementName + " object that needs to be created or updated. "+relationshipExamplesSb.toString()+"\n"); + pathSb.append(" required: true\n"); + pathSb.append(" schema:\n"); + pathSb.append(" $ref: \"#/patchDefinitions/" + xmlRootElementName + "\"\n"); + return pathSb.toString(); + } + } + private static class PutOperation { + String useOpId; + String xmlRootElementName; + String tag; + String path; + String pathParams; + + public PutOperation(String useOpId, String xmlRootElementName, String tag, String path, String pathParams) { + super(); + this.useOpId = useOpId; + this.xmlRootElementName = xmlRootElementName; + this.tag = tag; + this.path = path; + this.pathParams = pathParams; + } + + @Override + public String toString() { + StringBuffer pathSb = new StringBuffer(); + StringBuffer relationshipExamplesSb = new StringBuffer(); + if ( path.endsWith("/relationship") ) { + pathSb.append(" " + path + ":\n" ); + } + pathSb.append(" put:\n"); + pathSb.append(" tags:\n"); + pathSb.append(" - " + tag + "\n"); + + if ( path.endsWith("/relationship") ) { + pathSb.append(" summary: see node definition for valid relationships\n"); + } else { + pathSb.append(" summary: create or update an existing " + xmlRootElementName + "\n"); + pathSb.append(" description: |\n Create or update an existing " + xmlRootElementName + ".\n #\n Note! This PUT method has a corresponding PATCH method that can be used to update just a few of the fields of an existing object, rather than a full object replacement. An example can be found in the [PATCH section] below\n"); + } + relationshipExamplesSb.append("[Valid relationship examples shown here](apidocs/relations/"+GenerateXsd.apiVersion+"/"+useOpId.replace("RelationshipListRelationship", "")+".json)"); + pathSb.append(" operationId: createOrUpdate" + useOpId + "\n"); + pathSb.append(" consumes:\n"); + pathSb.append(" - application/json\n"); + pathSb.append(" - application/xml\n"); + pathSb.append(" produces:\n"); + pathSb.append(" - application/json\n"); + pathSb.append(" - application/xml\n"); + pathSb.append(" responses:\n"); + pathSb.append(" \"default\":\n"); + pathSb.append(" " + responsesUrl); + + pathSb.append(" parameters:\n"); + pathSb.append(pathParams); // for nesting + pathSb.append(" - name: body\n"); + pathSb.append(" in: body\n"); + pathSb.append(" description: " + xmlRootElementName + " object that needs to be created or updated. "+relationshipExamplesSb.toString()+"\n"); + pathSb.append(" required: true\n"); + pathSb.append(" schema:\n"); + pathSb.append(" $ref: \"#/definitions/" + xmlRootElementName + "\"\n"); + return pathSb.toString(); + } + public String tagRelationshipPathMapEntry() { + if ( path.endsWith("/relationship") ) { + putRelationPaths.put(useOpId, path); + } + return ""; + } + + } + + private static class GetOperation { + String useOpId; + String xmlRootElementName; + String tag; + String path; + @SuppressWarnings("unused") + String pathParams; + public GetOperation(String useOpId, String xmlRootElementName, String tag, String path, String pathParams) { + super(); + this.useOpId = useOpId; + this.xmlRootElementName = xmlRootElementName; + this.tag = tag; + this.path = path; + this.pathParams = pathParams; + } + @Override + public String toString() { + StringBuffer pathSb = new StringBuffer(); + pathSb.append(" " + path + ":\n" ); + pathSb.append(" get:\n"); + pathSb.append(" tags:\n"); + pathSb.append(" - " + tag + "\n"); + pathSb.append(" summary: returns " + xmlRootElementName + "\n"); + + pathSb.append(" description: returns " + xmlRootElementName + "\n"); + pathSb.append(" operationId: get" + useOpId + "\n"); + pathSb.append(" produces:\n"); + pathSb.append(" - application/json\n"); + pathSb.append(" - application/xml\n"); + + pathSb.append(" responses:\n"); + pathSb.append(" \"200\":\n"); + pathSb.append(" description: successful operation\n"); + pathSb.append(" schema:\n"); + pathSb.append(" $ref: \"#/getDefinitions/" + xmlRootElementName + "\"\n"); + pathSb.append(" \"default\":\n"); + pathSb.append(" " + responsesUrl); + + return pathSb.toString(); + } + + } + private static class DeleteOperation { + String useOpId; + String xmlRootElementName; + String tag; + String path; + String pathParams; + public DeleteOperation(String useOpId, String xmlRootElementName, String tag, String path, String pathParams) { + super(); + this.useOpId = useOpId; + this.xmlRootElementName = xmlRootElementName; + this.tag = tag; + this.path = path; + this.pathParams = pathParams; + } + @Override + public String toString() { + StringBuffer pathSb = new StringBuffer(); + pathSb.append(" delete:\n"); + pathSb.append(" tags:\n"); + pathSb.append(" - " + tag + "\n"); + pathSb.append(" summary: delete an existing " + xmlRootElementName + "\n"); + + pathSb.append(" description: delete an existing " + xmlRootElementName + "\n"); + + pathSb.append(" operationId: delete" + useOpId + "\n"); + pathSb.append(" consumes:\n"); + pathSb.append(" - application/json\n"); + pathSb.append(" - application/xml\n"); + pathSb.append(" produces:\n"); + pathSb.append(" - application/json\n"); + pathSb.append(" - application/xml\n"); + pathSb.append(" responses:\n"); + pathSb.append(" \"default\":\n"); + pathSb.append(" " + responsesUrl); + pathSb.append(" parameters:\n"); + + pathSb.append(pathParams); // for nesting + if ( !path.endsWith("/relationship") ) { + pathSb.append(" - name: resource-version\n"); + + pathSb.append(" in: query\n"); + pathSb.append(" description: resource-version for concurrency\n"); + pathSb.append(" required: true\n"); + pathSb.append(" type: string\n"); + } + return pathSb.toString(); + } + public String objectPathMapEntry() { + if (! path.endsWith("/relationship") ) { + deletePaths.put(path, xmlRootElementName); + } + return (xmlRootElementName+":"+path); + } + } private static boolean validVersion(String versionToGen) { @@ -247,57 +754,75 @@ public class GenerateXsd { } + String responsesLabel = System.getProperty("yamlresponses_url"); + responsesUrl = responsesLabel; + + List<Version> versionsToGen = new ArrayList<>(); if ( versionToGen == null ) { System.err.println("Version is required, ie v<n> or ALL."); System.exit(1); } - - responsesUrl = System.getProperty("yamlresponses_url"); - String responsesLabel = System.getProperty("yamlresponses_label"); - List<Version> versionsToGen = new ArrayList<>(); - - - if (!"ALL".equalsIgnoreCase(versionToGen) && !versionToGen.matches("v\\d+") && !validVersion(versionToGen)) { + else if (!"ALL".equalsIgnoreCase(versionToGen) && !versionToGen.matches("v\\d+") && !validVersion(versionToGen)) { System.err.println("Invalid version passed. " + versionToGen); System.exit(1); } - - if ("ALL".equalsIgnoreCase(versionToGen)) { + else if ("ALL".equalsIgnoreCase(versionToGen)) { versionsToGen = Arrays.asList(Version.values()); Collections.sort(versionsToGen); Collections.reverse(versionsToGen); } else { versionsToGen.add(Version.valueOf(versionToGen)); } - - if ( fileTypeToGen.equals(generateTypeYAML) ) { + + //process file type System property + fileTypeToGen = (fileTypeToGen == null ? generateTypeXSD : fileTypeToGen.toLowerCase()); + if ( !fileTypeToGen.equals( generateTypeXSD ) && !fileTypeToGen.equals( generateTypeYAML )) { + System.err.println("Invalid gen_type passed. " + fileTypeToGen); + System.exit(1); + } else if ( fileTypeToGen.equals(generateTypeYAML) ) { if ( responsesUrl == null || responsesUrl.length() < 1 || responsesLabel == null || responsesLabel.length() < 1 ) { System.err.println("generating swagger yaml file requires yamlresponses_url and yamlresponses_label properties" ); System.exit(1); + } else { + responsesUrl = "description: "+ "Response codes found in [response codes]("+responsesLabel+ ").\n"; } - responsesUrl = "description: "+ responsesLabel+ "(" + responsesUrl + ").\n"; } - String oxmPath = root + "/oxm/"; - + String oxmPath; + if(System.getProperty("user.dir") != null && !System.getProperty("user.dir").contains(normalStartDir)) { + oxmPath = autoGenRoot + "/oxm/"; + } + else { + oxmPath = root + "/oxm/"; + } + String outfileName; File outfile; String fileContent; for (Version v : versionsToGen) { apiVersion = v.toString(); - System.out.println("Generating " + apiVersion + " " + fileTypeToGen); + logger.info("Generating " + apiVersion + " " + fileTypeToGen); File oxm_file = new File(oxmPath + "aai_oxm_" + apiVersion + ".xml"); apiVersionFmt = "." + apiVersion + "."; generatedJavaType = new HashMap<String, String>(); appliedPaths = new HashMap<String, String>(); + putRelationPaths = new HashMap<String, String>(); + deletePaths = new HashMap<String, String>(); + if ( fileTypeToGen.equals(generateTypeXSD) ) { useAnnotationsInXsd = versionUsesAnnotations(apiVersion); outfileName = xsd_dir + "/aai_schema_" + apiVersion + "." + generateTypeXSD; - fileContent = processOxmFile(oxm_file, v); + fileContent = processOxmFile(oxm_file, v, null); } else if ( versionSupportsSwagger(apiVersion )) { - outfileName = yaml_dir + "/aai_swagger_" + apiVersion + "." + generateTypeYAML; - fileContent = generateSwaggerFromOxmFile( oxm_file); + if(System.getProperty("user.dir") != null && !System.getProperty("user.dir").contains(normalStartDir)) { + outfileName = alt_yaml_dir; + } + else { + outfileName = yaml_dir; + } + outfileName = outfileName + "/aai_swagger_" + apiVersion + "." + generateTypeYAML; + fileContent = generateSwaggerFromOxmFile( oxm_file, null); } else { continue; } @@ -309,7 +834,7 @@ public class GenerateXsd { try { outfile.createNewFile(); } catch (IOException e) { - System.out.println( "Exception creating output file " + outfileName); + logger.error( "Exception creating output file " + outfileName); e.printStackTrace(); } BufferedWriter bw = null; @@ -319,14 +844,14 @@ public class GenerateXsd { bw = Files.newBufferedWriter(path, charset); bw.write(fileContent); } catch ( IOException e) { - System.out.println( "Exception writing output file " + outfileName); + logger.error( "Exception writing output file " + outfileName); e.printStackTrace(); } finally { if ( bw != null ) { bw.close(); } } - System.out.println( "GeneratedXSD successful, saved in " + outfileName); + logger.info( "GeneratedXSD successful, saved in " + outfileName); } } @@ -339,7 +864,7 @@ public class GenerateXsd { NodeList parentNodes = javaTypeElement.getElementsByTagName("java-attributes"); StringBuffer sb = new StringBuffer(); if ( parentNodes.getLength() == 0 ) { - //System.out.println( "no java-attributes for java-type " + javaTypeName); + logger.trace( "no java-attributes for java-type " + javaTypeName); return ""; } @@ -354,17 +879,10 @@ public class GenerateXsd { String attrName = attr.getNodeName(); String attrValue = attr.getNodeValue(); - //System.out.println("Found xml-root-element attribute: " + attrName + " with value: " + attrValue); + logger.trace("Found xml-root-element attribute: " + attrName + " with value: " + attrValue); if ( attrName.equals("name")) xmlRootElementName = attrValue; } - /* - if ( javaTypeName.equals("RelationshipList")) { - System.out.println( "Skipping " + javaTypeName); - generatedJavaType.put(javaTypeName, null); - return ""; - } - */ Element parentElement = (Element)parentNodes.item(0); NodeList xmlElementNodes = parentElement.getElementsByTagName("xml-element"); @@ -381,14 +899,14 @@ public class GenerateXsd { sb1.append(" <xs:complexType>\n"); NodeList properties = GenerateXsd.locateXmlProperties(javaTypeElement); if (properties != null && useAnnotationsInXsd) { - //System.out.println("properties found for: " + xmlRootElementName); + logger.trace("properties found for: " + xmlRootElementName); sb1.append(" <xs:annotation>\r\n"); insertAnnotation(properties, false, "class", sb1, " "); sb1.append(" </xs:annotation>\r\n"); - } /*else { - System.out.println("no properties found for: " + xmlRootElementName); - }*/ + } else { + logger.trace("no properties found for: " + xmlRootElementName); + } sb1.append(" <xs:sequence>\n"); for ( int i = 0; i < xmlElementNodes.getLength(); ++i ) { @@ -406,7 +924,7 @@ public class GenerateXsd { String attrValue = attr.getNodeValue(); if ( attrName.equals("name")) { xmlElementWrapper = attrValue; - //System.out.println("found xml-element-wrapper " + xmlElementWrapper); + logger.trace("found xml-element-wrapper " + xmlElementWrapper); } } @@ -421,7 +939,7 @@ public class GenerateXsd { String attrName = attr.getNodeName(); String attrValue = attr.getNodeValue(); - //System.out.println("For " + xmlRootElementName + " Found xml-element attribute: " + attrName + " with value: " + attrValue); + logger.trace("For " + xmlRootElementName + " Found xml-element attribute: " + attrName + " with value: " + attrValue); if ( attrName.equals("name")) { elementName = attrValue; } @@ -460,33 +978,29 @@ public class GenerateXsd { sb1.append(" <xs:annotation>\r\n"); insertAnnotation(properties, false, "class", sb1, " "); sb1.append(" </xs:annotation>\r\n"); - } /*else { - System.out.println("no properties found for: " + xmlElementWrapper); - }*/ + } else { + logger.trace("no properties found for: " + xmlElementWrapper); + } sb1.append(" <xs:sequence>\n"); sb1.append(" "); } if ("Nodes".equals(addType)) { - //System.out.println ("Skipping nodes, temporary testing"); + logger.trace("Skipping nodes, temporary testing"); continue; } if ( addType != null ) { - //sb1.append(" <xs:element ref=\"tns:" + elementName +"\""); sb1.append(" <xs:element ref=\"tns:" + getXmlRootElementName(addType) +"\""); } else { sb1.append(" <xs:element name=\"" + elementName +"\""); } if ( elementType.equals("java.lang.String")) sb1.append(" type=\"xs:string\""); - //if ( elementType.equals("java.lang.String")) - //sb1.append(" type=\"xs:string\""); if ( elementType.equals("java.lang.Long")) sb1.append(" type=\"xs:unsignedInt\""); if ( elementType.equals("java.lang.Integer")) sb1.append(" type=\"xs:int\""); if ( elementType.equals("java.lang.Boolean")) sb1.append(" type=\"xs:boolean\""); - //if ( elementIsRequired != null && elementIsRequired.equals("true")||addType != null) { if ( elementIsRequired == null || !elementIsRequired.equals("true")||addType != null) { sb1.append(" minOccurs=\"0\""); } @@ -513,32 +1027,10 @@ public class GenerateXsd { sb1.append(" </xs:element>\n"); } } - /* - if ( xmlRootElementName.equals("notify") || - xmlRootElementName.equals("relationship") || - xmlRootElementName.equals("relationship-data") || - xmlRootElementName.equals("related-to-property") ) - - sb1.append(" <xs:any namespace=\"##other\" processContents=\"lax\" minOccurs=\"0\" maxOccurs=\"unbounded\"/>\n"); - */ sb1.append(" </xs:sequence>\n"); sb1.append(" </xs:complexType>\n"); sb1.append(" </xs:element>\n"); } - /* - NodeList valNodes = javaTypeElement.getElementsByTagName("xml-root-element"); - Element valElement = (Element) valNodes.item(0); - attributes = valElement.getAttributes(); - for ( int i = 0; i < attributes.getLength(); ++i ) { - Attr attr = (Attr) attributes.item(i); - String attrName = attr.getNodeName(); - - String attrValue = attr.getNodeValue(); - System.out.println("Found xml-root-element attribute: " + attrName + " with value: " + attrValue); - if ( attrValue.equals("name")) - xmlRootElementName = attrValue; - } - */ if ( xmlElementNodes.getLength() < 1 ) { sb.append(" <xs:element name=\"" + xmlRootElementName + "\">\n"); @@ -549,9 +1041,7 @@ public class GenerateXsd { generatedJavaType.put(javaTypeName, null); return sb.toString(); } - sb.append( sb1 ); - return sb.toString(); } @@ -576,8 +1066,6 @@ public class GenerateXsd { name = "extendsFrom"; } metadata.add(name + "=\"" + value.replaceAll("&", "&") + "\""); - //System.out.println("property name: " + name); - } } sb1.append( @@ -605,7 +1093,7 @@ public class GenerateXsd { return javaTypeElement; } } - System.out.println( "oxm file format error, missing java-type " + javaTypeName); + logger.error( "oxm file format error, missing java-type " + javaTypeName); return (Element) null; } @@ -626,7 +1114,7 @@ public class GenerateXsd { return javaTypeElement; } } - System.out.println( "oxm file format error, missing java-type " + javaTypeName); + logger.error( "oxm file format error, missing java-type " + javaTypeName); return (Element) null; } private static String getXmlRootElementName( String javaTypeName ) @@ -651,21 +1139,28 @@ public class GenerateXsd { attrName = attr.getNodeName(); attrValue = attr.getNodeValue(); - //System.out.println("Found xml-root-element attribute: " + attrName + " with value: " + attrValue); if ( attrName.equals("name")) return (attrValue); } } } } - System.out.println( "oxm file format error, missing java-type " + javaTypeName); + logger.error( "oxm file format error, missing java-type " + javaTypeName); return null; } - public static String processOxmFile( File oxmFile, Version v ) + public static String processOxmFile( File oxmFile, Version v, String xml ) { + if ( xml != null ){ + apiVersion = v.toString(); + useAnnotationsInXsd = true; + apiVersionFmt = "." + apiVersion + "."; + generatedJavaType = new HashMap<>(); + appliedPaths = new HashMap<>(); + } StringBuilder sb = new StringBuilder(); + logger.trace("processing starts"); sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n"); String namespace = "org.onap"; if (v.compareTo(Version.v11) < 0 || v.compareTo(Version.v12) < 0) { @@ -680,7 +1175,8 @@ public class GenerateXsd { " xmlns:annox=\"http://annox.dev.java.net\" \r\n" + " jaxb:extensionBindingPrefixes=\"annox\">\n\n"); } else { - sb.append("<xs:schema elementFormDefault=\"qualified\" version=\"1.0\" targetNamespace=\"http://" + namespace + ".aai.inventory/" + + sb.append("<xs:schema elementFormDefault=\"qualified\" version=\"1.0\" targetNamespace=\"http://" + namespace + ".aai.inventory/" + apiVersion + "\" xmlns:tns=\"http://" + namespace + ".aai.inventory/" + apiVersion + "\" xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\n\n"); } @@ -689,8 +1185,14 @@ public class GenerateXsd { DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); dbFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - Document doc = dBuilder.parse(oxmFile); - + Document doc; + + if ( xml == null ){ + doc = dBuilder.parse(oxmFile); + } else { + InputSource is = new InputSource(new StringReader(xml)); + doc = dBuilder.parse(is); + } NodeList bindingsNodes = doc.getElementsByTagName("xml-bindings"); Element bindingElement; NodeList javaTypesNodes; @@ -700,20 +1202,20 @@ public class GenerateXsd { if ( bindingsNodes == null || bindingsNodes.getLength() == 0 ) { - System.out.println( "missing <binding-nodes> in " + oxmFile ); + logger.error( "missing <binding-nodes> in " + oxmFile ); return null; } bindingElement = (Element) bindingsNodes.item(0); javaTypesNodes = bindingElement.getElementsByTagName("java-types"); if ( javaTypesNodes.getLength() < 1 ) { - System.out.println( "missing <binding-nodes><java-types> in " + oxmFile ); + logger.error( "missing <binding-nodes><java-types> in " + oxmFile ); return null; } javaTypesElement = (Element) javaTypesNodes.item(0); javaTypeNodes = javaTypesElement.getElementsByTagName("java-type"); if ( javaTypeNodes.getLength() < 1 ) { - System.out.println( "missing <binding-nodes><java-types><java-type> in " + oxmFile ); + logger.error( "missing <binding-nodes><java-types><java-type> in " + oxmFile ); return null; } @@ -732,11 +1234,11 @@ public class GenerateXsd { javaTypeName = attrValue; } if ( javaTypeName == null ) { - System.out.println( "<java-type> has no name attribute in " + oxmFile ); + logger.error( "<java-type> has no name attribute in " + oxmFile ); return null; } if ("Nodes".equals(javaTypeName)) { - //System.out.println("skipping Nodes entry (temporary feature)"); + logger.debug("skipping Nodes entry (temporary feature)"); continue; } if ( !generatedJavaType.containsKey(javaTypeName) ) { @@ -752,6 +1254,66 @@ public class GenerateXsd { sb.append("</xs:schema>\n"); return sb.toString(); } + + public static String toDeleteRules(String objectName) { + Collection<EdgeDescription> toEdges = GenerateXsd.getEdgeRulesTO(objectName); + logger.debug("TO Edges count: "+toEdges.size()+" Object: "+objectName); + String prevent=null; + String also=null; + LinkedHashSet<String> preventDelete = new LinkedHashSet<String>(); + LinkedHashSet<String> alsoDelete = new LinkedHashSet<String>(); + for (EdgeDescription ed : toEdges) { + logger.debug("{“comment”: From = "+ed.getFrom()+" To: "+ed.getTo()+" Object: "+objectName); + logger.debug("{“comment”: Direction = "+ed.getDirection()+" PreventDelete: "+ed.getPreventDelete()+" DeleteOtherV: "+ed.getDeleteOtherV()+" Object: "+objectName); + if(ed.getPreventDelete().equals("IN") && ed.getTo().equals(objectName)) { + preventDelete.add(ed.getFrom().toUpperCase()); + } + if(ed.getDeleteOtherV().equals("IN") && ed.getTo().equals(objectName) ) { + alsoDelete.add(ed.getFrom().toUpperCase()); + } + } + if(preventDelete.size() > 0) { + prevent = " - "+objectName.toUpperCase()+" cannot be deleted if linked to "+String.join(",",preventDelete); + logger.info(prevent); + } + if(alsoDelete.size() > 0) { + also = " - "+objectName.toUpperCase()+" is DELETED when these are DELETED "+String.join(",",alsoDelete); + // This commented out line is better (gets who deletes what correct) but still not accurate. + //also = " - Deletion of an instance of "+objectName.toUpperCase()+" causes instances of these directly related types to be DELETED ["+String.join(",",alsoDelete)+"]"; + logger.info(also); + } + return String.join((prevent == null || also == null) ? "" : "\n", prevent == null ? "" : prevent, also == null ? "" : also)+((prevent == null && also == null) ? "" : "\n"); + } + + public static String fromDeleteRules(String objectName) { + Collection<EdgeDescription> fromEdges = GenerateXsd.getEdgeRulesFROM(objectName); + LinkedHashSet<String> preventDelete = new LinkedHashSet <String>(); + LinkedHashSet<String> alsoDelete = new LinkedHashSet <String>(); + String prevent=null; + String also=null; + for (EdgeDescription ed : fromEdges) { + logger.debug("{“comment”: From = "+ed.getFrom()+" To: "+ed.getTo()+" Object: "+objectName); + logger.debug("{“comment”: Direction = "+ed.getDirection()+" PreventDelete: "+ed.getPreventDelete()+" DeleteOtherV: "+ed.getDeleteOtherV()+" Object: "+objectName); + if(ed.getPreventDelete().equals("OUT") && ed.getFrom().equals(objectName)) { + preventDelete.add(ed.getTo().toUpperCase()); + } + if(ed.getDeleteOtherV().equals("OUT") && ed.getFrom().equals(objectName) ) { + alsoDelete.add(ed.getTo().toUpperCase()); + } + } + if(preventDelete.size() > 0) { + prevent = " - "+objectName.toUpperCase()+" cannot be deleted if linked to "+String.join(",",preventDelete); + logger.info(prevent); + } + if(alsoDelete.size() > 0) { + also = " - "+objectName.toUpperCase()+" deletion means associated objects of these types are also DELETED:"+String.join(",",alsoDelete); + // This commented out line is better (gets who deletes what correct) but still not accurate. + //also = " - Deletion of an instance of "+objectName.toUpperCase()+" causes instances of these directly related types to be DELETED ["+String.join(",",alsoDelete)+"]"; + logger.info(also); + } + return String.join((prevent == null || also == null) ? "" : "\n", prevent == null ? "" : prevent, also == null ? "" : also)+((prevent == null && also == null) ? "" : "\n"); + } + private static boolean isStandardType( String elementType ) { @@ -778,7 +1340,7 @@ public class GenerateXsd { } return result; } - + /** * Guaranteed to at least return non null but empty collection of edge descriptions * @param nodeName name of the vertex whose edge relationships to return @@ -799,11 +1361,13 @@ public class GenerateXsd { Map<String, Object> edgeMap; String fromNode; String toNode; - String ruleKey; String direction; String multiplicity; String isParent; String hasDelTarget; + String deleteOtherV; + String preventDelete; + String description; EdgeDescription edgeDes; while( edgeRulesIterator.hasNext() ){ @@ -817,6 +1381,9 @@ public class GenerateXsd { } edgeDes = x.new EdgeDescription(); edgeDes.setRuleKey(fromNode + "|" + toNode); + edgeDes.setLabel((String)edgeMap.get("label")); + edgeDes.setTo((String)edgeMap.get("to")); + edgeDes.setFrom((String)edgeMap.get("from")); direction = (String)edgeMap.get("direction"); edgeDes.setDirection(direction); multiplicity = (String)edgeMap.get("multiplicity"); @@ -828,9 +1395,16 @@ public class GenerateXsd { edgeDes.setType(LineageType.UNRELATED); } hasDelTarget = (String)edgeMap.get(EdgeProperty.DELETE_OTHER_V.toString()); + deleteOtherV = (String)edgeMap.get(EdgeProperty.DELETE_OTHER_V.toString()); + edgeDes.setDeleteOtherV(deleteOtherV); edgeDes.setHasDelTarget(hasDelTarget); - result.add(edgeDes); + preventDelete = (String)edgeMap.get(EdgeProperty.PREVENT_DELETE.toString()); + edgeDes.setPreventDelete(preventDelete); + description = (String)edgeMap.get(EdgeProperty.DESCRIPTION.toString()); + edgeDes.setDescription(description); + result.add(edgeDes); + logger.debug("Edge: "+edgeDes.getRuleKey()); } } catch (Exception ex) { ex.printStackTrace(); @@ -854,13 +1428,26 @@ public class GenerateXsd { return edges; } + private static Collection<EdgeDescription> getEdgeRulesTO( String nodeName ) + { + String toRulesPath = "$['rules'][?(@['to']=='" + nodeName + "')]"; + Collection<EdgeDescription> edges = getEdgeRulesFromJson( toRulesPath, true ); + return edges; + } + + private static Collection<EdgeDescription> getEdgeRulesFROM( String nodeName ) + { + String fromRulesPath = "$['rules'][?(@['from']=='" + nodeName + "')]"; + Collection<EdgeDescription> edges = getEdgeRulesFromJson( fromRulesPath, true ); + return edges; + } public static String processJavaTypeElementSwagger( String javaTypeName, Element javaTypeElement, StringBuffer pathSb, StringBuffer definitionsSb, String path, String tag, String opId, String getItemName, StringBuffer pathParams, String queryParams, String validEdges) { String xmlRootElementName = null; - - //Map<String, String> addJavaType = new HashMap<String, String>(); + StringBuilder definitionsLocalSb = new StringBuilder(256); + String useTag = null; String useOpId = null; @@ -871,38 +1458,11 @@ public class GenerateXsd { case "Business": case "LicenseManagement": case "CloudInfrastructure": - case "ExternalSystem": break; default: return null; } } - /* - if ( path == null ) - System.out.println( "processJavaTypeElementSwagger called with null path for javaTypeName " + javaTypeName); - */ - /* - if ( path == null || !(path.contains("cloud-infrastructure"))) - switch ( javaTypeName) { - case "Inventory": - useTag = null; - break; - - case "CloudInfrastructure": - case "Search": - case "Actions": - case "ServiceDesignAndCreation": - case "LicenseManagement": - case "Network": - if ( tag == null ) - useTag = javaTypeName; - - break; - default: - return null; - - } - */ if ( !javaTypeName.equals("Inventory") ) { if ( javaTypeName.equals("AaiInternal")) @@ -915,20 +1475,11 @@ public class GenerateXsd { useTag = javaTypeName; } - /* - if ( javaTypeName.equals("GenericVnf")) - System.out.println( "Processing " + javaTypeName); - else if ( javaTypeName.equals("Service")) - System.out.println( "Processing " + javaTypeName); - else if ( javaTypeName.equals("SitePair")) - System.out.println( "Processing " + javaTypeName); - */ NodeList parentNodes = javaTypeElement.getElementsByTagName("java-attributes"); if ( parentNodes.getLength() == 0 ) { - //System.out.println( "no java-attributes for java-type " + javaTypeName); + logger.trace( "no java-attributes for java-type " + javaTypeName); return ""; - } NamedNodeMap attributes; @@ -941,20 +1492,11 @@ public class GenerateXsd { String attrName = attr.getNodeName(); String attrValue = attr.getNodeValue(); - //System.out.println("Found xml-root-element attribute: " + attrName + " with value: " + attrValue); + logger.trace("Found xml-root-element attribute: " + attrName + " with value: " + attrValue); if ( attrName.equals("name")) xmlRootElementName = attrValue; } - /* - if ( xmlRootElementName.equals("oam-networks")) - System.out.println( "xmlRootElement oam-networks with getItemData [" + getItemName + "]"); - */ - //already processed - /* - if ( generatedJavaType.containsKey(xmlRootElementName) ) { - return null; - } - */ + NodeList childNodes; Element childElement; NodeList xmlPropNodes = javaTypeElement.getElementsByTagName("xml-properties"); @@ -964,10 +1506,6 @@ public class GenerateXsd { Vector<String> indexedProps = null; - /*System.out.println( "javaTypeName " + javaTypeName + " has xml-properties length " + xmlPropNodes.getLength()); - if ( path != null && path.equals("/network/generic-vnfs")) - System.out.println("path is " + "/network/generic-vnfs with getItemName " + getItemName); - */ if ( xmlPropNodes.getLength() > 0 ) { for ( int i = 0; i < xmlPropNodes.getLength(); ++i ) { @@ -993,8 +1531,6 @@ public class GenerateXsd { } if ( useValue == VALUE_DESCRIPTION && attrName.equals("value")) { pathDescriptionProperty = attrValue; - //break; - //System.out.println("found xml-element-wrapper " + xmlElementWrapper); } if ( attrValue.equals("indexedProps")) { useValue = VALUE_INDEXED_PROPS; @@ -1007,15 +1543,7 @@ public class GenerateXsd { } } } - //System.out.println("javaTypeName " + javaTypeName + " description " + pathDescriptionProperty); - - /* - if ( javaTypeName.equals("RelationshipList")) { - System.out.println( "Skipping " + javaTypeName); - generatedJavaType.put(javaTypeName, null); - return ""; - } - */ + logger.trace("javaTypeName " + javaTypeName + " description " + pathDescriptionProperty); Element parentElement = (Element)parentNodes.item(0); NodeList xmlElementNodes = parentElement.getElementsByTagName("xml-element"); @@ -1040,24 +1568,16 @@ public class GenerateXsd { if ( xmlRootElementName.equals("inventory")) path = ""; else if ( path == null ) - //path = "/aai/" + apiVersion; path = "/" + xmlRootElementName; else path += "/" + xmlRootElementName; st = new StringTokenizer(path, "/"); - /* - if ( path.equals("/business/customers/customer/{global-customer-id}/service-subscriptions/service-subscription")) - System.out.println("processing path /business/customers/customer/{global-customer-id}/service-subscriptions with tag " + tag); - */ boolean genPath = false; - /* - if ( path != null && path.equals("/network/generic-vnfs/generic-vnf")) - System.out.println("path is " + "/network/generic-vnfs/generic-vnf"); - */ + if ( st.countTokens() > 1 && getItemName == null ) { if ( appliedPaths.containsKey(path)) return null; - appliedPaths.put(path, null); + appliedPaths.put(path, xmlRootElementName); genPath = true; if ( path.contains("/relationship/") ) { // filter paths with relationship-list genPath = false; @@ -1075,23 +1595,7 @@ public class GenerateXsd { xmlElementElement = (Element)xmlElementNodes.item(i); if ( !xmlElementElement.getParentNode().isSameNode(parentElement)) continue; - /*childNodes = xmlElementElement.getElementsByTagName("xml-element-wrapper"); - if ( childNodes.getLength() > 0 ) { - childElement = (Element)childNodes.item(0); - // get name - attributes = childElement.getAttributes(); - for ( int k = 0; k < attributes.getLength(); ++k ) { - Attr attr = (Attr) attributes.item(k); - String attrName = attr.getNodeName(); - String attrValue = attr.getNodeValue(); - if ( attrName.equals("name")) { - xmlElementWrapper = attrValue; - //System.out.println("found xml-element-wrapper " + xmlElementWrapper); - } - } - } - */ valNodes = xmlElementElement.getElementsByTagName("xml-properties"); attrDescription = null; if ( valNodes.getLength() > 0 ) { @@ -1115,7 +1619,6 @@ public class GenerateXsd { } if ( useValue && attrName.equals("value")) { attrDescription = attrValue; - //System.out.println("found xml-element-wrapper " + xmlElementWrapper); } } @@ -1133,7 +1636,7 @@ public class GenerateXsd { String attrName = attr.getNodeName(); String attrValue = attr.getNodeValue(); - //System.out.println("For " + xmlRootElementName + " Found xml-element attribute: " + attrName + " with value: " + attrValue); + logger.trace("For " + xmlRootElementName + " Found xml-element attribute: " + attrName + " with value: " + attrValue); if ( attrName.equals("name")) { elementName = attrValue; @@ -1162,18 +1665,18 @@ public class GenerateXsd { if ( getItemName != null ) { if ( getItemName.equals("array") ) { if ( elementContainerType != null && elementContainerType.equals("java.util.ArrayList")) { - //System.out.println( " returning array " + elementName ); + logger.trace( " returning array " + elementName ); return elementName; } } else { // not an array check if ( elementContainerType == null || !elementContainerType.equals("java.util.ArrayList")) { - //System.out.println( " returning object " + elementName ); + logger.trace( " returning object " + elementName ); return elementName; } } - //System.out.println( " returning null" ); + logger.trace( " returning null" ); return null; } if ( elementIsRequired != null ) { @@ -1190,7 +1693,7 @@ public class GenerateXsd { } if ( elementIsKey != null ) { - sbParameters.append((" - name: " + elementName + "\n")); + sbParameters.append((" - name: " + elementName + "\n")); sbParameters.append((" in: path\n")); if ( attrDescription != null && attrDescription.length() > 0 ) sbParameters.append((" description: " + attrDescription + "\n")); @@ -1205,10 +1708,12 @@ public class GenerateXsd { sbParameters.append(" type: integer\n"); sbParameters.append(" format: int32\n"); } - if ( elementType.equals("java.lang.Boolean")) + if ( elementType.equals("java.lang.Boolean")) { sbParameters.append(" type: boolean\n"); - - + } + if(StringUtils.isNotBlank(elementName)) { + sbParameters.append(" example: "+"__"+elementName.toUpperCase()+"__"+"\n"); + } } else if ( indexedProps != null && indexedProps.contains(elementName ) ) { sbIndexedParams.append((" - name: " + elementName + "\n")); @@ -1229,12 +1734,6 @@ public class GenerateXsd { if ( elementType.equals("java.lang.Boolean")) sbIndexedParams.append(" type: boolean\n"); } - - /* - if ( elementName != null && elementName.equals("inventory-item")) - System.out.println( "processing inventory-item elementName"); - */ - if ( isStandardType(elementType)) { sbProperties.append(" " + elementName + ":\n"); ++propertyCnt; @@ -1256,8 +1755,6 @@ public class GenerateXsd { sbProperties.append(" description: " + attrDescription + "\n"); } - //if ( addType != null && elementContainerType != null && elementContainerType.equals("java.util.ArrayList") ) { - if ( addTypeV != null ) { StringBuffer newPathParams = null; if ( pathParams != null ) { @@ -1293,21 +1790,17 @@ public class GenerateXsd { if ( itemName != null ) { if ( addType.equals("AaiInternal") ) { - //System.out.println( "addType AaiInternal, skip properties"); + logger.debug( "addType AaiInternal, skip properties"); } else if ( getItemName == null) { ++propertyCnt; sbProperties.append(" " + getXmlRootElementName(addType) + ":\n"); sbProperties.append(" type: array\n items:\n"); - sbProperties.append(" $ref: \"#/definitions/" + itemName + "\"\n"); + sbProperties.append(" $ref: \"#/definitions/" + (itemName == "" ? "aai-internal" : itemName) + "\"\n"); if ( attrDescription != null && attrDescription.length() > 0 ) sbProperties.append(" description: " + attrDescription + "\n"); } } else { - /*itemName = processJavaTypeElementSwagger( addType, getJavaTypeElementSwagger(addType), - pathSb, definitionsSb, path, tag == null ? useTag : tag, useOpId, "other" ); - if ( itemName != null ) { - */ if ( elementContainerType != null && elementContainerType.equals("java.util.ArrayList")) { // need properties for getXmlRootElementName(addType) newPathParams = null; @@ -1343,55 +1836,16 @@ public class GenerateXsd { if ( attrDescription != null && attrDescription.length() > 0 ) sbProperties.append(" description: " + attrDescription + "\n"); ++propertyCnt; - /*} - else { - System.out.println(" unable to define swagger object for " + addType); - } - */ } - //if ( getItemName == null) looking for missing properties - //generatedJavaType.put(addType, null); } } } } if ( genPath ) { - /* - if ( useOpId.equals("CloudInfrastructureComplexesComplexCtagPools")) - System.out.println( "adding path CloudInfrastructureComplexesComplexCtagPools"); - */ - if ( !path.endsWith("/relationship") ) { - pathSb.append(" " + path + ":\n" ); - pathSb.append(" get:\n"); - pathSb.append(" tags:\n"); - pathSb.append(" - " + tag + "\n"); - pathSb.append(" summary: returns " + xmlRootElementName + "\n"); - - pathSb.append(" description: returns " + xmlRootElementName + "\n"); - pathSb.append(" operationId: get" + useOpId + "\n"); - pathSb.append(" produces:\n"); - pathSb.append(" - application/json\n"); - pathSb.append(" - application/xml\n"); - - pathSb.append(" responses:\n"); - pathSb.append(" \"200\":\n"); - pathSb.append(" description: successful operation\n"); - pathSb.append(" schema:\n"); - pathSb.append(" $ref: \"#/definitions/" + xmlRootElementName + "\"\n"); - pathSb.append(" \"default\":\n"); - pathSb.append(" " + responsesUrl); - /* - pathSb.append(" \"200\":\n"); - pathSb.append(" description: successful operation\n"); - pathSb.append(" schema:\n"); - pathSb.append(" $ref: \"#/definitions/" + xmlRootElementName + "\"\n"); - pathSb.append(" \"404\":\n"); - pathSb.append(" description: resource was not found\n"); - pathSb.append(" \"400\":\n"); - pathSb.append(" description: bad request\n"); - */ - if ( path.indexOf('{') > 0 ) { + GetOperation get = new GetOperation(useOpId, xmlRootElementName, tag, path, pathParams == null ? "" : pathParams.toString()); + pathSb.append(get.toString()); +// if ( path.indexOf('{') > 0 ) { if ( sbParameters.toString().length() > 0 ) { if ( pathParams == null ) @@ -1402,7 +1856,7 @@ public class GenerateXsd { pathSb.append(" parameters:\n"); pathSb.append(pathParams); } else - System.out.println( "null pathParams for " + useOpId); + logger.trace( "null pathParams for " + useOpId); if ( sbIndexedParams.toString().length() > 0 ) { if ( queryParams == null ) queryParams = sbIndexedParams.toString(); @@ -1415,7 +1869,7 @@ public class GenerateXsd { } pathSb.append(queryParams); } - } +// } } boolean skipPutDelete = false; // no put or delete for "all" if ( !path.endsWith("/relationship") ) { @@ -1426,103 +1880,18 @@ public class GenerateXsd { } if ( path.indexOf('{') > 0 && !opId.startsWith("Search") &&!skipPutDelete) { // add PUT - if ( path.endsWith("/relationship") ) { - pathSb.append(" " + path + ":\n" ); - } - pathSb.append(" put:\n"); - pathSb.append(" tags:\n"); - pathSb.append(" - " + tag + "\n"); - - if ( path.endsWith("/relationship") ) { - pathSb.append(" summary: see node definition for valid relationships\n"); - } else { - pathSb.append(" summary: create or update an existing " + xmlRootElementName + "\n"); - pathSb.append(" description: create or update an existing " + xmlRootElementName + "\n"); - } - pathSb.append(" operationId: createOrUpdate" + useOpId + "\n"); - pathSb.append(" consumes:\n"); - pathSb.append(" - application/json\n"); - pathSb.append(" - application/xml\n"); - pathSb.append(" produces:\n"); - pathSb.append(" - application/json\n"); - pathSb.append(" - application/xml\n"); - pathSb.append(" responses:\n"); - pathSb.append(" \"default\":\n"); - pathSb.append(" " + responsesUrl); - /* - pathSb.append(" responses:\n"); - pathSb.append(" \"200\":\n"); - pathSb.append(" description: existing resource has been modified and there is a response buffer\n"); - pathSb.append(" \"201\":\n"); - pathSb.append(" description: new resource is created\n"); - pathSb.append(" \"202\":\n"); - pathSb.append(" description: action requested but may have taken other actions as well, which are returned in the response payload\n"); - pathSb.append(" \"204\":\n"); - pathSb.append(" description: existing resource has been modified and there is no response buffer\n"); - pathSb.append(" \"400\":\n"); - pathSb.append(" description: Bad Request will be returned if headers are missing\n"); - pathSb.append(" \"404\":\n"); - pathSb.append(" description: Not Found will be returned if an unknown URL is used\n"); - */ - pathSb.append(" parameters:\n"); - //pathSb.append(" - in: path\n"); - pathSb.append(pathParams); // for nesting - pathSb.append(" - name: body\n"); - pathSb.append(" in: body\n"); - pathSb.append(" description: " + xmlRootElementName + " object that needs to be created or updated\n"); - pathSb.append(" required: true\n"); - pathSb.append(" schema:\n"); - pathSb.append(" $ref: \"#/definitions/" + xmlRootElementName + "\"\n"); - /* - if ( queryParams != null ) { - pathSb.append(queryParams); - } - */ - // add DELETE - pathSb.append(" delete:\n"); - pathSb.append(" tags:\n"); - pathSb.append(" - " + tag + "\n"); - pathSb.append(" summary: delete an existing " + xmlRootElementName + "\n"); - - pathSb.append(" description: delete an existing " + xmlRootElementName + "\n"); - - pathSb.append(" operationId: delete" + useOpId + "\n"); - pathSb.append(" consumes:\n"); - pathSb.append(" - application/json\n"); - pathSb.append(" - application/xml\n"); - pathSb.append(" produces:\n"); - pathSb.append(" - application/json\n"); - pathSb.append(" - application/xml\n"); - pathSb.append(" responses:\n"); - pathSb.append(" \"default\":\n"); - pathSb.append(" " + responsesUrl); - /* - pathSb.append(" responses:\n"); - pathSb.append(" \"200\":\n"); - pathSb.append(" description: successful, the response includes an entity describing the status\n"); - pathSb.append(" \"204\":\n"); - pathSb.append(" description: successful, action has been enacted but the response does not include an entity\n"); - pathSb.append(" \"400\":\n"); - pathSb.append(" description: Bad Request will be returned if headers are missing\n"); - pathSb.append(" \"404\":\n"); - pathSb.append(" description: Not Found will be returned if an unknown URL is used\n"); - */ - pathSb.append(" parameters:\n"); - //pathSb.append(" - in: path\n"); - pathSb.append(pathParams); // for nesting + PutOperation put = new PutOperation(useOpId, xmlRootElementName, tag, path, pathParams == null ? "" : pathParams.toString()); + pathSb.append(put.toString()); if ( !path.endsWith("/relationship") ) { - pathSb.append(" - name: resource-version\n"); - - pathSb.append(" in: query\n"); - pathSb.append(" description: resource-version for concurrency\n"); - pathSb.append(" required: true\n"); - pathSb.append(" type: string\n"); + PatchOperation patch = new PatchOperation(useOpId, xmlRootElementName, tag, path, pathParams == null ? "" : pathParams.toString()); + pathSb.append(patch.toString1()); } - /* - if ( queryParams != null ) { - pathSb.append(queryParams); - } - */ + logger.debug(put.tagRelationshipPathMapEntry()); + + // add DELETE + DeleteOperation del = new DeleteOperation(useOpId, xmlRootElementName, tag, path, pathParams == null ? "" : pathParams.toString()); + pathSb.append(del.toString()); + logger.debug(del.objectPathMapEntry()); } } @@ -1531,11 +1900,13 @@ public class GenerateXsd { } definitionsSb.append(" " + xmlRootElementName + ":\n"); + definitionsLocalSb.append(" " + xmlRootElementName + ":\n"); Collection<EdgeDescription> edges = getEdgeRules(xmlRootElementName ); + if ( edges.size() > 0 ) { StringBuffer sbEdge = new StringBuffer(); sbEdge.append(" ###### Related Nodes\n"); - for (EdgeDescription ed : edges) { + for (EdgeDescription ed : edges) { if ( ed.getRuleKey().startsWith(xmlRootElementName)) { sbEdge.append(" - TO ").append(ed.getRuleKey().substring(ed.getRuleKey().indexOf("|")+1)); sbEdge.append(ed.getRelationshipDescription("TO", xmlRootElementName)); @@ -1549,51 +1920,94 @@ public class GenerateXsd { sbEdge.append("\n"); } } + // Delete rule processing is incorrect. One cannot express the delete rules in isolation from the + // specific edge. Take the case of allotted-resource and service-instance. When the service-instance owns the + // allotted-resource, yes, it deletes it. But when the service-instance only uses the allotted-resource, the deletion + // of the service instance does not cause the deletion of the allotted-resource. + // I put some lines into the toDeleteRules and fromDeleteRules to correct things to an extent, but it's still + // not right. + sbEdge.append(toDeleteRules(xmlRootElementName)); + sbEdge.append(fromDeleteRules(xmlRootElementName)); validEdges = sbEdge.toString(); } // Handle description property. Might have a description OR valid edges OR both OR neither. // Only put a description: tag if there is at least one. if (pathDescriptionProperty != null || validEdges != null) { - definitionsSb.append(" description: |\n"); + definitionsSb.append(" description: |\n"); + definitionsLocalSb.append(" description: |\n"); - if ( pathDescriptionProperty != null ) + if ( pathDescriptionProperty != null ) { definitionsSb.append(" " + pathDescriptionProperty + "\n" ); - if (validEdges != null) + definitionsLocalSb.append(" " + pathDescriptionProperty + "\n" ); + } + if (validEdges != null) { definitionsSb.append(validEdges); + definitionsLocalSb.append(validEdges); + } } - if ( requiredCnt > 0 ) + if ( requiredCnt > 0 ) { definitionsSb.append(sbRequired); + definitionsLocalSb.append(sbRequired); + } + if ( propertyCnt > 0 ) { definitionsSb.append(" properties:\n"); definitionsSb.append(sbProperties); + definitionsLocalSb.append(" properties:\n"); + definitionsLocalSb.append(sbProperties); + } + try { + javaTypeDefinitions.put(xmlRootElementName, definitionsLocalSb.toString()); + } catch (Exception e) { + e.printStackTrace(); } generatedJavaType.put(xmlRootElementName, null); return null; } - public static String generateSwaggerFromOxmFile( File oxmFile ) - { + public static void generateRelations() { + if(putRelationPaths == null) + return; + putRelationPaths.forEach((k,v)->{ + logger.trace("k="+k+"\n"+"v="+v+v.equals("/business/customers/customer/{global-customer-id}/service-subscriptions/service-subscription/{service-type}/service-instances/service-instance/{service-instance-id}/allotted-resources/allotted-resource/{id}/relationship-list/relationship")); + logger.debug("apiPath(Operation): "+v); + logger.debug("Target object: "+v.replace("/relationship-list/relationship", "")); + logger.debug("Relations: "); + PutRelationPathSet prp = new PutRelationPathSet(k, v); + prp.process(); + }); + } + public static String generateSwaggerFromOxmFile( File oxmFile, String xml ) + { + if ( xml != null ){ + apiVersion = Version.getLatest().toString(); + apiVersionFmt = "." + apiVersion + "."; + generatedJavaType = new HashMap<>(); + appliedPaths = new HashMap<>(); + responsesUrl = "Description: response-label\n"; + } StringBuffer sb = new StringBuffer(); - sb.append("swagger: \"2.0\"\ninfo:\n description: |\n Copyright © 2017 AT&T Intellectual Property. All rights reserved.\n\n Licensed under the Creative Commons License, Attribution 4.0 Intl. (the "License"); you may not use this documentation except in compliance with the License.\n\n You may obtain a copy of the License at\n\n (https://creativecommons.org/licenses/by/4.0/)\n\n Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\n\n ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property.\n\n This document is best viewed with Firefox or Chrome. Nodes can be found by appending /#/definitions/node-type-to-find to the path to this document. Edge definitions can be found with the node definitions.\n version: \"" + apiVersion +"\"\n"); + sb.append("swagger: \"2.0\"\ninfo:\n "); + sb.append("description: |"); + sb.append("\n\n [Differences versus the previous schema version]("+"apidocs/aai_swagger_" + apiVersion + ".diff)"); + sb.append("\n\n Copyright © 2017 AT&T Intellectual Property. All rights reserved.\n\n Licensed under the Creative Commons License, Attribution 4.0 Intl. (the "License"); you may not use this documentation except in compliance with the License.\n\n You may obtain a copy of the License at\n\n (https://creativecommons.org/licenses/by/4.0/)\n\n Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\n\n ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property.\n\n This document is best viewed with Firefox or Chrome. Nodes can be found by appending /#/definitions/node-type-to-find to the path to this document. Edge definitions can be found with the node definitions.\n version: \"" + apiVersion +"\"\n"); sb.append(" title: Active and Available Inventory REST API\n"); sb.append(" license:\n name: Apache 2.0\n url: http://www.apache.org/licenses/LICENSE-2.0.html\n"); sb.append(" contact:\n name:\n url:\n email:\n"); sb.append("host:\nbasePath: /aai/" + apiVersion + "\n"); sb.append("schemes:\n - https\npaths:\n"); - /* - sb.append("responses:\n"); - sb.append(" \"200\":\n"); - sb.append(" description: successful operation\n"); - sb.append(" \"404\":\n"); - sb.append(" description: resource was not found\n"); - sb.append(" \"400\":\n"); - sb.append(" description: bad request\n"); - */ + try { - File initialFile = new File("src/main/resources/dbedgerules/DbEdgeRules_" + apiVersion + ".json"); + File initialFile; + if(System.getProperty("user.dir") != null && !System.getProperty("user.dir").contains(normalStartDir)) { + initialFile = new File(normalStartDir + "/src/main/resources/dbedgerules/DbEdgeRules_" + apiVersion + ".json"); + } + else { + initialFile = new File("src/main/resources/dbedgerules/DbEdgeRules_" + apiVersion + ".json"); + } InputStream is = new FileInputStream(initialFile); Scanner scanner = new Scanner(is); @@ -1604,7 +2018,14 @@ public class GenerateXsd { DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); dbFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - Document doc = dBuilder.parse(oxmFile); + Document doc; + + if ( xml == null ) { + doc = dBuilder.parse(oxmFile); + } else { + InputSource isInput = new InputSource(new StringReader(xml)); + doc = dBuilder.parse(isInput); + } NodeList bindingsNodes = doc.getElementsByTagName("xml-bindings"); Element bindingElement; @@ -1615,21 +2036,21 @@ public class GenerateXsd { if ( bindingsNodes == null || bindingsNodes.getLength() == 0 ) { - System.out.println( "missing <binding-nodes> in " + oxmFile ); + logger.error( "missing <binding-nodes> in " + oxmFile ); return null; } bindingElement = (Element) bindingsNodes.item(0); javaTypesNodes = bindingElement.getElementsByTagName("java-types"); if ( javaTypesNodes.getLength() < 1 ) { - System.out.println( "missing <binding-nodes><java-types> in " + oxmFile ); + logger.error( "missing <binding-nodes><java-types> in " + oxmFile ); return null; } javaTypesElement = (Element) javaTypesNodes.item(0); javaTypeNodes = javaTypesElement.getElementsByTagName("java-type"); if ( javaTypeNodes.getLength() < 1 ) { - System.out.println( "missing <binding-nodes><java-types><java-type> in " + oxmFile ); + logger.error( "missing <binding-nodes><java-types><java-type> in " + oxmFile ); return null; } @@ -1638,8 +2059,7 @@ public class GenerateXsd { Attr attr; StringBuffer pathSb = new StringBuffer(); - StringBuffer definitionsSb = new StringBuffer("definitions:\n"); - + StringBuffer definitionsSb = new StringBuffer(); for ( int i = 0; i < javaTypeNodes.getLength(); ++ i ) { javaTypeElement = (Element) javaTypeNodes.item(i); NamedNodeMap attributes = javaTypeElement.getAttributes(); @@ -1652,28 +2072,67 @@ public class GenerateXsd { javaTypeName = attrValue; } if ( javaTypeName == null ) { - System.out.println( "<java-type> has no name attribute in " + oxmFile ); + logger.error( "<java-type> has no name attribute in " + oxmFile ); return null; } if ( !generatedJavaType.containsKey(getXmlRootElementName(javaTypeName)) ) { - //generatedJavaType.put(javaTypeName, null); - //if ( javaTypeName.equals("search")||javaTypeName.equals("actions")) - processJavaTypeElementSwagger( javaTypeName, javaTypeElement, pathSb, definitionsSb, null, null, null, null, null, null, null); } } sb.append(pathSb); - //System.out.println( "definitions block\n" + definitionsSb.toString()); - sb.append(definitionsSb.toString()); - //sb.append(definitionsSb); - } catch (Exception e) { e.printStackTrace(); return null; } - //System.out.println("generated " + sb.toString()); + //append definitions + sb.append("definitions:\n"); + Map<String, String> sortedJavaTypeDefinitions = new TreeMap<String, String>(javaTypeDefinitions); + for (Map.Entry<String, String> entry : sortedJavaTypeDefinitions.entrySet()) { + logger.debug("Key: "+entry.getKey()+"Test: "+ (entry.getKey() == "relationship")); + if(entry.getKey().matches("relationship")) { + String jb=entry.getValue(); + logger.debug("Value: "+jb); + int ndx=jb.indexOf("related-to-property:"); + if(ndx > 0) { + jb=jb.substring(0, ndx); + jb=jb.replaceAll(" +$", ""); + } + logger.debug("Value-after: "+jb); + sb.append(jb); + continue; + } + sb.append(entry.getValue()); + } + + sb.append("patchDefinitions:\n"); + for (Map.Entry<String, String> entry : sortedJavaTypeDefinitions.entrySet()) { + String jb=entry.getValue().replaceAll("/definitions/", "/patchDefinitions/"); + int ndx=jb.indexOf("relationship-list:"); + if(ndx > 0) { + jb=jb.substring(0, ndx); + jb=jb.replaceAll(" +$", ""); + } + int ndx1=jb.indexOf("resource-version:"); + logger.debug("Key: "+entry.getKey()+" index: " + ndx1); + logger.debug("Value: "+jb); + if(ndx1 > 0) { + jb=jb.substring(0, ndx1); + jb=jb.replaceAll(" +$", ""); + } + logger.debug("Value-after: "+jb); + sb.append(jb); + } + + sb.append("getDefinitions:\n"); + for (Map.Entry<String, String> entry : sortedJavaTypeDefinitions.entrySet()) { + String jb=entry.getValue().replaceAll("/definitions/", "/getDefinitions/"); + sb.append(jb); + } + + logger.debug("generated " + sb.toString()); + generateRelations(); return sb.toString(); } @@ -1686,7 +2145,7 @@ public class GenerateXsd { Object nodeset = expr.evaluate(element, XPathConstants.NODESET); if (nodeset != null) { NodeList nodes = (NodeList) nodeset; - if (nodes.getLength() > 0) { + if (nodes != null && nodes.getLength() > 0) { Element xmlProperty = (Element)nodes.item(0); result = xmlProperty.getElementsByTagName("xml-property"); } @@ -1698,4 +2157,4 @@ public class GenerateXsd { return result; } -} +}
\ No newline at end of file diff --git a/aai-core/src/main/java/org/onap/aai/util/HttpsAuthClient.java b/aai-core/src/main/java/org/onap/aai/util/HttpsAuthClient.java index 24ba1ba8..5deb48ae 100644 --- a/aai-core/src/main/java/org/onap/aai/util/HttpsAuthClient.java +++ b/aai-core/src/main/java/org/onap/aai/util/HttpsAuthClient.java @@ -61,7 +61,7 @@ public class HttpsAuthClient{ // System.out.println("Jersey result: "); // System.out.println(res.getEntity(String.class).toString()); - + } catch (KeyManagementException e) { e.printStackTrace(); } catch (Exception e) { diff --git a/aai-core/src/main/java/org/onap/aai/util/HttpsAuthExternalClient.java b/aai-core/src/main/java/org/onap/aai/util/HttpsAuthExternalClient.java new file mode 100644 index 00000000..a984c1e6 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/util/HttpsAuthExternalClient.java @@ -0,0 +1,150 @@ +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.util; + +import java.io.FileInputStream; +import java.security.KeyManagementException; +import java.security.KeyStore; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; + +import org.onap.aai.domain.yang.Customers; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.client.urlconnection.HTTPSProperties; + +public class HttpsAuthExternalClient { + + /** + * The main method. + * + * @param args the arguments + */ + public static void main(String[] args) { + try { + String url = AAIConfig.get(AAIConstants.AAI_SERVER_URL) + "business/customers"; + System.out.println("Making Jersey https call..."); + String keystore = args[0]; + String keypasswd = args[1]; + Client client = HttpsAuthExternalClient.getClient(keystore, keypasswd); + + ClientResponse res = client.resource(url) + .accept("application/json") + .header("X-TransactionId", "PROV001") + .header("X-FromAppId", "AAI") + .type("application/json") + .get(ClientResponse.class); + +// System.out.println("Jersey result: "); +// System.out.println(res.getEntity(String.class).toString()); + + Customers customers = res.getEntity(Customers.class); + System.out.println("Jersey result: "); + System.out.println("Number of customers: " + customers.getCustomer().size()); + + } catch (KeyManagementException e) { + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * Gets the client. + * + * @param keystoreFileName the keystore file name + * @param keystorePassword the keystore password + * @return the client + * @throws Exception the exception + */ + public static Client getClient ( String keystoreFileName, String keystorePassword ) throws Exception { + + ClientConfig config = new DefaultClientConfig(); + config.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE); + config.getClasses().add(org.onap.aai.restcore.CustomJacksonJaxBJsonProvider.class); + Client client = null; + SSLContext ctx = null; + + try { + String truststore_path = AAIConstants.AAI_HOME_ETC_AUTH + AAIConfig.get(AAIConstants.AAI_TRUSTSTORE_FILENAME); + String truststore_password = AAIConfig.get(AAIConstants.AAI_TRUSTSTORE_PASSWD); + String keystore_path = AAIConstants.AAI_HOME_ETC_AUTH + keystoreFileName; + String keystore_password = keystorePassword; + + //System.setProperty("javax.net.ssl.trustStore", truststore_path); + //System.setProperty("javax.net.ssl.trustStorePassword", truststore_password); + HttpsURLConnection.setDefaultHostnameVerifier( new HostnameVerifier(){ + public boolean verify(String string,SSLSession ssls) { + return true; + } + }); + + ctx = SSLContext.getInstance("TLS"); + KeyManagerFactory kmf = null; + + + /**** kmf = KeyManagerFactory.getInstance("SunX509"); + FileInputStream fin = new FileInputStream(keystore_path); + KeyStore ks = KeyStore.getInstance("PKCS12"); + char[] pwd = keystore_password.toCharArray(); + ks.load(fin, pwd); + kmf.init(ks, pwd); + ***/ + + String alg = TrustManagerFactory.getDefaultAlgorithm(); + TrustManagerFactory tmf = TrustManagerFactory.getInstance(alg); + FileInputStream tin = new FileInputStream(truststore_path); + KeyStore ts = KeyStore.getInstance("PKCS12"); + char[] tpwd = truststore_password.toCharArray(); + ts.load(tin, tpwd); + tmf.init(ts); + + //ctx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null); + // Updating key manager to null, to disable two way SSL + ctx.init(null, tmf.getTrustManagers(), null); + + config.getProperties().put(HTTPSProperties.PROPERTY_HTTPS_PROPERTIES, + new HTTPSProperties( new HostnameVerifier() { + @Override + public boolean verify( String s, SSLSession sslSession ) { + return true; + } + }, ctx)); + + client = Client.create(config); + // uncomment this line to get more logging for the request/response + // client.addFilter(new LoggingFilter(System.out)); + } catch (Exception e) { + throw e; + } + return client; + } + +} diff --git a/aai-core/src/main/java/org/onap/aai/util/Request.java b/aai-core/src/main/java/org/onap/aai/util/Request.java index 605aa2b9..46c891c2 100644 --- a/aai-core/src/main/java/org/onap/aai/util/Request.java +++ b/aai-core/src/main/java/org/onap/aai/util/Request.java @@ -33,10 +33,7 @@ import org.onap.aai.exceptions.AAIException; public class Request<T> { - public static final String V2 = "v2"; - public static final String V3 = "v3"; - public static final String V4 = "v4"; - public static final String V5 = "v5"; + public static final String V12 = "v12"; public final String fromAppId; public final String transactionId; public final String path; @@ -73,7 +70,7 @@ public class Request<T> { private String path; private RestObject<T> restObj; private boolean oldServer; - private String apiVersion = Request.V4; + private String apiVersion = Request.V12; /** diff --git a/aai-core/src/main/java/org/onap/aai/util/RestController.java b/aai-core/src/main/java/org/onap/aai/util/RestController.java new file mode 100644 index 00000000..421946ef --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/util/RestController.java @@ -0,0 +1,469 @@ +/*- + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.util; + +import java.security.KeyManagementException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.logging.LoggingContext; + +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.TypeFactory; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; + +public class RestController { + + private static final String TARGET_NAME = "AAI"; + private static EELFLogger LOGGER = EELFManager.getInstance().getLogger(RestController.class); + + private static Client client = null; + + private String restSrvrBaseURL; + + //To do - Come up with helper function that will automatically + //generate the REST API path based on path parameter(s) and query parameter(s)! + public static final String REST_APIPATH_COMPLEXES = "cloud-infrastructure/complexes"; + public static final String REST_APIPATH_COMPLEX = "cloud-infrastructure/complexes/complex/"; + public static final String REST_APIPATH_PSERVERS = "cloud-infrastructure/pservers"; + public static final String REST_APIPATH_PSERVER = "cloud-infrastructure/pservers/pserver/"; + public static final String REST_APIPATH_PHYSICALLINKS = "network/physical-links/"; + public static final String REST_APIPATH_PHYSICALLINK = "network/physical-links/physical-link/"; + public static final String REST_APIPATH_PINTERFACES = "network/p-interfaces/"; + public static final String REST_APIPATH_PINTERFACE = "network/p-interfaces/p-interface/"; + public static final String REST_APIPATH_VPLSPES = "network/vpls-pes/"; + public static final String REST_APIPATH_VPLSPE = "network/vpls-pes/vpls-pe/"; + public static final String REST_APIPATH_UPDATE = "actions/update/"; + public static final String REST_APIPATH_SEARCH = "search/nodes-query?search-node-type="; + + public static final String REST_APIPATH_CLOUDREGION = "cloud-infrastructure/cloud-regions/cloud-region/"; + public static final String REST_APIPATH_TENANT = "cloud-infrastructure/tenants/tenant/"; + public static final String REST_APIPATH_VIRTUAL_DATA_CENTER = "cloud-infrastructure/virtual-data-centers/virtual-data-center/"; + public static final String REST_APIPATH_VIRTUAL_DATA_CENTERS = "cloud-infrastructure/virtual-data-centers/"; + //network/generic-vnfs/generic-vnf/{vnf-id} + public static final String REST_APIPATH_GENERIC_VNF = "network/generic-vnfs/generic-vnf/"; + public static final String REST_APIPATH_GENERIC_VNFS = "network/generic-vnfs"; + public static final String REST_APIPATH_L3_NETWORK = "network/l3-networks/l3-network/"; + public static final String REST_APIPATH_L3_NETWORKS = "network/l3-networks"; + public static final String REST_APIPATH_INSTANCE_GROUP = "network/instance-groups/instance-group"; + public static final String REST_APIPATH_INSTANCE_GROUPS = "network/instance-groups"; + + public static final String REST_APIPATH_VCE = "network/vces/vce/"; + + public static final String REST_APIPATH_SERVICE = "service-design-and-creation/services/service/"; + public static final String REST_APIPATH_LOGICALLINKS = "network/logical-links/"; + public static final String REST_APIPATH_LOGICALLINK = "network/logical-links/logical-link/"; + + /** + * Inits the rest client. + * + * @throws AAIException the AAI exception + */ + private static void initRestClient() throws AAIException + { + if (client == null) { + try { + client = HttpsAuthClient.getClient(); + } + catch (KeyManagementException e){ + throw new AAIException("AAI_7117", "KeyManagementException in REST call to DB: " + e.toString()); + } catch (Exception e) { + throw new AAIException("AAI_7117", " Exception in REST call to DB: " + e.toString()); + } + } + } + + /** + * Sets the rest srvr base URL. + * + * @param baseURL the base URL + * @throws AAIException the AAI exception + */ + public void SetRestSrvrBaseURL(String baseURL) throws AAIException + { + if (baseURL == null) + throw new AAIException("AAI_7117", "REST Server base URL cannot be null."); + restSrvrBaseURL = baseURL; + } + + /** + * Gets the rest srvr base URL. + * + * @return the rest srvr base URL + */ + public String getRestSrvrBaseURL() + { + return restSrvrBaseURL; + } + + + public static <T> void Get(T t, String sourceID, String transId, String path, RestObject<T> restObject, boolean oldserver) throws AAIException { + RestController.<T>Get(t, sourceID, transId, path, restObject, oldserver, AAIConstants.AAI_RESOURCES_PORT); + } + /** + * To do - optimization and automation. Also make it as generic as possible. + * + * @param <T> the generic type + * @param t the t + * @param sourceID the source ID + * @param transId the trans id + * @param path the path + * @param restObject the rest object + * @param oldserver the oldserver + * @throws AAIException the AAI exception + */ + @SuppressWarnings("unchecked") + public static <T> void Get(T t, String sourceID, String transId, String path, RestObject<T> restObject, boolean oldserver, int port) throws AAIException { + String methodName = "Get"; + String url=""; + transId += ":" + UUID.randomUUID().toString(); + + LoggingContext.save(); + LoggingContext.partnerName(sourceID); + LoggingContext.targetEntity(TARGET_NAME); + LoggingContext.requestId(transId); + LoggingContext.serviceName(methodName); + LoggingContext.targetServiceName(methodName); + + LOGGER.debug(methodName + " start"); + + restObject.set(t); + + if (oldserver) + url = AAIConfig.get(AAIConstants.AAI_OLDSERVER_URL) + path; + else + url = String.format(AAIConstants.AAI_LOCAL_REST, port) + path; + initRestClient(); + LOGGER.debug(url + " for the get REST API"); + ClientResponse cres = client.resource(url) + .accept("application/json") + .header("X-TransactionId", transId) + .header("X-FromAppId", sourceID) + .header("Real-Time", "true") + .type("application/json") + .get(ClientResponse.class); + +// System.out.println("cres.EntityInputSream()="+cres.getEntityInputStream().toString()); +// System.out.println("cres.tostring()="+cres.toString()); + + if (cres.getStatus() == 200) { +// System.out.println(methodName + ": url=" + url); + t = (T) cres.getEntity(t.getClass()); + restObject.set(t); + LOGGER.debug(methodName + "REST api GET was successfull!"); + } else { + LoggingContext.restore(); +// System.out.println(methodName + ": url=" + url + " failed with status=" + cres.getStatus()); + throw new AAIException("AAI_7116", methodName +" with status="+cres.getStatus()+", url="+url); + } + + LoggingContext.restore(); + } + + /** + * Map json to object list. + * + * @param <T> the generic type + * @param typeDef the type def + * @param json the json + * @param clazz the clazz + * @return the list + * @throws Exception the exception + */ + private static <T> List<T> mapJsonToObjectList(T typeDef,String json, Class clazz) throws Exception + { + List<T> list; + ObjectMapper mapper = new ObjectMapper(); + System.out.println(json); + TypeFactory t = TypeFactory.defaultInstance(); + list = mapper.readValue(json, t.constructCollectionType(ArrayList.class,clazz)); + + return list; + } + + /** + * Put. + * + * @param <T> the generic type + * @param t the t + * @param sourceID the source ID + * @param transId the trans id + * @param path the path + * @throws AAIException the AAI exception + */ + public static <T> void Put(T t, String sourceID, String transId, String path) throws AAIException { + Put( t, sourceID, transId, path, false, AAIConstants.AAI_RESOURCES_PORT); + } + + /** + * Put. + * + * @param <T> the generic type + * @param t the t + * @param sourceID the source ID + * @param transId the trans id + * @param path the path + * @throws AAIException the AAI exception + */ + public static <T> void Put(T t, String sourceID, String transId, String path, boolean oldserver) throws AAIException { + Put( t, sourceID, transId, path, oldserver, AAIConstants.AAI_RESOURCES_PORT); + } + + /** + * Put. + * + * @param <T> the generic type + * @param t the t + * @param sourceID the source ID + * @param transId the trans id + * @param path the path + * @param oldserver the oldserver + * @throws AAIException the AAI exception + */ + public static <T> void Put(T t, String sourceID, String transId, String path, boolean oldserver, int port) throws AAIException { + String methodName = "Put"; + String url=""; + transId += ":" + UUID.randomUUID().toString(); + + LoggingContext.save(); + LoggingContext.partnerName(sourceID); + LoggingContext.targetEntity(TARGET_NAME); + LoggingContext.requestId(transId); + LoggingContext.serviceName(methodName); + LoggingContext.targetServiceName(methodName); + + LOGGER.debug(methodName + " start"); + + initRestClient(); + + if (oldserver) + url = AAIConfig.get(AAIConstants.AAI_OLDSERVER_URL) + path; + else + url = String.format(AAIConstants.AAI_LOCAL_REST, port) + path; + + ClientResponse cres = client.resource(url) + .accept("application/json") + .header("X-TransactionId", transId) + .header("X-FromAppId", sourceID) + .header("Real-Time", "true") + .type("application/json") + .entity(t) + .put(ClientResponse.class); + +// System.out.println("cres.tostring()="+cres.toString()); + + int statuscode = cres.getStatus(); + if ( statuscode >= 200 && statuscode <= 299 ) { + LOGGER.debug(methodName+": url=" + url + ", request=" + path); + LoggingContext.restore(); + } else { + LoggingContext.restore(); + throw new AAIException("AAI_7116", methodName +" with status="+statuscode+", url="+url + ", msg=" + cres.getEntity(String.class)); + } + } + + public static void Delete(String sourceID, String transId, String path) throws AAIException { + RestController.Delete(sourceID, transId, path, AAIConstants.AAI_RESOURCES_PORT); + } + /** + * Delete. + * + * @param sourceID the source ID + * @param transId the trans id + * @param path the path + * @throws AAIException the AAI exception + */ + public static void Delete(String sourceID, String transId, String path, int port) throws AAIException { + String methodName = "Delete"; + String url=""; + transId += ":" + UUID.randomUUID().toString(); + + LoggingContext.save(); + LoggingContext.partnerName(sourceID); + LoggingContext.targetEntity(TARGET_NAME); + LoggingContext.requestId(transId); + LoggingContext.serviceName(methodName); + LoggingContext.targetServiceName(methodName); + + LOGGER.debug(methodName + " start"); + + initRestClient(); + String request = "{}"; + url = String.format(AAIConstants.AAI_LOCAL_REST, port) + path; + ClientResponse cres = client.resource(url) + .accept("application/json") + .header("X-TransactionId", transId) + .header("X-FromAppId", sourceID) + .header("Real-Time", "true") + .type("application/json") + .entity(request) + .delete(ClientResponse.class); + + if (cres.getStatus() == 404) { // resource not found + LOGGER.info("Resource does not exist...: " + cres.getStatus() + + ":" + cres.getEntity(String.class)); + LoggingContext.restore(); + } else if (cres.getStatus() == 200 || cres.getStatus() == 204){ + LOGGER.info("Resource " + url + " deleted"); + LoggingContext.restore(); + } else { + LOGGER.error("Deleting Resource failed: " + cres.getStatus() + + ":" + cres.getEntity(String.class)); + LoggingContext.restore(); + throw new AAIException("AAI_7116", "Error during DELETE"); + } + } + + /** + * Post. + * + * @param <T> the generic type + * @param t the t + * @param sourceID the source ID + * @param transId the trans id + * @param path the path + * @return the string + * @throws Exception the exception + */ + public static <T> String Post(T t, String sourceID, String transId, String path) throws Exception { + String methodName = "Post"; + String url=""; + transId += ":" + UUID.randomUUID().toString(); + + LoggingContext.save(); + LoggingContext.partnerName(sourceID); + LoggingContext.targetEntity(TARGET_NAME); + LoggingContext.requestId(transId); + LoggingContext.serviceName(methodName); + LoggingContext.targetServiceName(methodName); + + LOGGER.debug(methodName + " start"); + + try { + + initRestClient(); + + url = AAIConfig.get(AAIConstants.AAI_SERVER_URL) + path; + + ClientResponse cres = client.resource(url) + .accept("application/json") + .header("X-TransactionId", transId) + .header("X-FromAppId", sourceID) + .header("Real-Time", "true") + .type("application/json") + .entity(t) + .post(ClientResponse.class); + + int statuscode = cres.getStatus(); + if ( statuscode >= 200 && statuscode <= 299 ) { + LOGGER.debug(methodName + "REST api POST was successful!"); + return cres.getEntity(String.class); + } else { + throw new AAIException("AAI_7116", methodName +" with status="+statuscode+", url="+url + ", msg=" + cres.getEntity(String.class)); + } + + } catch (AAIException e) { + throw new AAIException("AAI_7116", methodName + " with url="+url+ ", Exception: " + e.toString()); + } catch (Exception e) + { + throw new AAIException("AAI_7116", methodName + " with url="+url+ ", Exception: " + e.toString()); + + } + finally { + LoggingContext.restore(); + } + } + + + /** + * Gets the single instance of RestController. + * + * @param <T> the generic type + * @param clazz the clazz + * @return single instance of RestController + * @throws IllegalAccessException the illegal access exception + * @throws InstantiationException the instantiation exception + */ + public static <T> T getInstance(Class<T> clazz) throws IllegalAccessException, InstantiationException + { + return clazz.newInstance(); + } + + /** + * Does resource exist. + * + * @param <T> the generic type + * @param resourcePath the resource path + * @param resourceClassName the resource class name + * @param fromAppId the from app id + * @param transId the trans id + * @return the t + */ + /* + * DoesResourceExist + * + * To check whether a resource exist or get a copy of the existing version of the resource + * + * Resourcepath: should contain the qualified resource path (including encoded unique key identifier value), + * resourceClassName: is the canonical name of the resource class name, + * fromAppId: + * transId: + * + * Will return null (if the resource doesn’t exist) (or) + * Will return the specified resource from the Graph. + * + * Example: + * LogicalLink llink = new LogicalLink(); + * String resourceClassName = llink.getClass().getCanonicalName(); + * llink = RestController.DoesResourceExist("network/logical-links/logical-link/" + <encoded-link-name>, resourceClassName, fromAppId, transId); + */ + public static <T> T DoesResourceExist(String resourcePath, String resourceClassName, String fromAppId, String transId) { + + try { + + RestObject<T> restObj = new RestObject<T>(); + @SuppressWarnings("unchecked") + T resourceObj = (T)getInstance(Class.forName(resourceClassName)); + restObj.set(resourceObj); + RestController.<T>Get(resourceObj, fromAppId, transId, resourcePath, restObj, false, AAIConstants.AAI_RESOURCES_PORT); + + resourceObj = restObj.get(); + if (resourceObj != null) + return resourceObj; + + } catch (AAIException e) { + + } catch (ClientHandlerException che) { + + }catch (Exception e) { + + } + + return null; + } + +} diff --git a/aai-core/src/main/java/org/onap/aai/util/StoreNotificationEvent.java b/aai-core/src/main/java/org/onap/aai/util/StoreNotificationEvent.java index 96dbfeb4..0ad33cb9 100644 --- a/aai-core/src/main/java/org/onap/aai/util/StoreNotificationEvent.java +++ b/aai-core/src/main/java/org/onap/aai/util/StoreNotificationEvent.java @@ -49,7 +49,11 @@ public class StoreNotificationEvent { * Instantiates a new store notification event. */ public StoreNotificationEvent(String transactionId, String sourceOfTruth) { - this.messageProducer = new AAIDmaapEventJMSProducer(); + this(new AAIDmaapEventJMSProducer(), transactionId, sourceOfTruth); + } + + public StoreNotificationEvent(AAIDmaapEventJMSProducer producer, String transactionId, String sourceOfTruth) { + this.messageProducer = producer; this.transactionId = transactionId; this.sourceOfTruth = sourceOfTruth; } @@ -64,7 +68,7 @@ public class StoreNotificationEvent { * @throws AAIException * the AAI exception */ - public void storeEvent(NotificationEvent.EventHeader eh, Object obj) throws AAIException { + public String storeEvent(NotificationEvent.EventHeader eh, Object obj) throws AAIException { if (obj == null) { throw new AAIException("AAI_7350"); @@ -123,6 +127,7 @@ public class StoreNotificationEvent { PojoUtils pu = new PojoUtils(); String entityJson = pu.getJsonFromObject(ne); sendToDmaapJmsQueue(entityJson); + return entityJson; } catch (Exception e) { throw new AAIException("AAI_7350", e); } @@ -214,7 +219,7 @@ public class StoreNotificationEvent { } } - public void storeEvent(Loader loader, Introspector eventHeader, Introspector obj) throws AAIException { + public String storeEvent(Loader loader, Introspector eventHeader, Introspector obj) throws AAIException { if (obj == null) { throw new AAIException("AAI_7350"); } @@ -271,6 +276,7 @@ public class StoreNotificationEvent { String entityJson = notificationEvent.marshal(false); sendToDmaapJmsQueue(entityJson); + return entityJson; } catch (JSONException e) { throw new AAIException("AAI_7350", e); } catch (AAIUnknownObjectException e) { diff --git a/aai-core/src/main/java/org/onap/aai/util/UniquePropertyCheck.java b/aai-core/src/main/java/org/onap/aai/util/UniquePropertyCheck.java index ae2cdaae..72e5380f 100644 --- a/aai-core/src/main/java/org/onap/aai/util/UniquePropertyCheck.java +++ b/aai-core/src/main/java/org/onap/aai/util/UniquePropertyCheck.java @@ -32,6 +32,8 @@ import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.onap.aai.exceptions.AAIException; +import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LoggingContext.StatusCode; import org.slf4j.MDC; import com.att.eelf.configuration.Configuration; @@ -39,8 +41,7 @@ import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanGraph; - - +import org.onap.aai.dbmap.AAIGraphConfig; public class UniquePropertyCheck { @@ -61,11 +62,24 @@ public class UniquePropertyCheck { props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "uniquePropertyCheck-logback.xml"); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); EELFLogger logger = EELFManager.getInstance().getLogger(UniquePropertyCheck.class.getSimpleName()); + + LoggingContext.init(); + LoggingContext.partnerName(FROMAPPID); + LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS); + LoggingContext.component(COMPONENT); + LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS); + LoggingContext.targetServiceName("main"); + LoggingContext.requestId(TRANSID); + LoggingContext.statusCode(StatusCode.COMPLETE); + LoggingContext.responseCode(LoggingContext.SUCCESS); + MDC.put("logFilenameAppender", UniquePropertyCheck.class.getSimpleName()); if( args == null || args.length != 1 ){ String msg = "usage: UniquePropertyCheck propertyName \n"; System.out.println(msg); + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logAndPrint(logger, msg ); System.exit(1); } @@ -75,26 +89,34 @@ public class UniquePropertyCheck { try { AAIConfig.init(); System.out.println(" ---- NOTE --- about to open graph (takes a little while)--------\n"); - TitanGraph tGraph = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG); + TitanGraph tGraph = TitanFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(UniquePropertyCheck.class.getSimpleName()).withGraphType("realtime").buildConfiguration()); if( tGraph == null ) { + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); logAndPrint(logger, " Error: Could not get TitanGraph "); System.exit(1); } graph = tGraph.newTransaction(); if( graph == null ){ + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); logAndPrint(logger, "could not get graph object in UniquePropertyCheck() \n"); System.exit(0); } } catch (AAIException e1) { String msg = "Threw Exception: [" + e1.toString() + "]"; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); logAndPrint(logger, msg); System.exit(0); } catch (Exception e2) { String msg = "Threw Exception: [" + e2.toString() + "]"; + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); logAndPrint(logger, msg); System.exit(0); } @@ -180,6 +202,8 @@ public class UniquePropertyCheck { } } catch( Exception e2 ){ + LoggingContext.statusCode(StatusCode.ERROR); + LoggingContext.responseCode(LoggingContext.DATA_ERROR); logAndPrint(logger, "Threw Exception: [" + e2.toString() + "]"); } finally { diff --git a/aai-core/src/main/java/org/onap/aai/util/swagger/GenerateSwagger.java b/aai-core/src/main/java/org/onap/aai/util/swagger/GenerateSwagger.java index a85b548e..9d36c0e8 100644 --- a/aai-core/src/main/java/org/onap/aai/util/swagger/GenerateSwagger.java +++ b/aai-core/src/main/java/org/onap/aai/util/swagger/GenerateSwagger.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.dataformat.yaml.snakeyaml.constructor.SafeConstruct import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; +import org.onap.aai.introspection.Version; import java.io.*; import java.util.*; @@ -37,7 +38,11 @@ public class GenerateSwagger { public static final String DEFAULT_WIKI = ""; public static final String DEFAULT_SCHEMA_DIR = "../aai-schema"; - public static final String CURRENT_VERSION = "v11"; + public static final String CURRENT_VERSION = Version. getLatest().toString(); + //if the program is run from aai-common, use this directory as default" + public static final String ALT_SCHEMA_DIR = "aai-schema"; + //used to check to see if program is run from aai-core + public static final String DEFAULT_RUN_DIR = "aai-core"; public static void main(String[] args) throws IOException, TemplateException { @@ -46,8 +51,14 @@ public class GenerateSwagger { String wikiLink = System.getProperty("aai.wiki.link"); if(schemaDir == null){ - System.out.println("Warning: Schema directory is not set so using default schema dir: " + DEFAULT_SCHEMA_DIR); - schemaDir = DEFAULT_SCHEMA_DIR; + if(System.getProperty("user.dir") != null && !System.getProperty("user.dir").contains(DEFAULT_RUN_DIR)) { + System.out.println("Warning: Schema directory is not set so using default schema dir: " + ALT_SCHEMA_DIR); + schemaDir = ALT_SCHEMA_DIR; + } + else { + System.out.println("Warning: Schema directory is not set so using default schema dir: " + DEFAULT_SCHEMA_DIR); + schemaDir = DEFAULT_SCHEMA_DIR; + } } if(versionToGenerate == null){ @@ -136,8 +147,13 @@ public class GenerateSwagger { Configuration configuration = new Configuration(); configuration.setClassForTemplateLoading(Api.class, "/"); - configuration.setDirectoryForTemplateLoading(new File("src/main/resources/")); - + String resourcePath = "src/main/resources"; + if(System.getProperty("user.dir") != null && !System.getProperty("user.dir").contains(DEFAULT_RUN_DIR)) { + configuration.setDirectoryForTemplateLoading(new File(DEFAULT_RUN_DIR + "/" + resourcePath)); + } + else { + configuration.setDirectoryForTemplateLoading(new File(resourcePath)); + } Template template = configuration.getTemplate("swagger.html.ftl"); String outputDirStr = schemaDir + "/src/main/resources/aai_swagger_html"; |