diff options
author | LaMont, William(wl2432) <wl2432@att.com> | 2020-02-28 16:48:41 -0500 |
---|---|---|
committer | LaMont, William(wl2432) <wl2432@att.com> | 2020-03-03 18:10:37 -0500 |
commit | 9b65b7126bc77c18c196ed2b6e625a5a864c98dc (patch) | |
tree | 7e0dd55538ad2af20c414369f628134e9f3191ae /src/main | |
parent | 30282f9e3f6ab0ed9599f5bb2e2c743cbd099089 (diff) |
support for v17 through v19
Issue-ID: AAI-2751
Change-Id: I75c9b2057065e1374f25e2b6b12de0a8003ac781
Signed-off-by: LaMont, William(wl2432) <wl2432@att.com>
Diffstat (limited to 'src/main')
82 files changed, 7665 insertions, 3196 deletions
diff --git a/src/main/java/org/onap/aai/GraphAdminApp.java b/src/main/java/org/onap/aai/GraphAdminApp.java index 3b4a377..1030f3a 100644 --- a/src/main/java/org/onap/aai/GraphAdminApp.java +++ b/src/main/java/org/onap/aai/GraphAdminApp.java @@ -20,16 +20,16 @@ package org.onap.aai; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.onap.aai.aailog.logs.AaiDebugLog; import org.onap.aai.config.PropertyPasswordConfiguration; import org.onap.aai.dbmap.AAIGraph; import java.util.Properties; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.logging.LogFormatTools; -import org.onap.aai.logging.LoggingContext; import org.onap.aai.nodes.NodeIngestor; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; @@ -54,17 +54,19 @@ import java.util.UUID; // This will add the ScheduledTask that was created in aai-common // Add more packages where you would need to scan for files @ComponentScan(basePackages = { - "org.onap.aai.tasks", - "org.onap.aai.config", - "org.onap.aai.service", - "org.onap.aai.setup", - "org.onap.aai.rest", - "org.onap.aai.web", - "org.onap.aai.interceptors", - "org.onap.aai.datasnapshot", - "org.onap.aai.datagrooming", - "org.onap.aai.dataexport", - "org.onap.aai.datacleanup" + "org.onap.aai.tasks", + "org.onap.aai.config", + "org.onap.aai.service", + "org.onap.aai.setup", + "org.onap.aai.aaf", + "org.onap.aai.rest", + "org.onap.aai.web", + "org.onap.aai.interceptors", + "org.onap.aai.datasnapshot", + "org.onap.aai.datagrooming", + "org.onap.aai.dataexport", + "org.onap.aai.datacleanup", + "org.onap.aai.aailog" }) @EnableAsync @EnableScheduling @@ -72,10 +74,16 @@ import java.util.UUID; public class GraphAdminApp { public static final String APP_NAME = "GraphAdmin"; - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(GraphAdminApp.class); + private static final Logger LOGGER = LoggerFactory.getLogger(GraphAdminApp.class); private static final String FROMAPPID = "AAI-GA"; private static final String TRANSID = UUID.randomUUID().toString(); + + private static AaiDebugLog debugLog = new AaiDebugLog(); + static { + debugLog.setupMDC(); + } + @Autowired private Environment env; @@ -86,17 +94,6 @@ public class GraphAdminApp { @PostConstruct private void initialize(){ loadDefaultProps(); - initializeLoggingContext(); - } - - private static void initializeLoggingContext() { - LoggingContext.save(); - LoggingContext.component("init"); - LoggingContext.partnerName("NA"); - LoggingContext.targetEntity(APP_NAME); - LoggingContext.requestId(UUID.randomUUID().toString()); - LoggingContext.serviceName(APP_NAME); - LoggingContext.targetServiceName("contextInitialized"); } @PreDestroy @@ -107,8 +104,8 @@ public class GraphAdminApp { public static void main(String[] args) throws Exception { loadDefaultProps(); + ErrorLogHelper.loadProperties(); - initializeLoggingContext(); Environment env =null; AAIConfig.init(); @@ -121,9 +118,6 @@ public class GraphAdminApp { catch(Exception ex){ AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(ex); - LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("Problems starting GraphAdminApp "+aai.getMessage()); ErrorLogHelper.logException(aai); ErrorLogHelper.logError(aai.getCode(), ex.getMessage() + ", resolve and restart GraphAdmin"); throw aai; @@ -141,7 +135,6 @@ public class GraphAdminApp { System.setProperty("org.onap.aai.graphadmin.started", "true"); LOGGER.info("GraphAdmin MicroService Started"); - LOGGER.error("GraphAdmin MicroService Started"); LOGGER.debug("GraphAdmin MicroService Started"); System.out.println("GraphAdmin Microservice Started"); } diff --git a/src/main/java/org/onap/aai/aailog/logs/AaiDBGraphadminMetricLog.java b/src/main/java/org/onap/aai/aailog/logs/AaiDBGraphadminMetricLog.java new file mode 100644 index 0000000..5b761cc --- /dev/null +++ b/src/main/java/org/onap/aai/aailog/logs/AaiDBGraphadminMetricLog.java @@ -0,0 +1,115 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.aailog.logs; + +import org.onap.aai.util.AAIConstants; +import org.onap.logging.filter.base.Constants; +import org.onap.logging.filter.base.MDCSetup; +import org.onap.logging.filter.base.ONAPComponents; +import org.onap.logging.ref.slf4j.ONAPLogConstants; +import org.slf4j.*; +import org.springframework.beans.factory.annotation.Value; + +import java.net.URI; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Optional; + +public class AaiDBGraphadminMetricLog extends MDCSetup { + + protected static final Logger logger = LoggerFactory.getLogger(AaiDBGraphadminMetricLog.class); + private final String partnerName; + private static final Marker INVOKE_RETURN = MarkerFactory.getMarker("INVOKE-RETURN"); + private static final String TARGET_ENTITY = ONAPComponents.AAI.toString() + ".DB"; + public AaiDBGraphadminMetricLog(String subcomponent) { + partnerName = getPartnerName(subcomponent); + } + + + protected String getTargetServiceName(Optional<URI> uri) { + return (getServiceName(uri)); + } + + protected String getServiceName(Optional<URI> uri) { + String serviceName = Constants.DefaultValues.UNKNOWN; + if (uri.isPresent()) { + serviceName = uri.get().getPath(); + if (serviceName != null && (!serviceName.isEmpty())) { + serviceName = serviceName.replaceAll(",", "\\\\,"); + } + } + return serviceName; + } + + + protected String getTargetEntity(Optional<URI> uri) { + return TARGET_ENTITY; + } + + protected String getPartnerName(@Value(AAIConstants.AAI_TRAVERSAL_MS) String subcomponent ) { + StringBuilder sb = new StringBuilder(ONAPComponents.AAI.toString()).append(subcomponent); + return (sb.toString()); + } + + public void pre(Optional<URI> uri) { + try { + setupMDC(uri); + setLogTimestamp(); + logger.info(ONAPLogConstants.Markers.INVOKE, "Invoke"); + } catch (Exception e) { + logger.warn("Error in AaiDBMetricLog pre", e.getMessage()); + } + } + + public void post() { + try { + setLogTimestamp(); + setElapsedTimeInvokeTimestamp(); + setResponseStatusCode(200); + setResponseDescription(200); + MDC.put(ONAPLogConstants.MDCs.RESPONSE_CODE, "200"); + logger.info(INVOKE_RETURN, "InvokeReturn"); + clearClientMDCs(); + } catch (Exception e) { + logger.warn("Error in AaiDBMetricLog post", e.getMessage()); + } + } + + protected void setupMDC(Optional<URI> uri) { + MDC.put("InvokeTimestamp", ZonedDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); + MDC.put("TargetServiceName", this.getTargetServiceName(uri)); + MDC.put("StatusCode", ONAPLogConstants.ResponseStatus.INPROGRESS.toString()); + this.setInvocationIdFromMDC(); + if (MDC.get("TargetEntity") == null) { + String targetEntity = this.getTargetEntity(uri); + if (targetEntity != null) { + MDC.put("TargetEntity", targetEntity); + } else { + MDC.put("TargetEntity", "Unknown-Target-Entity"); + } + } + if (MDC.get("ServiceName") == null) { + MDC.put("ServiceName", this.getServiceName(uri)); + } + this.setServerFQDN(); + } +} diff --git a/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java b/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java index 6d97177..352e96f 100644 --- a/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java +++ b/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java @@ -20,27 +20,95 @@ */ package org.onap.aai.config; -import org.springframework.context.ApplicationContextInitializer; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.core.env.ConfigurableEnvironment; -import org.springframework.core.env.EnumerablePropertySource; -import org.springframework.core.env.MapPropertySource; -import org.springframework.core.env.PropertySource; - +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.io.IOUtils; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.*; + public class PropertyPasswordConfiguration implements ApplicationContextInitializer<ConfigurableApplicationContext> { private static final Pattern decodePasswordPattern = Pattern.compile("password\\((.*?)\\)"); - private PasswordDecoder passwordDecoder = new JettyPasswordDecoder(); + private static final Logger logger = LoggerFactory.getLogger(PropertyPasswordConfiguration.class.getName()); @Override public void initialize(ConfigurableApplicationContext applicationContext) { ConfigurableEnvironment environment = applicationContext.getEnvironment(); + String certPath = environment.getProperty("server.certs.location"); + File passwordFile = null; + File passphrasesFile = null; + InputStream passwordStream = null; + InputStream passphrasesStream = null; + Map<String, Object> sslProps = new LinkedHashMap<>(); + + // Override the passwords from application.properties if we find AAF certman files + if (certPath != null) { + try { + passwordFile = new File(certPath + ".password"); + passwordStream = new FileInputStream(passwordFile); + + if (passwordStream != null) { + String keystorePassword = null; + + keystorePassword = IOUtils.toString(passwordStream); + if (keystorePassword != null) { + keystorePassword = keystorePassword.trim(); + } + sslProps.put("server.ssl.key-store-password", keystorePassword); + sslProps.put("schema.service.ssl.key-store-password", keystorePassword); + } else { + logger.info("Not using AAF Certman password file"); + } + } catch (IOException e) { + logger.warn("Not using AAF Certman password file, e=" + e.getMessage()); + } finally { + if (passwordStream != null) { + try { + passwordStream.close(); + } catch (Exception e) { + } + } + } + try { + passphrasesFile = new File(certPath + ".passphrases"); + passphrasesStream = new FileInputStream(passphrasesFile); + + if (passphrasesStream != null) { + String truststorePassword = null; + Properties passphrasesProps = new Properties(); + passphrasesProps.load(passphrasesStream); + truststorePassword = passphrasesProps.getProperty("cadi_truststore_password"); + if (truststorePassword != null) { + truststorePassword = truststorePassword.trim(); + } + sslProps.put("server.ssl.trust-store-password", truststorePassword); + sslProps.put("schema.service.ssl.trust-store-password", truststorePassword); + } else { + logger.info("Not using AAF Certman passphrases file"); + } + } catch (IOException e) { + logger.warn("Not using AAF Certman passphrases file, e=" + e.getMessage()); + } finally { + if (passphrasesStream != null) { + try { + passphrasesStream.close(); + } catch (Exception e) { + } + } + } + } for (PropertySource<?> propertySource : environment.getPropertySources()) { Map<String, Object> propertyOverrides = new LinkedHashMap<>(); decodePasswords(propertySource, propertyOverrides); @@ -48,6 +116,12 @@ public class PropertyPasswordConfiguration implements ApplicationContextInitiali PropertySource<?> decodedProperties = new MapPropertySource("decoded "+ propertySource.getName(), propertyOverrides); environment.getPropertySources().addBefore(propertySource.getName(), decodedProperties); } + + } + if (!sslProps.isEmpty()) { + logger.info("Using AAF Certman files"); + PropertySource<?> additionalProperties = new MapPropertySource("additionalProperties", sslProps); + environment.getPropertySources().addFirst(additionalProperties); } } @@ -67,7 +141,7 @@ public class PropertyPasswordConfiguration implements ApplicationContextInitiali private String decodePasswordsInString(String input) { if (input == null) { return null; - }; + } StringBuffer output = new StringBuffer(); Matcher matcher = decodePasswordPattern.matcher(input); while (matcher.find()) { diff --git a/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java index f4372c1..dff22a4 100644 --- a/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java +++ b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java @@ -28,26 +28,37 @@ import java.nio.file.attribute.FileTime; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; +import org.onap.logging.filter.base.ONAPComponents; +import org.onap.logging.ref.slf4j.ONAPLogConstants; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; @Component @PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties") public class DataCleanupTasks { - - private static final EELFLogger logger = EELFManager.getInstance().getLogger(DataCleanupTasks.class); + + @Autowired + private AaiScheduledTaskAuditLog auditLog; + + private static final Logger logger = LoggerFactory.getLogger(DataCleanupTasks.class); private final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd"); + /**The function archives/deletes files that end in .out (Ie. dataGrooming.201511111305.out) that sit in our log/data directory structure. logDir is the {project_home}/logs archiveDir is the ARCHIVE directory where the files will be stored after 5 days. @@ -56,8 +67,9 @@ public class DataCleanupTasks { */ @Scheduled(cron = "${datagroomingcleanup.cron}" ) public void dataGroomingCleanup() throws AAIException, Exception { + auditLog.logBefore("dataGroomingCleanup", ONAPComponents.AAI.toString() ); - logger.info("Started cron job dataGroomingCleanup @ " + simpleDateFormat.format(new Date())); + logger.debug("Started cron job dataGroomingCleanup @ " + simpleDateFormat.format(new Date())); try { String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs"; @@ -68,15 +80,15 @@ public class DataCleanupTasks { File archivepath = new File(archiveDir); File dataGroomingPath = new File(dataGroomingArcDir); - logger.info("The logDir is " + logDir); - logger.info("The dataGroomingDir is " + dataGroomingDir); - logger.info("The archiveDir is " + archiveDir ); - logger.info("The dataGroomingArcDir is " + dataGroomingArcDir ); + logger.debug("The logDir is " + logDir); + logger.debug("The dataGroomingDir is " + dataGroomingDir); + logger.debug("The archiveDir is " + archiveDir ); + logger.debug("The dataGroomingArcDir is " + dataGroomingArcDir ); boolean exists = directoryExists(logDir); - logger.info("Directory" + logDir + "exists: " + exists); + logger.debug("Directory" + logDir + "exists: " + exists); if(!exists) - logger.error("The directory" + logDir +"does not exists"); + logger.debug("The directory" + logDir +"does not exists"); Integer ageZip = AAIConfig.getInt("aai.datagrooming.agezip"); Integer ageDelete = AAIConfig.getInt("aai.datagrooming.agedelete"); @@ -91,9 +103,9 @@ public class DataCleanupTasks { continue; } if(listFile.isFile()){ - logger.info("The file name in dataGrooming: " +listFile.getName()); + logger.debug("The file name in dataGrooming: " +listFile.getName()); Date fileCreateDate = fileCreationMonthDate(listFile); - logger.info("The fileCreateDate in dataGrooming is " + fileCreateDate); + logger.debug("The fileCreateDate in dataGrooming is " + fileCreateDate); if( fileCreateDate.compareTo(newAgeZip) < 0) { archive(listFile,archiveDir,dataGroomingArcDir); } @@ -107,9 +119,9 @@ public class DataCleanupTasks { if(listFilesArchive != null) { for(File listFileArchive : listFilesArchive) { if(listFileArchive.isFile()) { - logger.info("The file name in ARCHIVE/dataGrooming: " +listFileArchive.getName()); + logger.debug("The file name in ARCHIVE/dataGrooming: " +listFileArchive.getName()); Date fileCreateDate = fileCreationMonthDate(listFileArchive); - logger.info("The fileCreateDate in ARCHIVE/dataGrooming is " + fileCreateDate); + logger.debug("The fileCreateDate in ARCHIVE/dataGrooming is " + fileCreateDate); if(fileCreateDate.compareTo(newAgeDelete) < 0) { delete(listFileArchive); } @@ -118,10 +130,11 @@ public class DataCleanupTasks { } } catch (Exception e) { - ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+e.toString()); - logger.info("AAI_4000", "Exception running cron job for DataCleanup"+e.toString()); - throw e; + ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e)); + logger.debug("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e)); } + logger.debug("Ended cron job dataGroomingCleanup @ " + simpleDateFormat.format(new Date())); + auditLog.logAfter(); } /** @@ -142,11 +155,11 @@ public class DataCleanupTasks { public Date getZipDate(Integer days, Date date) throws Exception{ Calendar cal = Calendar.getInstance(); - logger.info("The current date is " + date ); + logger.debug("The current date is " + date ); cal.setTime(date); cal.add(Calendar.DATE, -days); Date newAgeZip = cal.getTime(); - logger.info("The newAgeDate is " +newAgeZip); + logger.debug("The newAgeDate is " +newAgeZip); return newAgeZip; } @@ -170,9 +183,9 @@ public class DataCleanupTasks { */ public void archive(File file, String archiveDir, String afterArchiveDir) throws AAIException, Exception { - logger.info("Inside the archive folder"); + logger.debug("Inside the archive folder"); String filename = file.getName(); - logger.info("file name is " +filename); + logger.debug("file name is " +filename); File archivepath = new File(archiveDir); String zipFile = afterArchiveDir + AAIConstants.AAI_FILESEP + filename; @@ -180,13 +193,13 @@ public class DataCleanupTasks { File dataGroomingPath = new File(afterArchiveDir); boolean exists = directoryExists(archiveDir); - logger.info("Directory" + archiveDir + "exists: " + exists); + logger.debug("Directory" + archiveDir + "exists: " + exists); if(!exists) { - logger.error("The directory" + archiveDir +"does not exists so will create a new archive folder"); + logger.debug("The directory" + archiveDir +"does not exists so will create a new archive folder"); //Create an archive folder if does not exists boolean flag = dataGroomingPath.mkdirs(); if(!flag) - logger.error("Failed to create ARCHIVE folder"); + logger.debug("Failed to create ARCHIVE folder"); } try(FileOutputStream outputstream = new FileOutputStream(zipFile + ".gz"); ZipOutputStream zoutputstream = new ZipOutputStream(outputstream); @@ -202,13 +215,8 @@ public class DataCleanupTasks { zoutputstream.closeEntry(); //Delete the file after been added to archive folder delete(file); - logger.info("The file archived is " + file + " at " + afterArchiveDir ); + logger.debug("The file archived is " + file + " at " + afterArchiveDir ); } - catch (IOException e) { - ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup " + e.getStackTrace()); - logger.info("AAI_4000", "Exception running cron job for DataCleanup", e); - throw e; - } } /** @@ -217,10 +225,10 @@ public class DataCleanupTasks { */ public static void delete(File file) { - logger.info("Deleting the file " + file); + logger.debug("Deleting the file " + file); boolean deleteStatus = file.delete(); if(!deleteStatus){ - logger.error("Failed to delete the file" +file); + logger.debug("Failed to delete the file" +file); } } @@ -233,7 +241,7 @@ public class DataCleanupTasks { @Scheduled(cron = "${datasnapshotcleanup.cron}" ) public void dataSnapshotCleanup() throws AAIException, Exception { - logger.info("Started cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date())); + logger.info(ONAPLogConstants.Markers.ENTRY, "Started cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date())); try { String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs"; @@ -244,15 +252,15 @@ public class DataCleanupTasks { File archivepath = new File(archiveDir); File dataSnapshotPath = new File(dataSnapshotArcDir); - logger.info("The logDir is " + logDir); - logger.info("The dataSnapshotDir is " + dataSnapshotDir); - logger.info("The archiveDir is " + archiveDir ); - logger.info("The dataSnapshotArcDir is " + dataSnapshotArcDir ); + logger.debug("The logDir is " + logDir); + logger.debug("The dataSnapshotDir is " + dataSnapshotDir); + logger.debug("The archiveDir is " + archiveDir ); + logger.debug("The dataSnapshotArcDir is " + dataSnapshotArcDir ); boolean exists = directoryExists(logDir); - logger.info("Directory" + logDir + "exists: " + exists); + logger.debug("Directory" + logDir + "exists: " + exists); if(!exists) - logger.error("The directory" + logDir +"does not exists"); + logger.debug("The directory" + logDir +"does not exists"); Integer ageZipSnapshot = AAIConfig.getInt("aai.datasnapshot.agezip"); Integer ageDeleteSnapshot = AAIConfig.getInt("aai.datasnapshot.agedelete"); @@ -267,9 +275,9 @@ public class DataCleanupTasks { continue; } if(listFile.isFile()){ - logger.info("The file name in dataSnapshot: " +listFile.getName()); + logger.debug("The file name in dataSnapshot: " +listFile.getName()); Date fileCreateDate = fileCreationMonthDate(listFile); - logger.info("The fileCreateDate in dataSnapshot is " + fileCreateDate); + logger.debug("The fileCreateDate in dataSnapshot is " + fileCreateDate); if( fileCreateDate.compareTo(newAgeZip) < 0) { archive(listFile,archiveDir,dataSnapshotArcDir); } @@ -283,9 +291,9 @@ public class DataCleanupTasks { if(listFilesArchive != null) { for(File listFileArchive : listFilesArchive) { if(listFileArchive.isFile()) { - logger.info("The file name in ARCHIVE/dataSnapshot: " +listFileArchive.getName()); + logger.debug("The file name in ARCHIVE/dataSnapshot: " +listFileArchive.getName()); Date fileCreateDate = fileCreationMonthDate(listFileArchive); - logger.info("The fileCreateDate in ARCHIVE/dataSnapshot is " + fileCreateDate); + logger.debug("The fileCreateDate in ARCHIVE/dataSnapshot is " + fileCreateDate); if(fileCreateDate.compareTo(newAgeDelete) < 0) { delete(listFileArchive); } @@ -294,9 +302,9 @@ public class DataCleanupTasks { } } catch (Exception e) { - ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+e.toString()); - logger.info("AAI_4000", "Exception running cron job for DataCleanup"+e.toString()); - throw e; + ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e)); + logger.debug("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e)); } + logger.info(ONAPLogConstants.Markers.EXIT, "Ended cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date())); } } diff --git a/src/main/java/org/onap/aai/dataexport/DataExportTasks.java b/src/main/java/org/onap/aai/dataexport/DataExportTasks.java index 359e2ba..0131650 100644 --- a/src/main/java/org/onap/aai/dataexport/DataExportTasks.java +++ b/src/main/java/org/onap/aai/dataexport/DataExportTasks.java @@ -38,24 +38,27 @@ import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog; import org.onap.aai.dbgen.DynamicPayloadGenerator; import org.onap.aai.edges.EdgeIngestor; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.logging.ErrorLogHelper; -import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; +import org.onap.logging.filter.base.ONAPComponents; +import org.onap.logging.ref.slf4j.ONAPLogConstants; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import org.apache.commons.io.comparator.LastModifiedFileComparator; import org.apache.commons.io.filefilter.DirectoryFileFilter; import org.apache.commons.io.filefilter.FileFileFilter; @@ -69,8 +72,10 @@ import org.apache.commons.io.filefilter.RegexFileFilter; @Component @PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties") public class DataExportTasks { + + private AaiScheduledTaskAuditLog auditLog; - private static final EELFLogger LOGGER; + private static final Logger LOGGER; private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); private static final String GA_MS = "aai-graphadmin"; @@ -79,7 +84,7 @@ public class DataExportTasks { Properties props = System.getProperties(); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); - LOGGER = EELFManager.getInstance().getLogger(DataExportTasks.class); + LOGGER = LoggerFactory.getLogger(DataExportTasks.class); } private LoaderFactory loaderFactory; @@ -98,10 +103,12 @@ public class DataExportTasks { */ @Scheduled(cron = "${dataexporttask.cron}" ) public void export() { + try { exportTask(); } catch (Exception e) { + ErrorLogHelper.logError("AAI_8002", "Exception while running export "+ LogFormatTools.getStackTop(e)); } } /** @@ -110,27 +117,20 @@ public class DataExportTasks { * @throws AAIException, Exception */ public void exportTask() throws AAIException, Exception { - - LoggingContext.init(); - LoggingContext.requestId(UUID.randomUUID().toString()); - LoggingContext.partnerName("AAI"); - LoggingContext.targetEntity(GA_MS); - LoggingContext.component("exportTask"); - LoggingContext.serviceName(GA_MS); - LoggingContext.targetServiceName("exportTask"); - LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE); - + auditLog = new AaiScheduledTaskAuditLog(); + auditLog.logBefore("dataExportTask", ONAPComponents.AAI.toString()); + LOGGER.info("Started exportTask: " + dateFormat.format(new Date())); if (AAIConfig.get("aai.dataexport.enable").equalsIgnoreCase("false")) { - LOGGER.info("Data Export is not enabled"); + LOGGER.debug("Data Export is not enabled"); return; } // Check if the process was started via command line if (isDataExportRunning()) { - LOGGER.info("There is a dataExport process already running"); + LOGGER.debug("There is a dataExport process already running"); return; } - LOGGER.info("Started exportTask: " + dateFormat.format(new Date())); + LOGGER.debug("Started exportTask: " + dateFormat.format(new Date())); String enableSchemaValidation = AAIConfig.get("aai.dataexport.enable.schema.validation", "false"); String outputLocation = AAIConstants.AAI_HOME_BUNDLECONFIG + AAIConfig.get("aai.dataexport.output.location"); @@ -187,20 +187,20 @@ public class DataExportTasks { String[] paramsArray = paramsList.toArray(new String[0]); try { DynamicPayloadGenerator.run(loaderFactory, edgeIngestor, schemaVersions, paramsArray, false); - LOGGER.info("DynamicPaylodGenerator completed"); + LOGGER.debug("DynamicPaylodGenerator completed"); // tar/gzip payload files String[] command = new String[1]; command[0] = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "bin" + AAIConstants.AAI_FILESEP + "dynamicPayloadArchive.sh"; runScript(command); } catch (Exception e) { - ErrorLogHelper.logError("AAI_8003", e.getMessage()); - LOGGER.info("Exception running dataExport task " + e.getMessage()); - throw e; + ErrorLogHelper.logError("AAI_8003", LogFormatTools.getStackTop(e)); + LOGGER.debug("Exception running dataExport task " + LogFormatTools.getStackTop(e)); } finally { - LOGGER.info("Completed dataExport task" ); - LoggingContext.clear(); + LOGGER.debug("Ended exportTask: " + dateFormat.format(new Date())); } + LOGGER.info("Ended exportTask: " + dateFormat.format(new Date())); + auditLog.logAfter(); } /** @@ -223,10 +223,10 @@ public class DataExportTasks { } int exitVal = process.waitFor(); - LOGGER.info("Check if dataExport is running returned: " + exitVal); + LOGGER.debug("Check if dataExport is running returned: " + exitVal); } catch (Exception e) { - ErrorLogHelper.logError("AAI_8002", "Exception while running the check to see if dataExport is running "+ e.getMessage()); - LOGGER.info("Exception while running the check to see if dataExport is running "+ e.getMessage()); + ErrorLogHelper.logError("AAI_8002", "Exception while running the check to see if dataExport is running "+ LogFormatTools.getStackTop(e)); + LOGGER.debug("Exception while running the check to see if dataExport is running "+ LogFormatTools.getStackTop(e)); } if(count > 0){ @@ -249,7 +249,7 @@ public class DataExportTasks { File[] allFilesArr = targetDirFile.listFiles((FileFilter) FileFileFilter.FILE); if ( allFilesArr == null || allFilesArr.length == 0 ) { ErrorLogHelper.logError("AAI_8001", "Unable to find data snapshots at " + targetDir); - LOGGER.info ("Unable to find data snapshots at " + targetDir); + LOGGER.debug("Unable to find data snapshots at " + targetDir); return (snapshot); } if ( allFilesArr.length > 1 ) { @@ -287,7 +287,7 @@ public class DataExportTasks { if ( allFilesArr == null || allFilesArr.length == 0 ) { ErrorLogHelper.logError("AAI_8001", "Unable to find data snapshots at " + targetDir); - LOGGER.info ("Unable to find data snapshots at " + targetDir); + LOGGER.debug("Unable to find data snapshots at " + targetDir); return (null); } @@ -329,7 +329,7 @@ public class DataExportTasks { snapshotName = snapshotName.substring(0,lastDot); } else { - LOGGER.info ("Invalid snapshot file name format " + snapshotName); + LOGGER.debug("Invalid snapshot file name format " + snapshotName); return null; } } @@ -348,7 +348,7 @@ public class DataExportTasks { File[] allFilesArr = targetDirFile.listFiles((FileFilter)DirectoryFileFilter.DIRECTORY); if ( allFilesArr == null || allFilesArr.length == 0 ) { - LOGGER.info ("No payload files found at " + targetDirFile.getPath()); + LOGGER.debug("No payload files found at " + targetDirFile.getPath()); return; } for ( File f : allFilesArr ) { @@ -357,7 +357,7 @@ public class DataExportTasks { } catch (IOException e) { - LOGGER.info ("Unable to delete directory " + f.getAbsolutePath() + " " + e.getMessage()); + LOGGER.debug("Unable to delete directory " + f.getAbsolutePath() + " " + e.getMessage()); } } @@ -372,10 +372,10 @@ public class DataExportTasks { try { process = new ProcessBuilder().command(script).start(); int exitVal = process.waitFor(); - LOGGER.info("dynamicPayloadArchive.sh returned: " + exitVal); + LOGGER.debug("dynamicPayloadArchive.sh returned: " + exitVal); } catch (Exception e) { - ErrorLogHelper.logError("AAI_8002", "Exception while running dynamicPayloadArchive.sh "+ e.getMessage()); - LOGGER.info("Exception while running dynamicPayloadArchive.sh" + e.getMessage()); + ErrorLogHelper.logError("AAI_8002", "Exception while running dynamicPayloadArchive.sh "+ LogFormatTools.getStackTop(e)); + LOGGER.debug("Exception while running dynamicPayloadArchive.sh" + LogFormatTools.getStackTop(e)); } } diff --git a/src/main/java/org/onap/aai/datagrooming/DataGrooming.java b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java index 167ec21..29a588b 100644 --- a/src/main/java/org/onap/aai/datagrooming/DataGrooming.java +++ b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java @@ -49,6 +49,7 @@ import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.onap.aai.GraphAdminApp; import org.onap.aai.config.PropertyPasswordConfiguration; import org.onap.aai.util.GraphAdminConstants; +import org.onap.logging.ref.slf4j.ONAPLogConstants; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Introspector; @@ -58,17 +59,15 @@ import org.onap.aai.introspection.ModelType; import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.logging.LogFormatTools; -import org.onap.aai.logging.LoggingContext; import org.onap.aai.edges.enums.AAIDirection; import org.onap.aai.edges.enums.EdgeProperty; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.setup.SchemaVersion; import org.onap.aai.util.*; -import org.onap.aai.logging.LoggingContext.StatusCode; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -78,8 +77,8 @@ import org.springframework.context.annotation.AnnotationConfigApplicationContext public class DataGrooming { - private static EELFLogger LOGGER = EELFManager.getInstance().getLogger(DataGrooming.class); - + private static Logger LOGGER = LoggerFactory.getLogger(DataGrooming.class); + private static boolean historyEnabled; private static final String FROMAPPID = "AAI-DB"; private static final String TRANSID = UUID.randomUUID().toString(); private int dupeGrpsDeleted = 0; @@ -107,6 +106,15 @@ public class DataGrooming { public void execute(String[] args){ String ver = "version"; // Placeholder + + // Note - if execute() is called from DataGroomingTasks, ie. from the cron, + // then 'historyEnabled' will default to false. In History, dataGrooming + // is never called via the cron, but this check will prevent it from + // being called from the command line. + if( historyEnabled ) { + LOGGER.debug("ERROR: DataGrooming may not be used when history.enabled=true. "); + return; + } // A value of 0 means that we will not have a time-window -- we will look // at all nodes of the passed-in nodeType. @@ -155,19 +163,19 @@ public class DataGrooming { jCommander.setProgramName(DataGrooming.class.getSimpleName()); //Print Defaults - LOGGER.info("EdgesOnlyFlag is [" + cArgs.edgesOnlyFlag + "]"); - LOGGER.info("DoAutoFix is [" + cArgs.doAutoFix + "]"); - LOGGER.info("skipHostCheck is [" + cArgs.skipHostCheck + "]"); - LOGGER.info("dontFixOrphansFlag is [" + cArgs.dontFixOrphansFlag + "]"); - LOGGER.info("dupeCheckOff is [" + cArgs.dupeCheckOff + "]"); - LOGGER.info("dupeFixOn is [" + cArgs.dupeFixOn + "]"); - LOGGER.info("ghost2CheckOff is [" + cArgs.ghost2CheckOff + "]"); - LOGGER.info("ghost2FixOn is [" + cArgs.ghost2FixOn + "]"); - LOGGER.info("neverUseCache is [" + cArgs.neverUseCache + "]"); - LOGGER.info("singleNodeType is [" + cArgs.singleNodeType + "]"); - LOGGER.info("skipEdgeChecks is [" + cArgs.skipEdgeCheckFlag + "]"); - LOGGER.info("skipIndexUpdateFix is [" + cArgs.skipIndexUpdateFix + "]"); - LOGGER.info("maxFix is [" + cArgs.maxRecordsToFix + "]"); + LOGGER.debug("EdgesOnlyFlag is [" + cArgs.edgesOnlyFlag + "]"); + LOGGER.debug("DoAutoFix is [" + cArgs.doAutoFix + "]"); + LOGGER.debug("skipHostCheck is [" + cArgs.skipHostCheck + "]"); + LOGGER.debug("dontFixOrphansFlag is [" + cArgs.dontFixOrphansFlag + "]"); + LOGGER.debug("dupeCheckOff is [" + cArgs.dupeCheckOff + "]"); + LOGGER.debug("dupeFixOn is [" + cArgs.dupeFixOn + "]"); + LOGGER.debug("ghost2CheckOff is [" + cArgs.ghost2CheckOff + "]"); + LOGGER.debug("ghost2FixOn is [" + cArgs.ghost2FixOn + "]"); + LOGGER.debug("neverUseCache is [" + cArgs.neverUseCache + "]"); + LOGGER.debug("singleNodeType is [" + cArgs.singleNodeType + "]"); + LOGGER.debug("skipEdgeChecks is [" + cArgs.skipEdgeCheckFlag + "]"); + LOGGER.debug("skipIndexUpdateFix is [" + cArgs.skipIndexUpdateFix + "]"); + LOGGER.debug("maxFix is [" + cArgs.maxRecordsToFix + "]"); String windowTag = "FULL"; @@ -181,9 +189,8 @@ public class DataGrooming { loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion()); } catch (Exception ex){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); - LOGGER.error("ERROR - Could not create loader " + LogFormatTools.getStackTop(ex)); + AAIException ae = new AAIException("AAI_6128", ex , "ERROR - Could not create loader "+args); + ErrorLogHelper.logException(ae); AAISystemExitUtil.systemExitCloseAAIGraph(1); } @@ -191,7 +198,7 @@ public class DataGrooming { if (!cArgs.prevFileName.equals("")) { // They are trying to fix some data based on a data in a // previous file. - LOGGER.info(" Call doTheGrooming() with a previous fileName [" + LOGGER.debug(" Call doTheGrooming() with a previous fileName [" + prevFileName + "] for cleanup. "); Boolean finalShutdownFlag = true; Boolean cacheDbOkFlag = false; @@ -201,7 +208,6 @@ public class DataGrooming { finalShutdownFlag, cacheDbOkFlag, cArgs.skipEdgeCheckFlag, cArgs.timeWindowMinutes, cArgs.singleNodeType, cArgs.skipIndexUpdateFix ); - } else if (cArgs.doAutoFix) { // They want us to run the processing twice -- first to look for // delete candidates, then after @@ -209,8 +215,8 @@ public class DataGrooming { // that were found by the first run. // Note: we will produce a separate output file for each of the // two runs. - LOGGER.info(" Doing an auto-fix call to Grooming. "); - LOGGER.info(" First, Call doTheGrooming() to look at what's out there. "); + LOGGER.debug(" Doing an auto-fix call to Grooming. "); + LOGGER.debug(" First, Call doTheGrooming() to look at what's out there. "); Boolean finalShutdownFlag = false; Boolean cacheDbOkFlag = true; int fixCandCount = doTheGrooming("", cArgs.edgesOnlyFlag, @@ -220,23 +226,23 @@ public class DataGrooming { cArgs.skipEdgeCheckFlag, cArgs.timeWindowMinutes, cArgs.singleNodeType, cArgs.skipIndexUpdateFix ); if (fixCandCount == 0) { - LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. "); + LOGGER.debug(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. "); } else { // We'll sleep a little and then run a fix-pass based on the // first-run's output file. try { - LOGGER.info("About to sleep for " + cArgs.sleepMinutes + LOGGER.debug("About to sleep for " + cArgs.sleepMinutes + " minutes."); int sleepMsec = cArgs.sleepMinutes * 60 * 1000; Thread.sleep(sleepMsec); } catch (InterruptedException ie) { - LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< "); + LOGGER.debug(" >>> Sleep Thread has been Interrupted <<< "); AAISystemExitUtil.systemExitCloseAAIGraph(0); } dteStr = fd.getDateTime(); String secondGroomOutFileName = "dataGrooming." + windowTag + "." + dteStr + ".out"; - LOGGER.info(" Now, call doTheGrooming() a second time and pass in the name of the file " + LOGGER.debug(" Now, call doTheGrooming() a second time and pass in the name of the file " + "generated by the first pass for fixing: [" + groomOutFileName + "]"); finalShutdownFlag = true; @@ -253,7 +259,7 @@ public class DataGrooming { // Do the grooming - plain vanilla (no fix-it-file, no // auto-fixing) Boolean finalShutdownFlag = true; - LOGGER.info(" Call doTheGrooming() "); + LOGGER.debug(" Call doTheGrooming() "); Boolean cacheDbOkFlag = true; if( cArgs.neverUseCache ){ // They have forbidden us from using a cached db connection. @@ -267,11 +273,9 @@ public class DataGrooming { cArgs.singleNodeType, cArgs.skipIndexUpdateFix ); } } catch (Exception ex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("Exception while grooming data " + LogFormatTools.getStackTop(ex)); + LOGGER.debug("Exception while grooming data " + LogFormatTools.getStackTop(ex)); } - LOGGER.info(" Done! "); + LOGGER.debug(" Done! "); AAISystemExitUtil.systemExitCloseAAIGraph(0); } @@ -285,16 +289,6 @@ public class DataGrooming { // Set the logging file properties to be used by EELFManager System.setProperty("aai.service.name", DataGrooming.class.getSimpleName()); - LoggingContext.init(); - LoggingContext.partnerName(FROMAPPID); - LoggingContext.serviceName(GraphAdminApp.APP_NAME); - LoggingContext.component("dataGrooming"); - LoggingContext.targetEntity(GraphAdminApp.APP_NAME); - LoggingContext.targetServiceName("main"); - LoggingContext.requestId(TRANSID); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - Properties props = System.getProperties(); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); @@ -312,11 +306,12 @@ public class DataGrooming { } catch (Exception e) { AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); - LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } + + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); + LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions"); DataGrooming dataGrooming = new DataGrooming(loaderFactory, schemaVersions); @@ -349,7 +344,7 @@ public class DataGrooming { Boolean skipEdgeCheckFlag, int timeWindowMinutes, String singleNodeType, Boolean skipIndexUpdateFix ) { - LOGGER.debug(" Entering doTheGrooming \n"); + LOGGER.debug(" Entering doTheGrooming "); int cleanupCandidateCount = 0; long windowStartTime = 0; // Translation of the window into a starting timestamp @@ -388,8 +383,6 @@ public class DataGrooming { } if (deleteCandidateList.size() > maxRecordsToFix) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(" >> WARNING >> Delete candidate list size (" + deleteCandidateList.size() + ") is too big. The maxFix we are using is: " @@ -410,11 +403,11 @@ public class DataGrooming { throw new AAIException("AAI_6124", emsg); } - LOGGER.info(" Will write to " + fullOutputFileName ); + LOGGER.debug(" Will write to " + fullOutputFileName ); bw = new BufferedWriter(new FileWriter(groomOutFile.getAbsoluteFile())); ErrorLogHelper.loadProperties(); - LOGGER.info(" ---- NOTE --- about to open graph (takes a little while)--------\n"); + LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while)-------- "); if( cacheDbOkFlag ){ // Since we're just reading (not deleting/fixing anything), we can use @@ -478,10 +471,10 @@ public class DataGrooming { Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet(); String ntList = ""; - LOGGER.info(" Starting DataGrooming Processing "); + LOGGER.debug(" Starting DataGrooming Processing "); if (edgesOnlyFlag) { - LOGGER.info(" NOTE >> Skipping Node processing as requested. Will only process Edges. << "); + LOGGER.debug(" NOTE >> Skipping Node processing as requested. Will only process Edges. << "); } else { for (Entry<String, Introspector> entry : entrySet) { @@ -632,16 +625,14 @@ public class DataGrooming { } } catch (Exception e) { okFlag = false; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("ERROR trying to delete delete Candidate VID = " + thisVid + " " + LogFormatTools.getStackTop(e)); + LOGGER.debug("ERROR trying to delete delete Candidate VID = " + thisVid + " " + LogFormatTools.getStackTop(e)); } if (okFlag){ if( updateOnlyFlag ) { - LOGGER.info(" Updated Indexes for Delete Candidate VID = " + thisVid); + LOGGER.debug(" Updated Indexes for Delete Candidate VID = " + thisVid); } else { - LOGGER.info(" DELETED Delete Candidate VID = " + thisVid); + LOGGER.debug(" DELETED Delete Candidate VID = " + thisVid); } } } else { @@ -653,7 +644,7 @@ public class DataGrooming { if( ob == null ){ // Group this with missing-node-type guys - which // we will delete more readily than orphans. - LOGGER.info(" >> Encountered a missingAaiNodeType while looking for the parent of a [" + nType + "] node."); + LOGGER.debug(" >> Encountered a missingAaiNodeType while looking for the parent of a [" + nType + "] node."); missingAaiNtNodeHash.put(thisVid, thisVtx); } else { @@ -665,7 +656,7 @@ public class DataGrooming { String checkDummyUid = thisVid + "dummy"; if( auid.equals(checkDummyUid) ){ // Group this with missing-node-type guys. - LOGGER.info(" >> Encountered a missingAaiNodeType mid-fix-node while looking for the parent of a [" + nType + "] node."); + LOGGER.debug(" >> Encountered a missingAaiNodeType mid-fix-node while looking for the parent of a [" + nType + "] node."); missingAaiNtNodeHash.put(thisVid, thisVtx); } else { @@ -742,16 +733,14 @@ public class DataGrooming { } } catch (Exception e) { okFlag = false; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("ERROR trying to delete phantom VID = " + thisVid + " " + LogFormatTools.getStackTop(e)); + LOGGER.debug("ERROR trying to delete phantom VID = " + thisVid + " " + LogFormatTools.getStackTop(e)); } if (okFlag){ if( updateOnlyFlag ) { - LOGGER.info(" Updated Indexes for Delete Candidate VID = " + thisVid); + LOGGER.debug(" Updated Indexes for Delete Candidate VID = " + thisVid); } else { - LOGGER.info(" DELETED VID = " + thisVid); + LOGGER.debug(" DELETED VID = " + thisVid); } } } else { @@ -760,7 +749,7 @@ public class DataGrooming { } else if( (secondGetList.size() > 1) && depNodeOk && !dupeCheckOff ){ // Found some DUPLICATES - need to process them - LOGGER.info(" - now check Dupes for this guy - "); + LOGGER.debug(" - now check Dupes for this guy - "); List<String> tmpDupeGroups = checkAndProcessDupes( TRANSID, FROMAPPID, g, source1, version, nType, secondGetList, dupeFixOn, @@ -769,28 +758,22 @@ public class DataGrooming { while (dIter.hasNext()) { // Add in any newly found dupes to our running list String tmpGrp = dIter.next(); - LOGGER.info("Found set of dupes: [" + tmpGrp + "]"); + LOGGER.debug("Found set of dupes: [" + tmpGrp + "]"); dupeGroups.add(tmpGrp); } } } catch (AAIException e1) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1); errArr.add(e1.getErrorObject().toString()); } catch (Exception e2) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(" For nodeType = " + nType + " Caught exception", e2); errArr.add(e2.getMessage()); } }// try block to enclose looping over each single vertex catch (Exception exx) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING from inside the while-verts-loop ", exx); } @@ -809,7 +792,7 @@ public class DataGrooming { Iterator<ArrayList<Vertex>> dsItr = nonDependentDupeSets.iterator(); while( dsItr.hasNext() ){ ArrayList<Vertex> dupeList = dsItr.next(); - LOGGER.info(" - now check Dupes for some non-dependent guys - "); + LOGGER.debug(" - now check Dupes for some non-dependent guys - "); List<String> tmpDupeGroups = checkAndProcessDupes( TRANSID, FROMAPPID, g, source1, version, nType, dupeList, dupeFixOn, @@ -818,7 +801,7 @@ public class DataGrooming { while (dIter.hasNext()) { // Add in any newly found dupes to our running list String tmpGrp = dIter.next(); - LOGGER.info("Found set of dupes: [" + tmpGrp + "]"); + LOGGER.debug("Found set of dupes: [" + tmpGrp + "]"); dupeGroups.add(tmpGrp); } } @@ -826,7 +809,7 @@ public class DataGrooming { }// end of extra dupe check for non-dependent nodes thisNtDeleteCount = 0; // Reset for the next pass - LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total (in window) overall. " ); + LOGGER.debug( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total (in window) overall. " ); }// While-loop for each node type @@ -843,7 +826,7 @@ public class DataGrooming { // -------------------------------------------------------------- // To do some strange checking - we need a second graph object - LOGGER.debug(" ---- NOTE --- about to open a SECOND graph (takes a little while)--------\n"); + LOGGER.debug(" ---- NOTE --- about to open a SECOND graph (takes a little while)-------- "); // Note - graph2 just reads - but we want it to use a fresh connection to // the database, so we are NOT using the CACHED DB CONFIG here. @@ -854,7 +837,7 @@ public class DataGrooming { String emsg = "null graph2 object in DataGrooming\n"; throw new AAIException("AAI_6101", emsg); } else { - LOGGER.debug("Got the graph2 object... \n"); + LOGGER.debug("Got the graph2 object... "); } g2 = graph2.newTransaction(); if (g2 == null) { @@ -873,7 +856,7 @@ public class DataGrooming { int counter = 0; int lastShown = 0; Iterator<Vertex> vItor2 = vertList.iterator(); - LOGGER.info(" Checking for bad edges --- "); + LOGGER.debug(" Checking for bad edges --- "); while (vItor2.hasNext()) { Vertex v = null; @@ -881,8 +864,6 @@ public class DataGrooming { try { v = vItor2.next(); } catch (Exception vex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 "); continue; } @@ -892,14 +873,12 @@ public class DataGrooming { try { thisVertId = v.id().toString(); } catch (Exception ev) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list. "); continue; } if (ghostNodeHash.containsKey(thisVertId)) { // We already know that this is a phantom node, so don't bother checking it - LOGGER.info(" >> Skipping edge check for edges from vertexId = " + LOGGER.debug(" >> Skipping edge check for edges from vertexId = " + thisVertId + ", since that guy is a Phantom Node"); continue; @@ -920,7 +899,7 @@ public class DataGrooming { if (counter == lastShown + 250) { lastShown = counter; - LOGGER.info("... Checking edges for vertex # " + LOGGER.debug("... Checking edges for vertex # " + counter); } Iterator<Edge> eItor = v.edges(Direction.BOTH); @@ -931,8 +910,6 @@ public class DataGrooming { try { e = eItor.next(); } catch (Exception iex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex); continue; } @@ -940,8 +917,6 @@ public class DataGrooming { try { vIn = e.inVertex(); } catch (Exception err) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err); } String vNtI = ""; @@ -967,8 +942,6 @@ public class DataGrooming { if( ! ghost2CheckOff ){ Vertex connectedVert = g2.traversal().V(vIdLong).next(); if( connectedVert == null ) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong); cantGetUsingVid = true; @@ -980,8 +953,6 @@ public class DataGrooming { ghost2 = g.traversal().V(vIdLong).next(); } catch( Exception ex){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex); } if( ghost2 != null ){ @@ -991,8 +962,6 @@ public class DataGrooming { }// end of the ghost2 checking } catch (Exception err) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err); } } @@ -1017,13 +986,11 @@ public class DataGrooming { deleteCount++; } catch (Exception e1) { okFlag = false; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = " + vIdI, e1); } if (okFlag) { - LOGGER.info(" DELETED vertex from bad edge = " + LOGGER.debug(" DELETED vertex from bad edge = " + vIdI); } } else { @@ -1038,13 +1005,11 @@ public class DataGrooming { // that this edge has already been // removed okFlag = false; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when trying to delete edge = " + thisEid); } if (okFlag) { - LOGGER.info(" DELETED edge = " + thisEid); + LOGGER.debug(" DELETED edge = " + thisEid); } } } else { @@ -1059,8 +1024,6 @@ public class DataGrooming { try { vOut = e.outVertex(); } catch (Exception err) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get edge's Out-vertex "); } String vNtO = ""; @@ -1087,14 +1050,12 @@ public class DataGrooming { Vertex connectedVert = g2.traversal().V(vIdLong).next(); if( connectedVert == null ) { cantGetUsingVid = true; - LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong); + LOGGER.debug( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong); // If we can get this ghost with the other graph-object, then get it -- it's still a ghost try { ghost2 = g.traversal().V(vIdLong).next(); } catch( Exception ex){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex); } if( ghost2 != null ){ @@ -1103,8 +1064,6 @@ public class DataGrooming { } } } catch (Exception err) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err); } } @@ -1128,13 +1087,11 @@ public class DataGrooming { deleteCount++; } catch (Exception e1) { okFlag = false; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = " + vIdO, e1); } if (okFlag) { - LOGGER.info(" DELETED vertex from bad edge = " + LOGGER.debug(" DELETED vertex from bad edge = " + vIdO); } } else { @@ -1149,13 +1106,11 @@ public class DataGrooming { // that this edge has already been // removed okFlag = false; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING when trying to delete edge = " + thisEid, ex); } if (okFlag) { - LOGGER.info(" DELETED edge = " + thisEid); + LOGGER.debug(" DELETED edge = " + thisEid); } } } else { @@ -1168,12 +1123,10 @@ public class DataGrooming { } }// End of while-edges-loop } catch (Exception exx) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn("WARNING from in the while-verts-loop ", exx); } }// End of while-vertices-loop (the edge-checking) - LOGGER.info(" Done checking for bad edges --- "); + LOGGER.debug(" Done checking for bad edges --- "); } // end of -- if we're not skipping the edge-checking @@ -1272,9 +1225,7 @@ public class DataGrooming { bw.write(info + "\n"); } } catch (Exception dex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("error trying to print detail info for a ghost-node: " + LogFormatTools.getStackTop(dex)); + LOGGER.debug("error trying to print detail info for a ghost-node: " + LogFormatTools.getStackTop(dex)); } } @@ -1296,9 +1247,7 @@ public class DataGrooming { bw.write(info + "\n"); } } catch (Exception dex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("error trying to print detail info for a node missing its aai-node-type " + LogFormatTools.getStackTop(dex)); + LOGGER.debug("error trying to print detail info for a node missing its aai-node-type " + LogFormatTools.getStackTop(dex)); } } @@ -1320,9 +1269,7 @@ public class DataGrooming { bw.write(info + "\n"); } } catch (Exception dex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("error trying to print detail info for a node with a bad aai-uri " + LogFormatTools.getStackTop(dex)); + LOGGER.debug("error trying to print detail info for a node with a bad aai-uri " + LogFormatTools.getStackTop(dex)); } } @@ -1344,9 +1291,7 @@ public class DataGrooming { bw.write(info + "\n"); } } catch (Exception dex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("error trying to print detail info for a node with bad index " + LogFormatTools.getStackTop(dex)); + LOGGER.debug("error trying to print detail info for a node with bad index " + LogFormatTools.getStackTop(dex)); } } @@ -1368,9 +1313,7 @@ public class DataGrooming { bw.write(info + "\n"); } } catch (Exception dex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge " + LogFormatTools.getStackTop(dex)); + LOGGER.debug("error trying to print detail info for a Orphan Node /missing dependent edge " + LogFormatTools.getStackTop(dex)); } } @@ -1390,9 +1333,7 @@ public class DataGrooming { + propKey.value() + "]\n"); } } catch (Exception pex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("error trying to print empty/bad vertex data: " + LogFormatTools.getStackTop(pex)); + LOGGER.debug("error trying to print empty/bad vertex data: " + LogFormatTools.getStackTop(pex)); } } @@ -1471,9 +1412,7 @@ public class DataGrooming { }// else last entry }// for each vertex in a group } catch (Exception dex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("error trying to print duplicate vertex data " + LogFormatTools.getStackTop(dex)); + LOGGER.debug("error trying to print duplicate vertex data " + LogFormatTools.getStackTop(dex)); } }// while - work on each group of dupes @@ -1493,8 +1432,8 @@ public class DataGrooming { bw.close(); - LOGGER.info("\n ------------- Done doing all the checks ------------ "); - LOGGER.info("Output will be written to " + fullOutputFileName); + LOGGER.debug(" ------------- Done doing all the checks ------------ "); + LOGGER.debug("Output will be written to " + fullOutputFileName); if (cleanupCandidateCount > 0 || badUriNodeCount > 0 || badIndexNodeCount > 0) { // Technically, this is not an error -- but we're throwing this @@ -1504,14 +1443,10 @@ public class DataGrooming { + "] and investigate delete candidates. "); } } catch (AAIException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("Caught AAIException while grooming data"); + LOGGER.debug("Caught AAIException while grooming data"); ErrorLogHelper.logException(e); } catch (Exception ex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("Caught exception while grooming data"); + LOGGER.debug("Caught exception while grooming data"); ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming"); } finally { @@ -1519,34 +1454,26 @@ public class DataGrooming { try { bw.close(); } catch (IOException iox) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); - LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox); + LOGGER.warn("Got an IOException trying to close bufferedWriter() ", iox); } } if (executeFinalCommit) { // If we were holding off on commits till the end - then now is the time. if( g == null ){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error(" >>>> ERROR <<<< Could not commit changes. graph was null when we wanted to commit."); + LOGGER.debug(" >>>> ERROR <<<< Could not commit changes. graph was null when we wanted to commit."); } else if( !g.tx().isOpen() ){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error(" >>>> ERROR <<<< Could not commit changes. Transaction was not open when we wanted to commit."); + LOGGER.debug(" >>>> ERROR <<<< Could not commit changes. Transaction was not open when we wanted to commit."); } else { try { - LOGGER.info("About to do the commit for " + LOGGER.debug("About to do the commit for " + deleteCount + " removes. "); g.tx().commit(); - LOGGER.info("Commit was successful "); + LOGGER.debug("Commit was successful "); } catch (Exception excom) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error(" >>>> ERROR <<<< Could not commit changes. " + LogFormatTools.getStackTop(excom)); + LOGGER.debug(" >>>> ERROR <<<< Could not commit changes. " + LogFormatTools.getStackTop(excom)); deleteCount = 0; } } @@ -1558,8 +1485,6 @@ public class DataGrooming { g.tx().rollback(); } catch (Exception ex) { // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); LOGGER.warn("WARNING from final graphTransaction.rollback()", ex); } } @@ -1570,8 +1495,6 @@ public class DataGrooming { g2.tx().rollback(); } catch (Exception ex) { // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex); } } @@ -1590,8 +1513,6 @@ public class DataGrooming { } } catch (Exception ex) { // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); LOGGER.warn("WARNING from final graph.shutdown()", ex); } @@ -1608,8 +1529,6 @@ public class DataGrooming { } } catch (Exception ex) { // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); LOGGER.warn("WARNING from final graph2.shutdown()", ex); } } @@ -1629,7 +1548,7 @@ public class DataGrooming { // NOTE -- as of 1902-P2, this is deprecated -------------- - LOGGER.info(" We will try to re-set the indexed properties for this node without changing any property values. VID = " + thisVidStr ); + LOGGER.debug(" We will try to re-set the indexed properties for this node without changing any property values. VID = " + thisVidStr ); // These reserved-prop-names are all indexed for all nodes ArrayList <String> propList = new ArrayList <String> (); @@ -1644,7 +1563,7 @@ public class DataGrooming { try { Object valObj = thisVtx.<Object>property(propName).orElse(null); if( valObj != null ){ - LOGGER.info(" We will try resetting prop [" + propName + LOGGER.debug(" We will try resetting prop [" + propName + "], to val = [" + valObj.toString() + "] for VID = " + thisVidStr); thisVtx.property(propName, valObj); } @@ -1667,7 +1586,7 @@ public class DataGrooming { // this pass. A future pass will just treat this node like a regular orphan // and delete it (if appropriate). - LOGGER.info(" We will be updating the indexed properties for this node to dummy values. VID = " + thisVidStr ); + LOGGER.debug(" We will be updating the indexed properties for this node to dummy values. VID = " + thisVidStr ); String dummyPropValStr = thisVidStr + "dummy"; // These reserved-prop-names are all indexed for all nodes thisVtx.property("aai-node-type",nType); @@ -2327,8 +2246,6 @@ public class DataGrooming { } } } catch (Exception e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e); } @@ -2454,9 +2371,7 @@ public class DataGrooming { // like, "KeepVid=12345" String[] prefArr = prefString.split("="); if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("Bad format. Expecting KeepVid=999999"); + LOGGER.debug("Bad format. Expecting KeepVid=999999"); return false; } else { String keepVidStr = prefArr[1]; @@ -2480,20 +2395,16 @@ public class DataGrooming { } catch (Exception e) { okFlag = false; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("ERROR trying to delete VID = " + thisVid + " " + LogFormatTools.getStackTop(e)); + LOGGER.debug("ERROR trying to delete VID = " + thisVid + " " + LogFormatTools.getStackTop(e)); } if (okFlag) { - LOGGER.info(" DELETED VID = " + thisVid); + LOGGER.debug(" DELETED VID = " + thisVid); deletedSomething = true; } } } } else { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = [" + LOGGER.debug("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = [" + dupeInfoString + "]"); return false; } @@ -2563,9 +2474,7 @@ public class DataGrooming { } } catch( Exception ex ){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error(" ERROR trying to get node with aai-uri: [" + aaiUriStr + "]" + LogFormatTools.getStackTop(ex)); + LOGGER.debug(" ERROR trying to get node with aai-uri: [" + aaiUriStr + "]" + LogFormatTools.getStackTop(ex)); } return true; @@ -2636,9 +2545,7 @@ public class DataGrooming { } } catch( Exception ex ){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); - LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]" + LogFormatTools.getStackTop(ex)); + LOGGER.debug( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]" + LogFormatTools.getStackTop(ex)); } if( verts != null ){ @@ -2979,7 +2886,7 @@ public class DataGrooming { catch ( AAIException ae ){ String emsg = "Error trying to get node just by key " + ae.getMessage(); //System.out.println(emsg); - LOGGER.error(emsg); + LOGGER.debug(emsg); } return returnVid; diff --git a/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java b/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java index 749a264..4309ece 100644 --- a/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java +++ b/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java @@ -20,31 +20,37 @@ package org.onap.aai.datagrooming; import java.io.BufferedReader; -import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.logging.ErrorLogHelper; -import org.onap.aai.logging.LoggingContext; +import org.onap.aai.logging.LogFormatTools; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConfig; +import org.onap.logging.filter.base.ONAPComponents; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; - @Component @PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties") public class DataGroomingTasks { + + private AaiScheduledTaskAuditLog auditLog; - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DataGroomingTasks.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DataGroomingTasks.class); private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); @Autowired @@ -55,25 +61,17 @@ public class DataGroomingTasks { @Scheduled(cron = "${datagroomingtasks.cron}" ) public void groomingScheduleTask() throws AAIException, Exception { - - LoggingContext.init(); - LoggingContext.requestId(UUID.randomUUID().toString()); - LoggingContext.partnerName("AAI"); - LoggingContext.targetEntity("CronApp"); - LoggingContext.component("dataGrooming"); - LoggingContext.serviceName("groomingScheduleTask"); - LoggingContext.targetServiceName("groomingScheduleTask"); - LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE); - + auditLog = new AaiScheduledTaskAuditLog(); + auditLog.logBefore("dataGroomingTask", ONAPComponents.AAI.toString()); if(!"true".equals(AAIConfig.get("aai.disable.check.grooming.running", "false"))){ if(checkIfDataGroomingIsRunning()){ - LOGGER.info("Data Grooming is already running on the system"); + LOGGER.debug("Data Grooming is already running on the system"); return; } } - - LOGGER.info("Started cron job dataGrooming @ " + dateFormat.format(new Date())); + + LOGGER.debug("Started cron job dataGrooming @ " + dateFormat.format(new Date())); Map<String, String> dataGroomingFlagMap = new HashMap<>(); append("enableautofix" , AAIConfig.get("aai.datagrooming.enableautofix"), dataGroomingFlagMap); @@ -162,13 +160,12 @@ public class DataGroomingTasks { } } catch (Exception e) { - ErrorLogHelper.logError("AAI_4000", "Exception running cron job for dataGrooming"+e.toString()); - LOGGER.info("AAI_4000", "Exception running cron job for dataGrooming"+e.toString()); - throw e; + ErrorLogHelper.logError("AAI_4000", "Exception running cron job for dataGrooming"+LogFormatTools.getStackTop(e)); + LOGGER.debug("AAI_4000", "Exception running cron job for dataGrooming"+LogFormatTools.getStackTop(e)); } finally { - LOGGER.info("Ended cron job dataGrooming @ " + dateFormat.format(new Date())); - LoggingContext.clear(); + LOGGER.debug("Ended cron job dataGrooming @ " + dateFormat.format(new Date())); } + auditLog.logAfter(); } private boolean checkIfDataGroomingIsRunning(){ @@ -187,9 +184,9 @@ public class DataGroomingTasks { } int exitVal = process.waitFor(); - LOGGER.info("Exit value of the dataGrooming check process: " + exitVal); + LOGGER.debug("Exit value of the dataGrooming check process: " + exitVal); } catch (Exception e) { - e.printStackTrace(); + LOGGER.debug("AAI_4000", "Exception running dataGrooming check process "+LogFormatTools.getStackTop(e)); } if(count > 0){ diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java index e7ae5ec..217d6c0 100644 --- a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java +++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java @@ -25,7 +25,14 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.SequenceInputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.Vector; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -33,34 +40,33 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.io.IoCore; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.JanusGraphFactory; +import org.janusgraph.core.util.JanusGraphCleanup; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.dbmap.AAIGraphConfig; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; -import org.onap.aai.util.GraphAdminConstants; import org.onap.aai.util.AAISystemExitUtil; import org.onap.aai.util.FormatDate; +import org.onap.aai.util.GraphAdminConstants; import org.onap.aai.util.GraphAdminDBUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; -import org.janusgraph.core.JanusGraph; -import org.janusgraph.core.JanusGraphFactory; -import org.janusgraph.core.util.JanusGraphCleanup; - public class DataSnapshot { - private static EELFLogger LOGGER; + private static Logger LOGGER; /* Using realtime d */ private static final String REALTIME_DB = "realtime"; @@ -108,17 +114,16 @@ public class DataSnapshot { public boolean executeCommand(String[] args, boolean success, Boolean dbClearFlag, JanusGraph graph, String command, String oldSnapshotFileName) { - + // Set the logging file properties to be used by EELFManager System.setProperty("aai.service.name", DataSnapshot.class.getSimpleName()); Properties props = System.getProperties(); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); - LOGGER = EELFManager.getInstance().getLogger(DataSnapshot.class); + LOGGER = LoggerFactory.getLogger(DataSnapshot.class); cArgs = new CommandLineArgs(); String itemName = "aai.datasnapshot.threads.for.create"; - try { String val = AAIConfig.get(itemName); if( val != null && !val.equals("") ){ @@ -128,9 +133,19 @@ public class DataSnapshot { LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); } int threadCount4Create = cArgs.threadCount; - + + itemName = "aai.datasnapshot.max.nodes.per.file.for.create"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.maxNodesPerFile = Long.parseLong(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + long maxNodesPerFile4Create = cArgs.maxNodesPerFile; + cArgs.snapshotType = "graphson"; - Long vertAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS; itemName = "aai.datasnapshot.vertex.add.delay.ms"; try { @@ -215,10 +230,10 @@ public class DataSnapshot { jCommander = new JCommander(cArgs, args); jCommander.setProgramName(DataSnapshot.class.getSimpleName()); } catch (ParameterException e1) { - LOGGER.error("Error - invalid value passed to list of args - "+args); + AAIException ae = new AAIException("AAI_6128", e1 , "Error - invalid value passed to list of args - "+args); + ErrorLogHelper.logException(ae); AAISystemExitUtil.systemExitCloseAAIGraph(1); } - if (args.length >= 1) { command = cArgs.command; @@ -253,6 +268,23 @@ public class DataSnapshot { } LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount4Create ); + try { + maxNodesPerFile4Create = cArgs.maxNodesPerFile; + } + catch ( NumberFormatException nfe ){ + ErrorLogHelper.logError("AAI_6128", "Bad (non-long) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]"); + LOGGER.debug("Bad (non-long) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]"); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + if( maxNodesPerFile4Create < 1000 || maxNodesPerFile4Create > 1000000 ){ + ErrorLogHelper.logError("AAI_6128", "Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]"); + LOGGER.debug("Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]"); + LOGGER.debug("Out of range (1000-1000000) maxNodesPerFile >> Recommended value = 120000)"); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + LOGGER.debug(" Will do Threaded Snapshot with maxNodesPerFile = " + maxNodesPerFile4Create ); + // If doing a "threaded" snapshot, they need to specify how many threads to use // They can also use debug mode if they pass the word "DEBUG" to do the nodes one at a time to see where it breaks. if( cArgs.debugFlag.equals("DEBUG") ){ @@ -326,42 +358,45 @@ public class DataSnapshot { } + threadCount4Create = cArgs.threadCount; + maxNodesPerFile4Create = cArgs.maxNodesPerFile; //Print Defaults - LOGGER.info("DataSnapshot command is [" + cArgs.command + "]"); - LOGGER.info("File name to reload snapshot [" + cArgs.oldFileName + "]"); - LOGGER.info("snapshotType is [" + cArgs.snapshotType + "]"); - LOGGER.info("Thread count is [" + cArgs.threadCount + "]"); - LOGGER.info("Debug Flag is [" + cArgs.debugFlag + "]"); - LOGGER.info("DebugAddDelayTimer is [" + cArgs.debugAddDelayTime + "]"); - LOGGER.info("VertAddDelayMs is [" + cArgs.vertAddDelayMs + "]"); - LOGGER.info("FailureDelayMs is [" + cArgs.failureDelayMs + "]"); - LOGGER.info("RetryDelayMs is [" + cArgs.retryDelayMs + "]"); - LOGGER.info("MaxErrorsPerThread is [" + cArgs.maxErrorsPerThread + "]"); - LOGGER.info("VertToEdgeProcDelay is [" + cArgs.vertToEdgeProcDelay + "]"); - LOGGER.info("StaggerThreadDelay is [" + cArgs.staggerThreadDelay + "]"); - LOGGER.info("Caller process is ["+ cArgs.caller + "]"); + LOGGER.debug("DataSnapshot command is [" + cArgs.command + "]"); + LOGGER.debug("File name to reload snapshot [" + cArgs.oldFileName + "]"); + LOGGER.debug("snapshotType is [" + cArgs.snapshotType + "]"); + LOGGER.debug("Thread count is [" + cArgs.threadCount + "]"); + LOGGER.debug("Max Nodes Per File is [" + cArgs.maxNodesPerFile + "]"); + LOGGER.debug("Debug Flag is [" + cArgs.debugFlag + "]"); + LOGGER.debug("DebugAddDelayTimer is [" + cArgs.debugAddDelayTime + "]"); + LOGGER.debug("VertAddDelayMs is [" + cArgs.vertAddDelayMs + "]"); + LOGGER.debug("FailureDelayMs is [" + cArgs.failureDelayMs + "]"); + LOGGER.debug("RetryDelayMs is [" + cArgs.retryDelayMs + "]"); + LOGGER.debug("MaxErrorsPerThread is [" + cArgs.maxErrorsPerThread + "]"); + LOGGER.debug("VertToEdgeProcDelay is [" + cArgs.vertToEdgeProcDelay + "]"); + LOGGER.debug("StaggerThreadDelay is [" + cArgs.staggerThreadDelay + "]"); + LOGGER.debug("Caller process is ["+ cArgs.caller + "]"); + //Print non-default values if (!AAIConfig.isEmpty(cArgs.fileName)){ - LOGGER.info("Snapshot file name (if not default) to use is [" + cArgs.fileName + "]"); + LOGGER.debug("Snapshot file name (if not default) to use is [" + cArgs.fileName + "]"); } if (!AAIConfig.isEmpty(cArgs.snapshotDir)){ - LOGGER.info("Snapshot file Directory path (if not default) to use is [" + cArgs.snapshotDir + "]"); + LOGGER.debug("Snapshot file Directory path (if not default) to use is [" + cArgs.snapshotDir + "]"); } if (!AAIConfig.isEmpty(cArgs.oldFileDir)){ - LOGGER.info("Directory path (if not default) to load the old snapshot file from is [" + cArgs.oldFileDir + "]"); + LOGGER.debug("Directory path (if not default) to load the old snapshot file from is [" + cArgs.oldFileDir + "]"); } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { - AAIConfig.init(); ErrorLogHelper.loadProperties(); LOGGER.debug("Command = " + command + ", oldSnapshotFileName = [" + oldSnapshotFileName + "]."); String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dataSnapshots"; // Make sure the dataSnapshots directory is there new File(targetDir).mkdirs(); - LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while) "); if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT")) @@ -370,7 +405,7 @@ public class DataSnapshot { // They want to take a snapshot on a single thread and have it go in a single file // NOTE - they can't use the DEBUG option in this case. // ------------------------------------------------------------------------------- - LOGGER.debug("\n>>> Command = " + command ); + LOGGER.debug(" Command = " + command ); verifyGraph(AAIGraph.getInstance().getGraph()); FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); String dteStr = fd.getDateTime(); @@ -390,19 +425,20 @@ public class DataSnapshot { } else if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT")) - && threadCount4Create > 1 ){ + && threadCount4Create > 1 ){ // ------------------------------------------------------------ // They want the creation of the snapshot to be spread out via // threads and go to multiple files // ------------------------------------------------------------ - LOGGER.debug("\n>>> Command = " + command ); + LOGGER.debug(" Command = " + command ); String newSnapshotOutFname; if (!AAIConfig.isEmpty(cArgs.fileName)){ newSnapshotOutFname = cArgs.fileName; } else { - FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); - String dteStr = fd.getDateTime(); - newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr; + FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); + String dteStr = fd.getDateTime(); + newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + + "dataSnapshot.graphSON." + dteStr; } verifyGraph(AAIGraph.getInstance().getGraph()); graph = AAIGraph.getInstance().getGraph(); @@ -410,43 +446,57 @@ public class DataSnapshot { GraphAdminDBUtils.logConfigs(graph.configuration()); long timeA = System.nanoTime(); - LOGGER.debug(" Need to divide vertexIds across this many threads: " + threadCount4Create ); - HashMap <String,ArrayList> vertListHash = new HashMap <String,ArrayList> (); - for( int t = 0; t < threadCount4Create; t++ ){ - ArrayList <Vertex> vList = new ArrayList <Vertex> (); - String tk = "" + t; - vertListHash.put( tk, vList); - } LOGGER.debug("Count how many nodes are in the db. "); long totalVertCount = graph.traversal().V().count().next(); - LOGGER.debug(" Total Count of Nodes in DB = " + totalVertCount + "."); - long nodesPerFile = totalVertCount / threadCount4Create; - LOGGER.debug(" Thread count = " + threadCount4Create + ", each file will get (roughly): " + nodesPerFile + " nodes."); long timeA2 = System.nanoTime(); long diffTime = timeA2 - timeA; long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); LOGGER.debug(" -- To count all vertices in DB it took: " + minCount + " minutes, " + secCount + " seconds " ); + LOGGER.debug(" Total Count of Nodes in DB = " + totalVertCount + "."); - long vtxIndex = 0; + int fileCount4Create = figureOutFileCount( totalVertCount, threadCount4Create, + maxNodesPerFile4Create ); + int threadPassesNeeded = (int) Math.ceil((double)fileCount4Create / (double)threadCount4Create); + long nodesPerFile = (long) Math.ceil((double)totalVertCount / (double)fileCount4Create); + + LOGGER.debug(" We will run this many simultaneous threads: " + threadCount4Create ); + LOGGER.debug(" Required number of passes: " + threadPassesNeeded ); + LOGGER.debug(" Max Nodes per file: " + maxNodesPerFile4Create ); + LOGGER.debug(" We will generate this many files: " + fileCount4Create ); + LOGGER.debug(" Each file will have (roughly): " + nodesPerFile + " nodes."); + LOGGER.debug(" Now, divide vertexIds across this many files: " + fileCount4Create ); + + HashMap <String,ArrayList<Long>> vertIdListHash = new HashMap <String,ArrayList<Long>> (); + for( int t = 0; t < fileCount4Create; t++ ){ + ArrayList <Long> vIdList = new ArrayList <Long> (); + String tk = "" + t; + vertIdListHash.put( tk, vIdList); + } + int currentTNum = 0; String currentTKey = "0"; long thisThrIndex = 0; - Iterator <Vertex> vtxItr = graph.vertices(); + Iterator <Vertex> vtxItr = graph.vertices(); // Getting ALL vertices! while( vtxItr.hasNext() ){ - // Divide up all the vertices so we can process them on different threads - vtxIndex++; + // Divide up ALL the vertices so we can process them on different threads thisThrIndex++; - if( (thisThrIndex > nodesPerFile) && (currentTNum < threadCount4Create -1) ){ - // We will need to start adding to the Hash for the next thread + if( (thisThrIndex >= nodesPerFile) && (currentTNum < (fileCount4Create - 1)) ){ + // We will need to start adding to the Hash for the next file currentTNum++; currentTKey = "" + currentTNum; thisThrIndex = 0; } - (vertListHash.get(currentTKey)).add(vtxItr.next()); + long vid = (long)(vtxItr.next()).id(); + (vertIdListHash.get(currentTKey)).add(vid); } + // close this graph instance thing here since we have all the ids + graph.tx().rollback(); + graph.tx().close(); + + long timeB = System.nanoTime(); diffTime = timeB - timeA2; minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); @@ -455,21 +505,38 @@ public class DataSnapshot { minCount + " minutes, " + secCount + " seconds " ); // Need to print out each set of vertices using it's own thread - ArrayList <Thread> threadArr = new ArrayList <Thread> (); - for( int thNum = 0; thNum < threadCount4Create; thNum++ ){ - String thNumStr = "" + thNum; - String subFName = newSnapshotOutFname + ".P" + thNumStr; - Thread thr = new Thread(new PrintVertexDetails(graph, subFName, vertListHash.get(thNumStr), - debug4Create, debugAddDelayTime, snapshotType) ); - thr.start(); - threadArr.add(thr); - } + // NOTE - we may have more files to generate than number of threads - which + // just means that ALL the files won't necessarily be generated in parallel. - // Make sure all the threads finish before moving on. - for( int thNum = 0; thNum < threadCount4Create; thNum++ ){ - if( null != threadArr.get(thNum) ){ - (threadArr.get(thNum)).join(); + int fileNo = 0; + for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){ + ArrayList <Thread> threadArr = new ArrayList <Thread> (); + // For each Pass, kick off all the threads and wait until they finish + long timeP1 = System.nanoTime(); + for( int thNum = 0; thNum < threadCount4Create; thNum++ ){ + String fileNoStr = "" + fileNo; + String subFName = newSnapshotOutFname + ".P" + fileNoStr; + LOGGER.debug(" DEBUG >>> kick off pass # " + passNo + ", thread # " + thNum); + Thread thr = new Thread(new PrintVertexDetails(graph, subFName, + vertIdListHash.get(fileNoStr), + debug4Create, debugAddDelayTime, + snapshotType, LOGGER) ); + thr.start(); + threadArr.add(thr); + fileNo++; + } + // Make sure all the threads finish before considering this Pass finished. + for( int thNum = 0; thNum < threadCount4Create; thNum++ ){ + if( null != threadArr.get(thNum) ){ + (threadArr.get(thNum)).join(); + } } + long timeP2 = System.nanoTime(); + diffTime = timeP2 - timeP1; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" Pass number " + passNo + " (out of " + threadPassesNeeded + + ") took " + minCount + " minutes, " + secCount + " seconds "); } long timeC = System.nanoTime(); @@ -487,7 +554,7 @@ public class DataSnapshot { // snapshot is written to. Ie. if you have a single-file snapshot, // then this will be single-threaded. // - LOGGER.debug("\n>>> Command = " + command ); + LOGGER.debug(" Command = " + command ); if (cArgs.oldFileDir != null && cArgs.oldFileDir != ""){ targetDir = cArgs.oldFileDir; @@ -497,7 +564,7 @@ public class DataSnapshot { JanusGraph graph1 = AAIGraph.getInstance().getGraph(); long timeStart = System.nanoTime(); - HashMap <String,String> old2NewVertIdMap = new <String,String> HashMap (); + HashMap <String,String> old2NewVertIdMap = new HashMap <String,String> (); // We're going to try loading in the vertices - without edges or properties // using Separate threads @@ -535,11 +602,13 @@ public class DataSnapshot { } catch (InterruptedException e) { threadFailCount++; - e.printStackTrace(); + AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); + ErrorLogHelper.logException(ae); } catch (ExecutionException e) { threadFailCount++; - e.printStackTrace(); + AAIException ae = new AAIException("AAI_6128", e , "ExecutionException"); + ErrorLogHelper.logException(ae); } } @@ -602,11 +671,13 @@ public class DataSnapshot { } catch (InterruptedException e) { threadFailCount++; - e.printStackTrace(); + AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); + ErrorLogHelper.logException(ae); } catch (ExecutionException e) { threadFailCount++; - e.printStackTrace(); + AAIException ae = new AAIException("AAI_6128", e , "ExecutionException"); + ErrorLogHelper.logException(ae); } } @@ -639,13 +710,13 @@ public class DataSnapshot { // They are calling this to clear the db before re-loading it // later // ------------------------------------------------------------------ - LOGGER.debug("\n>>> Command = " + command ); + LOGGER.debug(" Command = " + command ); // First - make sure the backup file(s) they will be using can be // found and has(have) data. // getFilesToProcess makes sure the file(s) exist and have some data. getFilesToProcess(targetDir, oldSnapshotFileName, true); - LOGGER.debug("\n>>> WARNING <<<< "); + LOGGER.debug(" WARNING <<<< "); LOGGER.debug(">>> All data and schema in this database will be removed at this point. <<<"); LOGGER.debug(">>> Processing will begin in 5 seconds. <<<"); LOGGER.debug(">>> WARNING <<<< "); @@ -660,7 +731,7 @@ public class DataSnapshot { LOGGER.debug(" Begin clearing out old data. "); String rtConfig = AAIConstants.REALTIME_DB_CONFIG; - String serviceName = System.getProperty("aai.service.name", "NA"); + String serviceName = System.getProperty("aai.service.name", DataSnapshot.class.getSimpleName()); LOGGER.debug("Getting new configs for clearig"); PropertiesConfiguration propertiesConfiguration = new AAIGraphConfig.Builder(rtConfig).forService(serviceName).withGraphType(REALTIME_DB).buildConfiguration(); LOGGER.debug("Open New Janus Graph"); @@ -681,7 +752,7 @@ public class DataSnapshot { // of snapshot files. Either way, this command will restore via single // threaded processing. // --------------------------------------------------------------------------- - LOGGER.debug("\n>>> Command = " + command ); + LOGGER.debug(" Command = " + command ); verifyGraph(AAIGraph.getInstance().getGraph()); graph = AAIGraph.getInstance().getGraph(); GraphAdminDBUtils.logConfigs(graph.configuration()); @@ -771,14 +842,11 @@ public class DataSnapshot { } } catch (AAIException e) { - ErrorLogHelper.logError("AAI_6128", e.getMessage()); - LOGGER.error("Encountered an exception during the datasnapshot: ", e); - e.printStackTrace(); + ErrorLogHelper.logException(e); success = false; } catch (Exception ex) { - ErrorLogHelper.logError("AAI_6128", ex.getMessage()); - LOGGER.error("Encountered an exception during the datasnapshot: ", ex); - ex.printStackTrace(); + AAIException ae = new AAIException("AAI_6128", ex , "Encountered an exception during the datasnapshot"); + ErrorLogHelper.logException(ae); success = false; } finally { if (!dbClearFlag && graph != null && !MIGRATION_PROCESS_NAME.equalsIgnoreCase(source)) { @@ -879,9 +947,28 @@ public class DataSnapshot { } - class CommandLineArgs { - + + public static int figureOutFileCount( long totalVertCount, int threadCount4Create, + long maxNodesPerFile ) { + + // NOTE - we would always like to use all of our threads. That is, if + // we could process all the data with 16 threads, but our threadCount4Create is + // only 15, we will do two passes and use all 15 threads each pass which will + // create a total of 30 files. Each file will be a bit smaller so the overall + // time for the two passes should be faster. + if( totalVertCount <= 0 || threadCount4Create <= 0 || maxNodesPerFile <= 0) { + return 1; + } + long maxNodesPerPass = threadCount4Create * maxNodesPerFile; + int numberOfPasses = (int) Math.ceil( (double)totalVertCount / (double)maxNodesPerPass); + int fileCt = threadCount4Create * numberOfPasses; + + return fileCt; + } + + + class CommandLineArgs { @Parameter(names = "--help", help = true) public boolean help; @@ -897,6 +984,9 @@ public class DataSnapshot { @Parameter(names = "-threadCount", description = "thread count for create") public int threadCount = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_THREADS_FOR_CREATE; + + @Parameter(names = "-maxNodesPerFile", description = "Max nodes per file") + public long maxNodesPerFile = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_NODES_PER_FILE_FOR_CREATE; @Parameter(names = "-debugFlag", description = "DEBUG flag") public String debugFlag = ""; @@ -924,7 +1014,7 @@ public class DataSnapshot { @Parameter(names = "-staggerThreadDelay", description = "thread delay stagger time in ms") public long staggerThreadDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_STAGGER_THREAD_DELAY_MS; - + @Parameter(names = "-fileName", description = "file name for generating snapshot ") public String fileName = ""; diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java new file mode 100644 index 0000000..03ff51a --- /dev/null +++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java @@ -0,0 +1,1079 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.datasnapshot; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.Map.Entry; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.Vector; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.onap.aai.config.PropertyPasswordConfiguration; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.tinkerpop.gremlin.structure.io.IoCore; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.JanusGraphFactory; +import org.janusgraph.core.util.JanusGraphCleanup; +import org.onap.aai.dbmap.AAIGraph; +import org.onap.aai.dbmap.AAIGraphConfig; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.Loader; + + +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.LoaderUtil; +import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; +import org.onap.aai.util.AAIConstants; +import org.onap.aai.util.GraphAdminConstants; +import org.onap.aai.util.AAISystemExitUtil; +import org.onap.aai.util.FormatDate; +import org.onap.aai.util.GraphAdminDBUtils; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import com.att.eelf.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; + +public class DataSnapshot4HistInit { + + private static Logger LOGGER; + + /* Using realtime d */ + private static final String REALTIME_DB = "realtime"; + + private static final Set<String> SNAPSHOT_RELOAD_COMMANDS = new HashSet<>(); + + private static final String MIGRATION_PROCESS_NAME = "migration"; + + private static boolean historyEnabled; + + private LoaderFactory loaderFactory; + private SchemaVersions schemaVersions; + + static { + SNAPSHOT_RELOAD_COMMANDS.add("RELOAD_DATA"); + SNAPSHOT_RELOAD_COMMANDS.add("RELOAD_DATA_MULTI"); + } + + private CommandLineArgs cArgs; + + public DataSnapshot4HistInit(LoaderFactory loaderFactory, SchemaVersions schemaVersions){ + this.loaderFactory = loaderFactory; + this.schemaVersions = schemaVersions; + } + + /** + * The main method. + * + * @param args + * the arguments + */ + public static void main(String[] args) { + + // Set the logging file properties to be used by EELFManager + System.setProperty("aai.service.name", DataSnapshot4HistInit.class.getSimpleName()); + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); + LOGGER = LoggerFactory.getLogger(DataSnapshot4HistInit.class); + + + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); + PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); + initializer.initialize(ctx); + try { + ctx.scan( + "org.onap.aai.config", + "org.onap.aai.setup" + ); + ctx.refresh(); + } catch (Exception e) { + AAIException ae = new AAIException("AAI_6128", e , "Error - Could not initialize context beans for DataSnapshot4HistInit."); + ErrorLogHelper.logException(ae); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); + if( !historyEnabled ) { + String emsg = "Error - DataSnapshot4HistInit may only be used when history.enabled=true. "; + System.out.println(emsg); + AAIException ae = new AAIException("AAI_6128", emsg); + ErrorLogHelper.logException(ae); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); + SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions"); + DataSnapshot4HistInit dataSnapshotHI = new DataSnapshot4HistInit(loaderFactory, schemaVersions); + + boolean success = dataSnapshotHI.executeCommand(args); + if(success){ + AAISystemExitUtil.systemExitCloseAAIGraph(0); + } else { + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + }// End of main() + + + public boolean executeCommand(String[] args) { + + Boolean dbClearFlag = false; + JanusGraph graph = null; + String command = "UNKNOWN"; + String oldSnapshotFileName = ""; + boolean success = true; + + cArgs = new CommandLineArgs(); + String itemName = "aai.datasnapshot.threads.for.create"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.threadCount = Integer.parseInt(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + int threadCount4Create = cArgs.threadCount; + + itemName = "aai.datasnapshot.max.nodes.per.file.for.create"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.maxNodesPerFile = Long.parseLong(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + long maxNodesPerFile4Create = cArgs.maxNodesPerFile; + + cArgs.snapshotType = "graphson"; + Long vertAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS; + itemName = "aai.datasnapshot.vertex.add.delay.ms"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.vertAddDelayMs = Long.parseLong(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + + Long edgeAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_EDGE_ADD_DELAY_MS; + itemName = "aai.datasnapshot.edge.add.delay.ms"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.edgeAddDelayMs = Long.parseLong(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + + Long failureDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_FAILURE_DELAY_MS; + itemName = "aai.datasnapshot.failure.delay.ms"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.failureDelayMs = Long.parseLong(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + + Long retryDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_RETRY_DELAY_MS; + itemName = "aai.datasnapshot.retry.delay.ms"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.retryDelayMs = Long.parseLong(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + + int maxErrorsPerThread = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_ERRORS_PER_THREAD; + itemName = "aai.datasnapshot.max.errors.per.thread"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.maxErrorsPerThread = Integer.parseInt(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + + Long vertToEdgeProcDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_TO_EDGE_PROC_DELAY_MS; + itemName = "aai.datasnapshot.vertex.to.edge.proc.delay.ms"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.vertToEdgeProcDelay = Long.parseLong(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + + itemName = "aai.datasnapshot.stagger.thread.delay.ms"; + try { + String val = AAIConfig.get(itemName); + if( val != null && !val.equals("") ){ + cArgs.staggerThreadDelay = Long.parseLong(val); + } + }catch ( Exception e ){ + LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage()); + } + + long debugAddDelayTime = 1; // Default to 1 millisecond + Boolean debug4Create = false; // By default we do not use debugging for snapshot creation + JCommander jCommander; + try { + jCommander = new JCommander(cArgs, args); + jCommander.setProgramName(DataSnapshot4HistInit.class.getSimpleName()); + } catch (ParameterException e1) { + AAIException ae = new AAIException("AAI_6128", e1 , "Error - invalid value passed to list of args - "+args); + ErrorLogHelper.logException(ae); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + + if (args.length >= 1) { + command = cArgs.command; + } + + String source = cArgs.caller; + String snapshotType = "graphson"; + if( SNAPSHOT_RELOAD_COMMANDS.contains(cArgs.command)){ + if (args.length >= 2) { + // If re-loading, they need to also pass the snapshot file name to use. + // We expected the file to be found in our snapshot directory. + oldSnapshotFileName = cArgs.oldFileName; + snapshotType = cArgs.snapshotType; + } + } + else if( command.equals("THREADED_SNAPSHOT") ){ + if (args.length >= 2) { + // If doing a "threaded" snapshot, they need to specify how many threads to use + try { + threadCount4Create = cArgs.threadCount; + } + catch ( NumberFormatException nfe ){ + ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) threadCount passed to DataSnapshot [" + cArgs.threadCount + "]"); + LOGGER.debug("Bad (non-integer) threadCount passed to DataSnapshot [" + cArgs.threadCount + "]"); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + if( threadCount4Create < 1 || threadCount4Create > 100 ){ + ErrorLogHelper.logError("AAI_6128", "Out of range (1-100) threadCount passed to DataSnapshot [" + cArgs.threadCount + "]"); + LOGGER.debug("Out of range (1-100) threadCount passed to DataSnapshot [" + cArgs.threadCount + "]"); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount4Create ); + + if( maxNodesPerFile4Create < 1000 || maxNodesPerFile4Create > 1000000 ){ + ErrorLogHelper.logError("AAI_6128", "Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]"); + LOGGER.debug("Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]"); + LOGGER.debug("Out of range (1000-1000000) maxNodesPerFile >> Recommended value = 120000)"); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + LOGGER.debug(" Will do Threaded Snapshot with maxNodesPerFile = " + maxNodesPerFile4Create ); + + // If doing a "threaded" snapshot, they need to specify how many threads to use + // They can also use debug mode if they pass the word "DEBUG" to do the nodes one at a time to see where it breaks. + if( cArgs.debugFlag.equals("DEBUG") ){ + debug4Create = true; + } + LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount4Create + + ", and DEBUG-flag set to: " + debug4Create ); + + if (debug4Create) { + // If doing a "threaded" snapshot, they need to specify how many threads to use (param 1) + // They can also use debug mode if they pass the word "DEBUG" to do the nodes one (param 2) + // They can also pass a delayTimer - how many milliseconds to put between each node's ADD (param 3) + try { + debugAddDelayTime = cArgs.debugAddDelayTime; + } catch (NumberFormatException nfe) { + ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) debugAddDelayTime passed to DataSnapshot [" + + cArgs.debugAddDelayTime + "]"); + LOGGER.debug("Bad (non-integer) debugAddDelayTime passed to DataSnapshot ["+ cArgs.debugAddDelayTime + "]"); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + LOGGER.debug(" Will do Threaded Snapshot with threadCount = "+ threadCount4Create + ", DEBUG-flag set to: " + + debug4Create + ", and addDelayTimer = " + debugAddDelayTime + " mSec. "); + } + } + else { + ErrorLogHelper.logError("AAI_6128", "Wrong param count (should be 2,3 or 4) when using THREADED_SNAPSHOT."); + LOGGER.debug("Wrong param count (should be 2,3 or 4) when using THREADED_SNAPSHOT."); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + } + else if( command.equals("MULTITHREAD_RELOAD") ){ + // Note - this will use as many threads as the snapshot file is + // broken up into. (up to a limit) + if (args.length >= 2) { + // Since they are re-loading, they need to pass the snapshot file name to use. + // We expected the file to be found in our snapshot directory. Note - if + // it is a multi-part snapshot, then this should be the root of the name. + // We will be using the default delay timers. + oldSnapshotFileName = cArgs.oldFileName; + vertAddDelayMs = cArgs.vertAddDelayMs; + edgeAddDelayMs = cArgs.edgeAddDelayMs; + failureDelayMs = cArgs.failureDelayMs; + retryDelayMs = cArgs.retryDelayMs; + try { + maxErrorsPerThread = cArgs.maxErrorsPerThread; + } + catch ( NumberFormatException nfe ){ + ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) maxErrorsPerThread passed to DataSnapshot [" + cArgs.maxErrorsPerThread + "]"); + LOGGER.debug("Bad (non-integer) maxErrorsPerThread passed to DataSnapshot [" + cArgs.maxErrorsPerThread + "]"); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + if( maxErrorsPerThread < 1 ){ + ErrorLogHelper.logError("AAI_6128", "Out of range (>0) maxErrorsPerThread passed to DataSnapshot [" + cArgs.maxErrorsPerThread + "]"); + LOGGER.debug("Out of range (>0) maxErrorsPerThread passed to DataSnapshot [" + cArgs.maxErrorsPerThread + "]"); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + } + else { + ErrorLogHelper.logError("AAI_6128", "Wrong param count (should be either 2 or 7) when using MUTLITHREAD_RELOAD."); + LOGGER.debug("Wrong param count (should be 2 or 7) when using MUTLITHREAD_RELOAD."); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + } + else if (command.equals("CLEAR_ENTIRE_DATABASE")) { + if (args.length >= 2) { + oldSnapshotFileName = cArgs.oldFileName; + } + } + long scriptStartTime = System.currentTimeMillis(); + + threadCount4Create = cArgs.threadCount; + + //Print Defaults + LOGGER.debug("DataSnapshot4HistInit command is [" + cArgs.command + "]"); + LOGGER.debug("File name to reload snapshot [" + cArgs.oldFileName + "]"); + LOGGER.debug("snapshotType is [" + cArgs.snapshotType + "]"); + LOGGER.debug("Thread count is [" + cArgs.threadCount + "]"); + LOGGER.debug("Debug Flag is [" + cArgs.debugFlag + "]"); + LOGGER.debug("DebugAddDelayTimer is [" + cArgs.debugAddDelayTime + "]"); + LOGGER.debug("VertAddDelayMs is [" + cArgs.vertAddDelayMs + "]"); + LOGGER.debug("FailureDelayMs is [" + cArgs.failureDelayMs + "]"); + LOGGER.debug("RetryDelayMs is [" + cArgs.retryDelayMs + "]"); + LOGGER.debug("MaxErrorsPerThread is [" + cArgs.maxErrorsPerThread + "]"); + LOGGER.debug("VertToEdgeProcDelay is [" + cArgs.vertToEdgeProcDelay + "]"); + LOGGER.debug("StaggerThreadDelay is [" + cArgs.staggerThreadDelay + "]"); + LOGGER.debug("Caller process is ["+ cArgs.caller + "]"); + + //Print non-default values + if (!AAIConfig.isEmpty(cArgs.fileName)){ + LOGGER.debug("Snapshot file name (if not default) to use is [" + cArgs.fileName + "]"); + } + if (!AAIConfig.isEmpty(cArgs.snapshotDir)){ + LOGGER.debug("Snapshot file Directory path (if not default) to use is [" + cArgs.snapshotDir + "]"); + } + if (!AAIConfig.isEmpty(cArgs.oldFileDir)){ + LOGGER.debug("Directory path (if not default) to load the old snapshot file from is [" + cArgs.oldFileDir + "]"); + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + AAIConfig.init(); + ErrorLogHelper.loadProperties(); + LOGGER.debug("Command = " + command + ", oldSnapshotFileName = [" + oldSnapshotFileName + "]."); + String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dataSnapshots"; + // Make sure the dataSnapshots directory is there + new File(targetDir).mkdirs(); + + LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while) "); + + if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT")) + && threadCount4Create == 1 ){ + // ------------------------------------------------------------------------------- + // They want to take a snapshot on a single thread and have it go in a single file + // NOTE - they can't use the DEBUG option in this case. + // ------------------------------------------------------------------------------- + LOGGER.debug(" Command = " + command ); + verifyGraph(AAIGraph.getInstance().getGraph()); + FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); + String dteStr = fd.getDateTime(); + graph = AAIGraph.getInstance().getGraph(); + GraphAdminDBUtils.logConfigs(graph.configuration()); + String newSnapshotOutFname = null; + long timeA = System.nanoTime(); + newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr; + graph.io(IoCore.graphson()).writeGraph(newSnapshotOutFname); + LOGGER.debug("Snapshot written to " + newSnapshotOutFname); + long timeB = System.nanoTime(); + long diffTime = timeB - timeA; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- Single-Thread dataSnapshot took: " + + minCount + " minutes, " + secCount + " seconds " ); + + } + else if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT")) + && threadCount4Create > 1 ){ + // ------------------------------------------------------------ + // They want the creation of the snapshot to be spread out via + // threads and go to multiple files + // ------------------------------------------------------------ + LOGGER.debug(" Command = " + command ); + String newSnapshotOutFname; + if (!AAIConfig.isEmpty(cArgs.fileName)){ + newSnapshotOutFname = cArgs.fileName; + } else { + FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT"); + String dteStr = fd.getDateTime(); + newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr; + } + verifyGraph(AAIGraph.getInstance().getGraph()); + graph = AAIGraph.getInstance().getGraph(); + LOGGER.debug(" Successfully got the Graph instance. "); + GraphAdminDBUtils.logConfigs(graph.configuration()); + long timeA = System.nanoTime(); + + LOGGER.debug("Count how many nodes are in the db. "); + long totalVertCount = graph.traversal().V().count().next(); + LOGGER.debug(" Total Count of Nodes in DB = " + totalVertCount + "."); + long timeA2 = System.nanoTime(); + long diffTime = timeA2 - timeA; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To count all vertices in DB it took: " + + minCount + " minutes, " + secCount + " seconds " ); + + int fileCount4Create = figureOutFileCount( totalVertCount, threadCount4Create, + maxNodesPerFile4Create ); + int threadPassesNeeded = (int) Math.ceil((double)fileCount4Create / (double)threadCount4Create); + long nodesPerFile = (long) Math.ceil((double)totalVertCount / (double)fileCount4Create); + + LOGGER.debug(" We will run this many simultaneous threads: " + threadCount4Create ); + LOGGER.debug(" Required number of passes: " + threadPassesNeeded ); + LOGGER.debug(" Max Nodes per file: " + maxNodesPerFile4Create ); + LOGGER.debug(" We will generate this many files: " + fileCount4Create ); + LOGGER.debug(" Each file will have (roughly): " + nodesPerFile + " nodes."); + LOGGER.debug(" Now, divide vertexIds across this many files: " + fileCount4Create ); + + HashMap <String,ArrayList<Long>> vertIdListHash = new HashMap <String,ArrayList<Long>> (); + for( int t = 0; t < fileCount4Create; t++ ){ + ArrayList <Long> vIdList = new ArrayList <Long> (); + String tk = "" + t; + vertIdListHash.put( tk, vIdList); + } + + int currentTNum = 0; + String currentTKey = "0"; + long thisThrIndex = 0; + Iterator <Vertex> vtxItr = graph.vertices(); // Getting ALL vertices! + while( vtxItr.hasNext() ){ + // Divide up ALL the vertices so we can process them on different threads + thisThrIndex++; + if( (thisThrIndex >= nodesPerFile) && (currentTNum < (fileCount4Create - 1)) ){ + // We will need to start adding to the Hash for the next file + currentTNum++; + currentTKey = "" + currentTNum; + thisThrIndex = 0; + } + long vid = (long)(vtxItr.next()).id(); + (vertIdListHash.get(currentTKey)).add(vid); + } + + // close this graph instance thing here since we have all the ids + graph.tx().rollback(); + graph.tx().close(); + + long timeB = System.nanoTime(); + diffTime = timeB - timeA2; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To Loop over all vertices, and put them into sub-Arrays it took: " + + minCount + " minutes, " + secCount + " seconds " ); + + // Need to print out each set of vertices using it's own thread + // NOTE - we may have more files to generate than number of threads - which + // just means that ALL the files won't necessarily be generated in parallel. + + int fileNo = 0; + for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){ + ArrayList <Thread> threadArr = new ArrayList <Thread> (); + // For each Pass, kick off all the threads and wait until they finish + long timeP1 = System.nanoTime(); + for( int thNum = 0; thNum < threadCount4Create; thNum++ ){ + String fileNoStr = "" + fileNo; + String subFName = newSnapshotOutFname + ".P" + fileNoStr; + LOGGER.debug(" DEBUG >>> kick off pass # " + passNo + ", thread # " + thNum); + Thread thr = new Thread(new PrintVertexDetails(graph, subFName, + vertIdListHash.get(fileNoStr), + debug4Create, debugAddDelayTime, + snapshotType, LOGGER) ); + thr.start(); + threadArr.add(thr); + fileNo++; + } + // Make sure all the threads finish before considering this Pass finished. + for( int thNum = 0; thNum < threadCount4Create; thNum++ ){ + if( null != threadArr.get(thNum) ){ + (threadArr.get(thNum)).join(); + } + } + long timeP2 = System.nanoTime(); + diffTime = timeP2 - timeP1; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" Pass number " + passNo + " (out of " + threadPassesNeeded + + ") took " + minCount + " minutes, " + secCount + " seconds "); + } + + long timeC = System.nanoTime(); + diffTime = timeC - timeB; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To write all the data out to snapshot files, it took: " + + minCount + " minutes, " + secCount + " seconds " ); + + + } else if( command.equals("MULTITHREAD_RELOAD") ){ + // --------------------------------------------------------------------- + // They want the RELOAD of the snapshot to be spread out via threads + // NOTE - it will only use as many threads as the number of files the + // snapshot is written to. Ie. if you have a single-file snapshot, + // then this will be single-threaded. + // + LOGGER.debug(" Command = " + command ); + if (cArgs.oldFileDir != null && cArgs.oldFileDir != ""){ + targetDir = cArgs.oldFileDir; + } + ArrayList <File> snapFilesArr = getFilesToProcess(targetDir, oldSnapshotFileName, false); + int fCount = snapFilesArr.size(); + JanusGraph graph1 = AAIGraph.getInstance().getGraph(); + GraphAdminDBUtils.logConfigs(graph1.configuration()); + long timeStart = System.nanoTime(); + HashMap <String,String> old2NewVertIdMap = new <String,String> HashMap (); + HashMap <String,ArrayList<String>> nodeKeyNames = new <String,ArrayList<String>> HashMap (); + + try { + LOGGER.debug("call getNodeKeyNames ()" ); + nodeKeyNames = getNodeKeyNames(); + } catch (Exception e) { + AAIException ae = new AAIException("AAI_6128", e , "Error - Could not get node Key names "+e.getMessage()); + ErrorLogHelper.logException(ae); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + // We're going to try loading in the vertices - without edges or properties + // using Separate threads + + ExecutorService executor = Executors.newFixedThreadPool(fCount); + List<Future<HashMap<String,String>>> list = new ArrayList<Future<HashMap<String,String>>>(); + for( int i=0; i < fCount; i++ ){ + File f = snapFilesArr.get(i); + String fname = f.getName(); + String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname; + Thread.sleep(cArgs.staggerThreadDelay); // Stagger the threads a bit + LOGGER.debug(" -- Read file: [" + fullSnapName + "]"); + LOGGER.debug(" -- Call the PartialVertexLoader to just load vertices ----"); + LOGGER.debug(" -- vertAddDelayMs = " + vertAddDelayMs + + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs + + ", maxErrorsPerThread = " + maxErrorsPerThread ); + Callable <HashMap<String,String>> vLoader = new PartialVertexLoader(graph1, fullSnapName, + vertAddDelayMs, failureDelayMs, retryDelayMs, maxErrorsPerThread, LOGGER); + Future <HashMap<String,String>> future = (Future<HashMap<String, String>>) executor.submit(vLoader); + + // add Future to the list, we can get return value using Future + list.add(future); + LOGGER.debug(" -- Starting PartialDbLoad VERT_ONLY thread # "+ i ); + } + + int threadCount4Reload = 0; + int threadFailCount = 0; + for(Future<HashMap<String,String>> fut : list){ + threadCount4Reload++; + try { + old2NewVertIdMap.putAll(fut.get()); + LOGGER.debug(" -- back from PartialVertexLoader. returned thread # " + threadCount4Reload + + ", current size of old2NewVertMap is: " + old2NewVertIdMap.size() ); + } + catch (InterruptedException e) { + threadFailCount++; + AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); + ErrorLogHelper.logException(ae); + } + catch (ExecutionException e) { + threadFailCount++; + AAIException ae = new AAIException("AAI_6128", e , "ExecutionException"); + ErrorLogHelper.logException(ae); + } + } + executor.shutdown(); + + if( threadFailCount > 0 ) { + String emsg = " FAILURE >> " + threadFailCount + " Vertex-loader thread(s) failed to complete successfully. "; + LOGGER.debug(emsg); + throw new Exception( emsg ); + } + + long timeX = System.nanoTime(); + long diffTime = timeX - timeStart; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To reload just the vertex ids from the snapshot files, it took: " + + minCount + " minutes, " + secCount + " seconds " ); + + // Give the DB a little time to chew on all those vertices + Thread.sleep(vertToEdgeProcDelay); + + // ---------------------------------------------------------------------------------------- + LOGGER.debug("\n\n\n -- Now do the edges/props ----------------------"); + // ---------------------------------------------------------------------------------------- + + // We're going to try loading in the edges and missing properties + // Note - we're passing the whole oldVid2newVid mapping to the PartialPropAndEdgeLoader + // so that the String-updates to the GraphSON will happen in the threads instead of + // here in the un-threaded calling method. + executor = Executors.newFixedThreadPool(fCount); + ArrayList<Future<ArrayList<String>>> listEdg = new ArrayList<Future<ArrayList<String>>>(); + for( int i=0; i < fCount; i++ ){ + File f = snapFilesArr.get(i); + String fname = f.getName(); + String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname; + Thread.sleep(cArgs.staggerThreadDelay); // Stagger the threads a bit + LOGGER.debug(" -- Read file: [" + fullSnapName + "]"); + LOGGER.debug(" -- Call the PartialPropAndEdgeLoader4HistInit for Properties and EDGEs ----"); + LOGGER.debug(" -- edgeAddDelayMs = " + vertAddDelayMs + + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs + + ", maxErrorsPerThread = " + maxErrorsPerThread ); + + + Callable eLoader = new PartialPropAndEdgeLoader4HistInit(graph1, fullSnapName, + edgeAddDelayMs, failureDelayMs, retryDelayMs, + old2NewVertIdMap, maxErrorsPerThread, LOGGER, + scriptStartTime, nodeKeyNames); + Future <ArrayList<String>> future = (Future<ArrayList<String>>) executor.submit(eLoader); + + //add Future to the list, we can get return value using Future + listEdg.add(future); + LOGGER.debug(" -- Starting PartialPropAndEdge thread # "+ i ); + } + threadCount4Reload = 0; + for(Future<ArrayList<String>> fut : listEdg){ + threadCount4Reload++; + try{ + fut.get(); // DEBUG -- should be doing something with the return value if it's not empty - ie. errors + LOGGER.debug(" -- back from PartialPropAndEdgeLoader. thread # " + threadCount4Reload ); + } + catch (InterruptedException e) { + threadFailCount++; + AAIException ae = new AAIException("AAI_6128", e , "InterruptedException"); + ErrorLogHelper.logException(ae); + } + catch (ExecutionException e) { + threadFailCount++; + AAIException ae = new AAIException("AAI_6128", e , "ExecutionException"); + ErrorLogHelper.logException(ae); + } + } + + executor.shutdown(); + if( threadFailCount > 0 ) { + String emsg = " FAILURE >> " + threadFailCount + " Property/Edge-loader thread(s) failed to complete successfully. "; + LOGGER.debug(emsg); + throw new Exception( emsg ); + } + + // This is needed so we can see the data committed by the called threads + graph1.tx().commit(); + + long timeEnd = System.nanoTime(); + diffTime = timeEnd - timeX; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To reload the edges and properties from snapshot files, it took: " + + minCount + " minutes, " + secCount + " seconds " ); + + long totalDiffTime = timeEnd - timeStart; + long totalMinCount = TimeUnit.NANOSECONDS.toMinutes(totalDiffTime); + long totalSecCount = TimeUnit.NANOSECONDS.toSeconds(totalDiffTime) - (60 * totalMinCount); + LOGGER.debug(" -- TOTAL multi-threaded reload time: " + + totalMinCount + " minutes, " + totalSecCount + " seconds " ); + } else if (command.equals("CLEAR_ENTIRE_DATABASE")) { + // ------------------------------------------------------------------ + // They are calling this to clear the db before re-loading it + // later + // ------------------------------------------------------------------ + LOGGER.debug(" Command = " + command ); + // First - make sure the backup file(s) they will be using can be + // found and has(have) data. + // getFilesToProcess makes sure the file(s) exist and have some data. + getFilesToProcess(targetDir, oldSnapshotFileName, true); + LOGGER.debug(" WARNING <<<< "); + LOGGER.debug(">>> All data and schema in this database will be removed at this point. <<<"); + LOGGER.debug(">>> Processing will begin in 5 seconds. <<<"); + LOGGER.debug(">>> WARNING <<<< "); + + try { + // Give them a chance to back out of this + Thread.sleep(5000); + } catch (java.lang.InterruptedException ie) { + LOGGER.debug(" DB Clearing has been aborted. "); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + LOGGER.debug(" Begin clearing out old data. "); + String rtConfig = AAIConstants.REALTIME_DB_CONFIG; + String serviceName = System.getProperty("aai.service.name", DataSnapshot4HistInit.class.getSimpleName()); + LOGGER.debug("Getting new configs for clearig"); + + PropertiesConfiguration propertiesConfiguration = new AAIGraphConfig.Builder(rtConfig).forService(serviceName).withGraphType(REALTIME_DB).buildConfiguration(); + LOGGER.debug("Open New Janus Graph"); + + JanusGraph janusGraph = JanusGraphFactory.open(propertiesConfiguration); + verifyGraph(janusGraph); + GraphAdminDBUtils.logConfigs(janusGraph.configuration()); + janusGraph.close(); + JanusGraphCleanup.clear(janusGraph); + LOGGER.debug(" Done clearing data. "); + LOGGER.debug(">>> IMPORTANT - NOTE >>> you need to run the SchemaGenerator (use GenTester) before "); + LOGGER.debug(" reloading data or the data will be put in without indexes. "); + dbClearFlag = true; + LOGGER.debug("All done clearing DB"); + + } else if (command.equals("RELOAD_DATA")) { + // --------------------------------------------------------------------------- + // They want to restore the database from either a single file, or a group + // of snapshot files. Either way, this command will restore via single + // threaded processing. + // --------------------------------------------------------------------------- + LOGGER.debug(" Command = " + command ); + verifyGraph(AAIGraph.getInstance().getGraph()); + graph = AAIGraph.getInstance().getGraph(); + GraphAdminDBUtils.logConfigs(graph.configuration()); + if (oldSnapshotFileName.equals("")) { + String emsg = "No oldSnapshotFileName passed to DataSnapshot when RELOAD_DATA used."; + LOGGER.debug(emsg); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + long timeA = System.nanoTime(); + + ArrayList <File> snapFilesArr = new ArrayList <File> (); + + // First, see if this is a single file (ie. no ".P#" suffix) + String onePieceSnapshotFname = targetDir + AAIConstants.AAI_FILESEP + oldSnapshotFileName; + File sf = new File(onePieceSnapshotFname); + if( sf.exists() ){ + snapFilesArr.add(sf); + } + else { + // If it's a multi-part snapshot, then collect all the files for it + String thisSnapPrefix = oldSnapshotFileName + ".P"; + File fDir = new File(targetDir); // Snapshot directory + File[] allFilesArr = fDir.listFiles(); + for (File snapFile : allFilesArr) { + String snapFName = snapFile.getName(); + if( snapFName.startsWith(thisSnapPrefix)){ + snapFilesArr.add(snapFile); + } + } + } + + if( snapFilesArr.isEmpty() ){ + String emsg = "oldSnapshotFile " + onePieceSnapshotFname + "(with or without .P0) could not be found."; + LOGGER.debug(emsg); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + int fCount = snapFilesArr.size(); + Vector<InputStream> inputStreamsV = new Vector<>(); + for( int i = 0; i < fCount; i++ ){ + File f = snapFilesArr.get(i); + String fname = f.getName(); + if (!f.canRead()) { + String emsg = "oldSnapshotFile " + fname + " could not be read."; + LOGGER.debug(emsg); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } else if (f.length() == 0) { + String emsg = "oldSnapshotFile " + fname + " had no data."; + LOGGER.debug(emsg); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + String fullFName = targetDir + AAIConstants.AAI_FILESEP + fname; + InputStream fis = new FileInputStream(fullFName); + inputStreamsV.add(fis); + } + + // Now add inputStreams.elements() to the Vector, + // inputStreams.elements() will return Enumerations + InputStream sis = new SequenceInputStream(inputStreamsV.elements()); + LOGGER.debug("Begin loading data from " + fCount + " files -----"); + if("gryo".equalsIgnoreCase(snapshotType)){ + graph.io(IoCore.gryo()).reader().create().readGraph(sis, graph); + } else { + graph.io(IoCore.graphson()).reader().create().readGraph(sis, graph); + } + LOGGER.debug("Completed the inputGraph command, now try to commit()... "); + graph.tx().commit(); + LOGGER.debug("Completed reloading data."); + + long vCount = graph.traversal().V().count().next(); + LOGGER.debug("A little after repopulating from an old snapshot, we see: " + vCount + " vertices in the db."); + + long timeB = System.nanoTime(); + long diffTime = timeB - timeA; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" -- To Reload this snapshot, it took: " + + minCount + " minutes, " + secCount + " seconds " ); + + LOGGER.debug("A little after repopulating from an old snapshot, we see: " + vCount + " vertices in the db."); + + } else { + String emsg = "Bad command passed to DataSnapshot: [" + command + "]"; + LOGGER.debug(emsg); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + } catch (AAIException e) { + AAIException ae = new AAIException("AAI_6128", e , "Encountered an exception during the datasnapshot:"+e.getMessage()); + ErrorLogHelper.logException(ae); + success = false; + } catch (Exception ex) { + AAIException ae = new AAIException("AAI_6128", ex , "Encountered an exception during the datasnapshot"); + ErrorLogHelper.logException(ae); + success = false; + } finally { + if (!dbClearFlag && graph != null && !MIGRATION_PROCESS_NAME.equalsIgnoreCase(source)) { + // Any changes that worked correctly should have already done + // thier commits. + if(!"true".equals(System.getProperty("org.onap.aai.graphadmin.started"))) { + if (graph.isOpen()) { + graph.tx().rollback(); + graph.close(); + } + } + } + try { + baos.close(); + } catch (IOException iox) { + } + } + + return success; + } + + public static HashMap <String,ArrayList<String>> getNodeKeyNames() { + + HashMap <String,ArrayList<String>> keyNameHash = new HashMap <String,ArrayList<String>> (); + Loader loader = LoaderUtil.getLatestVersion(); + Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet(); + // Get a collection of the names of the key properties for each nodeType + for (Entry<String, Introspector> entry : entrySet) { + String nType = entry.getKey(); + Set <String> keyPropsSet = entry.getValue().getKeys(); + ArrayList <String> keyProps = new ArrayList <String> (); + keyProps.addAll(keyPropsSet); + keyNameHash.put(nType, keyProps); + } + return keyNameHash; + } + + + private static ArrayList <File> getFilesToProcess(String targetDir, String oldSnapshotFileName, boolean doingClearDb) + throws Exception { + + if( oldSnapshotFileName == null || oldSnapshotFileName.equals("") ){ + String emsg = "No oldSnapshotFileName passed to DataSnapshot for Reload. "; + if( doingClearDb ) { + emsg = "No oldSnapshotFileName passed to DataSnapshot. Needed when Clearing the db in case we need a backup. "; + } + LOGGER.debug(emsg); + throw new Exception( emsg ); + } + + ArrayList <File> snapFilesArrList = new ArrayList <File> (); + + // First, we'll assume that this is a multi-file snapshot and + // look for names based on that. + String thisSnapPrefix = oldSnapshotFileName + ".P"; + File fDir = new File(targetDir); // Snapshot directory + File[] allFilesArr = fDir.listFiles(); + for (File snapFile : allFilesArr) { + String snapFName = snapFile.getName(); + if( snapFName.startsWith(thisSnapPrefix)){ + if (!snapFile.canRead()) { + String emsg = "oldSnapshotFile " + snapFName + " could not be read."; + LOGGER.debug(emsg); + throw new Exception (emsg); + } else if (snapFile.length() == 0) { + String emsg = "oldSnapshotFile " + snapFName + " had no data."; + LOGGER.debug(emsg); + throw new Exception (emsg); + } + snapFilesArrList.add(snapFile); + } + } + + if( snapFilesArrList.isEmpty() ){ + // Multi-file snapshot check did not find files, so this may + // be a single-file snapshot. + String oldSnapshotFullFname = targetDir + AAIConstants.AAI_FILESEP + oldSnapshotFileName; + File f = new File(oldSnapshotFullFname); + if (!f.exists()) { + String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be found."; + LOGGER.debug(emsg); + throw new Exception (emsg); + } else if (!f.canRead()) { + String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be read."; + LOGGER.debug(emsg); + throw new Exception (emsg); + } else if (f.length() == 0) { + String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " had no data."; + LOGGER.debug(emsg); + throw new Exception (emsg); + } + snapFilesArrList.add(f); + } + + if( snapFilesArrList.isEmpty() ){ + // Still haven't found anything.. that was not a good file name. + String fullFName = targetDir + AAIConstants.AAI_FILESEP + thisSnapPrefix; + String emsg = "oldSnapshotFile " + fullFName + "* could not be found."; + LOGGER.debug(emsg); + throw new Exception(emsg); + } + + return snapFilesArrList; + } + + + public static void verifyGraph(JanusGraph graph) { + + if (graph == null) { + String emsg = "Not able to get a graph object in DataSnapshot.java\n"; + LOGGER.debug(emsg); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + } + + + public static int figureOutFileCount( long totalVertCount, int threadCount4Create, + long maxNodesPerFile ) { + + // NOTE - we would always like to use all of our threads. That is, if + // we could process all the data with 16 threads, but our threadCount4Create is + // only 15, we will do two passes and use all 15 threads each pass which will + // create a total of 30 files. Each file will be a bit smaller so the overall + // time for the two passes should be faster. + if( totalVertCount <= 0 || threadCount4Create <= 0 || maxNodesPerFile <= 0) { + return 1; + } + + long maxNodesPerPass = threadCount4Create * maxNodesPerFile; + int numberOfPasses = (int) Math.ceil( (double)totalVertCount / (double)maxNodesPerPass); + int fileCt = threadCount4Create * numberOfPasses; + + return fileCt; + } + + + class CommandLineArgs { + + @Parameter(names = "--help", help = true) + public boolean help; + + @Parameter(names = "-c", description = "command for taking data snapshot") + public String command = "JUST_TAKE_SNAPSHOT"; + + @Parameter(names = "-f", description = "previous snapshot file to reload") + public String oldFileName = ""; + + @Parameter(names = "-snapshotType", description = "snapshot type of gryo or graphson") + public String snapshotType = "graphson"; + + @Parameter(names = "-threadCount", description = "thread count for create") + public int threadCount = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_THREADS_FOR_CREATE; + + @Parameter(names = "-maxNodesPerFile", description = "Max nodes per file") + public long maxNodesPerFile = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_NODES_PER_FILE_FOR_CREATE; + + @Parameter(names = "-debugFlag", description = "DEBUG flag") + public String debugFlag = ""; + + @Parameter(names = "-debugAddDelayTime", description = "delay in ms between each Add for debug mode") + public long debugAddDelayTime = 1L; + + @Parameter(names = "-vertAddDelayMs", description = "delay in ms while adding each vertex") + public long vertAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS.longValue(); + + @Parameter(names = "-edgeAddDelayMs", description = "delay in ms while adding each edge") + public long edgeAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_EDGE_ADD_DELAY_MS.longValue(); + + @Parameter(names = "-failureDelayMs", description = "delay in ms when failure to load vertex or edge in snapshot") + public long failureDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_FAILURE_DELAY_MS.longValue(); + + @Parameter(names = "-retryDelayMs", description = "time in ms after which load snapshot is retried") + public long retryDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_FAILURE_DELAY_MS.longValue(); + + @Parameter(names = "-maxErrorsPerThread", description = "max errors allowed per thread") + public int maxErrorsPerThread = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_ERRORS_PER_THREAD; + + @Parameter(names = "-vertToEdgeProcDelay", description = "vertex to edge processing delay in ms") + public long vertToEdgeProcDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_TO_EDGE_PROC_DELAY_MS.longValue(); + + @Parameter(names = "-staggerThreadDelay", description = "thread delay stagger time in ms") + public long staggerThreadDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_STAGGER_THREAD_DELAY_MS; + + @Parameter(names = "-fileName", description = "file name for generating snapshot ") + public String fileName = ""; + + @Parameter(names = "-snapshotDir", description = "file path for generating snapshot ") + public String snapshotDir = ""; + + @Parameter(names = "-oldFileDir", description = "directory containing the old snapshot file for reloading") + public String oldFileDir = ""; + + @Parameter(names = "-caller", description = "process invoking the dataSnapshot") + public String caller = ""; + + } + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java index 7092aa8..15fff4b 100644 --- a/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java +++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java @@ -25,61 +25,58 @@ import java.io.InputStreamReader; import java.text.SimpleDateFormat; import java.util.*; +import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog; import org.onap.aai.datagrooming.DataGrooming; import org.onap.aai.datagrooming.DataGroomingTasks; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.logging.LogFormatTools; -import org.onap.aai.logging.LoggingContext; import org.onap.aai.util.AAIConfig; +import org.onap.logging.filter.base.ONAPComponents; +import org.onap.logging.ref.slf4j.ONAPLogConstants; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; @Component @PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties") public class DataSnapshotTasks { - - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DataSnapshotTasks.class); + + private AaiScheduledTaskAuditLog auditLog; + + private static final Logger LOGGER = LoggerFactory.getLogger(DataSnapshotTasks.class); private final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); + @Scheduled(cron = "${datasnapshottasks.cron}" ) public void snapshotScheduleTask() throws AAIException, Exception { - - LoggingContext.init(); - LoggingContext.requestId(UUID.randomUUID().toString()); - LoggingContext.partnerName("AAI"); - LoggingContext.targetEntity("CronApp"); - LoggingContext.component("dataSnapshot"); - LoggingContext.serviceName("snapshotScheduleTask"); - LoggingContext.targetServiceName("snapshotScheduleTask"); - LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE); - + auditLog = new AaiScheduledTaskAuditLog(); + auditLog.logBefore("dataSnapshotTask", ONAPComponents.AAI.toString() ); if(!"true".equals(AAIConfig.get("aai.disable.check.snapshot.running", "false"))){ if(checkIfDataSnapshotIsRunning()){ - LOGGER.info("Data Snapshot is already running on the system"); + LOGGER.debug("Data Snapshot is already running on the system"); return; } } - - LOGGER.info("Started cron job dataSnapshot @ " + dateFormat.format(new Date())); + LOGGER.debug("Started cron job dataSnapshot @ " + dateFormat.format(new Date())); try { if (AAIConfig.get("aai.cron.enable.dataSnapshot").equals("true")) { String [] dataSnapshotParms = {"-c",AAIConfig.get("aai.datasnapshot.params", "JUST_TAKE_SNAPSHOT")}; - LOGGER.info("DataSnapshot Params {}", Arrays.toString(dataSnapshotParms)); + LOGGER.debug("DataSnapshot Params {}", Arrays.toString(dataSnapshotParms)); DataSnapshot.main(dataSnapshotParms); } } catch (Exception e) { - ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataSnapshot"+e.toString()); - LOGGER.info("AAI_4000", "Exception running cron job for DataSnapshot"+e.toString()); - throw e; + ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataSnapshot"+LogFormatTools.getStackTop(e)); + LOGGER.debug("AAI_4000", "Exception running cron job for DataSnapshot"+LogFormatTools.getStackTop(e)); } finally { - LOGGER.info("Ended cron job dataSnapshot @ " + dateFormat.format(new Date())); - LoggingContext.clear(); + LOGGER.debug("Ended cron job dataSnapshot @ " + dateFormat.format(new Date())); } + auditLog.logAfter(); } @@ -98,9 +95,9 @@ public class DataSnapshotTasks { } int exitVal = process.waitFor(); - LOGGER.info("Exit value of the dataSnapshot check process: " + exitVal); + LOGGER.debug("Exit value of the dataSnapshot check process: " + exitVal); } catch (Exception e) { - LOGGER.error("Exception in checkIfDataSnapshotIsRunning" + LogFormatTools.getStackTop(e)); + ErrorLogHelper.logError("AAI_4000", "Exception in checkIfDataSnapshotIsRunning" + LogFormatTools.getStackTop(e)); } return count > 0; diff --git a/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java index 0f03ee0..6f9065a 100644 --- a/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java +++ b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java @@ -31,14 +31,15 @@ import org.apache.tinkerpop.gremlin.structure.Edge; import org.json.JSONArray; import org.json.JSONObject; -import com.att.eelf.configuration.EELFLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PartialPropAndEdgeLoader implements Callable <ArrayList<String>>{ - private EELFLogger LOGGER; + private Logger LOGGER; private JanusGraph jg; private String fName; @@ -51,7 +52,7 @@ public class PartialPropAndEdgeLoader implements Callable <ArrayList<String>>{ public PartialPropAndEdgeLoader (JanusGraph graph, String fn, Long edgeDelay, Long failureDelay, Long retryDelay, - HashMap<String,String> vidMap, int maxErrors, EELFLogger elfLog ){ + HashMap<String,String> vidMap, int maxErrors, Logger elfLog ){ jg = graph; fName = fn; edgeAddDelayMs = edgeDelay; diff --git a/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader4HistInit.java b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader4HistInit.java new file mode 100644 index 0000000..fa8483e --- /dev/null +++ b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader4HistInit.java @@ -0,0 +1,454 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.datasnapshot; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.concurrent.Callable; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.janusgraph.core.JanusGraph; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.json.JSONArray; +import org.json.JSONObject; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + + + +public class PartialPropAndEdgeLoader4HistInit implements Callable <ArrayList<String>>{ + + private Logger LOGGER; + + private JanusGraph jg; + private String fName; + private Long edgeAddDelayMs; + private Long retryDelayMs; + private Long failureDelayMs; + private HashMap<String,String> old2NewVidMap; + private int maxAllowedErrors; + private long currentTs; + private HashMap <String,ArrayList<String>> keyNamesHash; + + + public PartialPropAndEdgeLoader4HistInit (JanusGraph graph, String fn, Long edgeDelay, Long failureDelay, Long retryDelay, + HashMap<String,String> vidMap, int maxErrors, Logger elfLog, + Long cTs, HashMap <String,ArrayList<String>> kNHash){ + jg = graph; + fName = fn; + edgeAddDelayMs = edgeDelay; + failureDelayMs = failureDelay; + retryDelayMs = retryDelay; + old2NewVidMap = vidMap; + maxAllowedErrors = maxErrors; + LOGGER = elfLog; + currentTs = cTs; + keyNamesHash = kNHash; + } + + + public ArrayList<String> call() throws Exception { + + // This is a partner to the "PartialVertexLoader" code. + // That code loads in vertex-id's/vertex-label's for a + // multi-file data snapshot. + // This code assumes that the all vertex-id's are now in the target db. + // This code loads vertex properties and edges for a + // multi-file data snapshot (the same one that loaded + // the vertex-ids). + // + + + // NOTE - We will be loading parameters and edges for one node at a time so that problems can be + // identified or ignored or re-tried instead of causing the entire load to fail. + // + // Return an arrayList of Strings to give info on what nodes encountered problems + + int entryCount = 0; + int retryCount = 0; + int failureCount = 0; + int retryFailureCount = 0; + HashMap <String,String> failedAttemptHash = new HashMap <String,String> (); + ArrayList <String> failedAttemptInfo = new ArrayList <String> (); + + int passNum = 1; + try( BufferedReader br = new BufferedReader(new FileReader(fName))) { + // loop through the file lines and do PUT for each vertex or the edges depending on what the loadtype is + for(String origLine; (origLine = br.readLine()) != null; ) { + entryCount++; + Thread.sleep(edgeAddDelayMs); // Space the edge requests out a little + + String errInfoStr = processThisLine(origLine, passNum); + if( !errInfoStr.equals("") ){ + // There was a problem with this line + String vidStr = getTheVidForThisLine(origLine); + // We'll use the failedAttemptHash to reTry this item + failedAttemptHash.put(vidStr,origLine); + failedAttemptInfo.add(errInfoStr); + failureCount++; + if( failureCount > maxAllowedErrors ) { + LOGGER.debug(">>> Abandoning PartialPropAndEdgeLoader() because " + + "Max Allowed Error count was exceeded for this thread. (max = " + + maxAllowedErrors + ". "); + throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". "); + } + Thread.sleep(failureDelayMs); // take a little nap if it failed + } + } // End of looping over each line + if( br != null ){ + br.close(); + } + } + catch (Exception e) { + LOGGER.debug(" --- Failed in the main loop for Buffered-Reader item # " + entryCount + + ", fName = " + fName ); + LOGGER.debug(" --- msg = " + e.getMessage() ); + throw e; + } + + // --------------------------------------------------------------------------- + // Now Re-Try any failed requests that might have Failed on the first pass. + // --------------------------------------------------------------------------- + passNum++; + try { + for (String failedVidStr : failedAttemptHash.keySet()) { + // Take a little nap, and retry this failed attempt + LOGGER.debug("DEBUG >> We will sleep for " + retryDelayMs + " and then RETRY any failed edge/property ADDs. "); + Thread.sleep(retryDelayMs); + retryCount++; + Long failedVidL = Long.parseLong(failedVidStr); + // When an Edge/Property Add fails, we store the whole (translated) graphSON line as the data in the failedAttemptHash + // We're really just doing a GET of this one vertex here... + String jsonLineToRetry = failedAttemptHash.get(failedVidStr); + String errInfoStr = processThisLine(jsonLineToRetry, passNum); + if( !errInfoStr.equals("") ){ + // There was a problem with this line + String translatedVidStr = getTheVidForThisLine(jsonLineToRetry); + failedAttemptHash.put(translatedVidStr,jsonLineToRetry); + failedAttemptInfo.add(errInfoStr); + retryFailureCount++; + if( retryFailureCount > maxAllowedErrors ) { + LOGGER.debug(">>> Abandoning PartialPropAndEdgeLoader() because " + + "Max Allowed Error count was exceeded while doing retries for this thread. (max = " + + maxAllowedErrors + ". "); + throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". "); + } + Thread.sleep(failureDelayMs); // take a little nap if it failed + } + } // End of looping over each failed line + } + catch (Exception e) { + LOGGER.debug(" -- error in RETRY block. ErrorMsg = [" + e.getMessage() + "]" ); + throw e; + } + + LOGGER.debug(">>> After Processing in PartialPropAndEdgeLoader() " + + entryCount + " records processed. " + failureCount + " records failed. " + + retryCount + " RETRYs processed. " + retryFailureCount + " RETRYs failed. "); + + return failedAttemptInfo; + + }// end of call() + + + + private String translateThisVid(String oldVid) throws Exception { + + if( old2NewVidMap == null ){ + throw new Exception(" ERROR - null old2NewVidMap found in translateThisVid. "); + } + + if( old2NewVidMap.containsKey(oldVid) ){ + return old2NewVidMap.get(oldVid); + } + else { + throw new Exception(" ERROR - could not find VID translation for original VID = " + oldVid ); + } + } + + + private String getTheVidForThisLine(String graphSonLine) throws Exception { + + if( graphSonLine == null ){ + throw new Exception(" ERROR - null graphSonLine passed to getTheVidForThisLine. "); + } + + // We are assuming that the graphSonLine has the vertexId as the first ID: + // {"id":100995128,"label":"vertex","inE":{"hasPinterface":[{"id":"7lgg0e-2... etc... + + // The vertexId for this line is the numeric part after the initial {"id":xxxxx up to the first comma + int x = graphSonLine.indexOf(':') + 1; + int y = graphSonLine.indexOf(','); + String initialVid = graphSonLine.substring(x,y); + if( initialVid != null && !initialVid.isEmpty() && initialVid.matches("^[0-9]+$") ){ + return initialVid; + } + else { + throw new Exception(" ERROR - could not determine initial VID for graphSonLine: " + graphSonLine ); + } + } + + + private String processThisLine(String graphSonLine, int passNum){ + + String passInfo = ""; + if( passNum > 1 ) { + passInfo = " >> RETRY << pass # " + passNum + " "; + } + + JSONObject jObj = new JSONObject(); + String originalVid = ""; + + try{ + jObj = new JSONObject(graphSonLine); + originalVid = jObj.get("id").toString(); + } + catch ( Exception e ){ + LOGGER.debug(" -- Could not convert line to JsonObject [ " + graphSonLine + "]" ); + LOGGER.debug(" -- ErrorMsg = [" +e.getMessage() + "]"); + + return(" JSON translation or getVid exception when processing this line [" + graphSonLine + "]"); + } + + // ----------------------------------------------------------------------------------------- + // Note - this assumes that any vertices referred to by an edge will already be in the DB. + // ----------------------------------------------------------------------------------------- + Vertex dbVtx = null; + + String newVidStr = ""; + Long newVidL = 0L; + try { + newVidStr = translateThisVid(originalVid); + newVidL = Long.parseLong(newVidStr); + } + catch ( Exception e ){ + LOGGER.debug(" -- " + passInfo + " translate VertexId before adding edges failed for this: vtxId = " + + originalVid + ". ErrorMsg = [" +e.getMessage() + "]"); + + return(" VID-translation error when processing this line ---"); + } + + try { + dbVtx = getVertexFromDbForVid(newVidStr); + } + catch ( Exception e ){ + LOGGER.debug(" -- " + passInfo + " READ Vertex from DB before adding edges failed for this: vtxId = " + originalVid + + ", newVidId = " + newVidL + ". ErrorMsg = [" +e.getMessage() + "]"); + + return(" ERROR getting Vertex based on VID = " + newVidStr + "]"); + } + + + String edResStr = processEdgesForVtx( jObj, dbVtx, passInfo, originalVid ); + if( edResStr.equals("") ){ + // We will commit the edges by themselves in case the properties stuff below fails + try { + jg.tx().commit(); + } + catch ( Exception e ){ + LOGGER.debug(" -- " + passInfo + " COMMIT FAILED adding EDGES for this vertex: vtxId = " + + originalVid + ". ErrorMsg = [" +e.getMessage() + "]"); + return(" ERROR with committing edges for vertexId = " + originalVid ); + } + } + + // Add the properties that we didn't have when we added the 'bare-bones' vertex + String pResStr = processPropertiesForVtx( jObj, dbVtx, passInfo, originalVid ); + if( pResStr.equals("") ){ + try { + jg.tx().commit(); + return ""; + } + catch ( Exception e ){ + LOGGER.debug(" -- " + passInfo + " COMMIT FAILED adding Properties for this vertex: vtxId = " + + originalVid + ". ErrorMsg = [" +e.getMessage() + "]"); + return(" ERROR with committing properties for vertexId = " + originalVid ); + } + } + else { + LOGGER.debug("DEBUG " + passInfo + " Error processing Properties for this vertex: vtxId = " + + originalVid + ", [" + pResStr + "]"); + return(" ERROR processing properties for vertexId = " + originalVid + ", [" + pResStr + "]"); + } + } + + + private String processPropertiesForVtx( JSONObject jObj, Vertex dbVtx, String passInfo, String originalVid ){ + + try { + JSONObject propsOb = (JSONObject) jObj.get("properties"); + String thisVNType = "unknown"; + String thisVSot = "N/A"; + Iterator <String> propsItr = propsOb.keys(); + // First, loop through to get the nodeType and the node-level source-of-truth + while( propsItr.hasNext() ){ + String pKey = propsItr.next(); + JSONArray propsDetArr = propsOb.getJSONArray(pKey); + for( int i=0; i< propsDetArr.length(); i++ ){ + JSONObject prop = propsDetArr.getJSONObject(i); + if( pKey.equals("aai-node-type")) { + thisVNType = (String) prop.get("value"); + } + else if( pKey.equals("source-of-truth") ){ + thisVSot = (String) prop.get("value"); + } + } + } + + // Get the key(s) for this nodeType + ArrayList <String> popSotArr = new ArrayList<String>(); + if( keyNamesHash.containsKey(thisVNType) ) { + // This one is the node's key prop, so it will get SOT. + popSotArr = keyNamesHash.get(thisVNType); + } + // other required properties that we can give the SOT + popSotArr.add("aai-node-type"); + popSotArr.add("aai-uri"); + popSotArr.add("source-of-truth"); // odd, but they want it... + + // 2nd pass, add each property to the db vertex. + propsItr = propsOb.keys(); + while( propsItr.hasNext() ){ + String pKey = propsItr.next(); + JSONArray propsDetArr = propsOb.getJSONArray(pKey); + for( int i=0; i< propsDetArr.length(); i++ ){ + JSONObject prop = propsDetArr.getJSONObject(i); + Object val = prop.get("value"); + String sotVal = "N/A"; // default + if( popSotArr.contains(pKey) ){ + sotVal = thisVSot; // This prop will have SOT populated. + } + dbVtx.property(pKey, val,"start-ts", currentTs,"source-of-truth",sotVal); + } + // The vertex itself gets the history data also - and since we're at + // the node level, we give "start-ts" it's own sub-properties just so it + // will match everybody else. + // Note - All vertices have always had a 'source-of-truth' property. + dbVtx.property("start-ts", currentTs,"start-ts", currentTs,"source-of-truth",thisVSot); + } + } + catch ( Exception e ){ + LOGGER.debug(" -- " + passInfo + " failure getting/setting properties for: vtxId = " + + originalVid + ". ErrorMsg = [" + e.getMessage() + "]"); + return(" error processing properties for vtxId = " + originalVid); + } + + return ""; + } + + + private Vertex getVertexFromDbForVid( String vtxIdStr ) throws Exception { + Vertex thisVertex = null; + Long vtxIdL = 0L; + + try { + vtxIdL = Long.parseLong(vtxIdStr); + Iterator <Vertex> vItr = jg.vertices(vtxIdL); + // Note - we only expect to find one vertex found for this ID. + while( vItr.hasNext() ){ + thisVertex = vItr.next(); + } + } + catch ( Exception e ){ + String emsg = "Error finding vertex for vid = " + vtxIdStr + "[" + e.getMessage() + "]"; + throw new Exception ( emsg ); + } + + if( thisVertex == null ){ + String emsg = "Could not find vertex for passed vid = " + vtxIdStr; + throw new Exception ( emsg ); + } + + return thisVertex; + } + + + private String processEdgesForVtx( JSONObject jObj, Vertex dbVtx, String passInfo, String originalVid ){ + + // Process the edges for this vertex -- but, just the "OUT" ones so edges don't get added twice (once from + // each side of the edge). + JSONObject edOb = null; + try { + edOb = (JSONObject) jObj.get("outE"); + } + catch (Exception e){ + // There were no OUT edges. This is OK. + return ""; + } + + try { + if( edOb == null ){ + // There were no OUT edges. This is OK. Not all nodes have out edges. + return ""; + } + Iterator <String> edItr = edOb.keys(); + while( edItr.hasNext() ){ + String eLabel = edItr.next(); + JSONArray edArr = edOb.getJSONArray(eLabel); + for( int i=0; i< edArr.length(); i++ ){ + JSONObject eObj = edArr.getJSONObject(i); + String inVidStr = eObj.get("inV").toString(); + String translatedInVidStr = translateThisVid(inVidStr); + Vertex newInVertex = getVertexFromDbForVid(translatedInVidStr); + + // Note - addEdge automatically adds the edge in the OUT direction from the + // 'anchor' node that the call is being made from. + Edge tmpE = dbVtx.addEdge(eLabel, newInVertex); + JSONObject ePropsOb = null; + try { + ePropsOb = (JSONObject) eObj.get("properties"); + } + catch (Exception e){ + // NOTE - model definition related edges do not have edge properties. That is OK. + // Ie. when a model-element node has an "isA" edge to a "model-ver" node, that edge does + // not have edge properties on it. + } + if( ePropsOb != null ){ + Iterator <String> ePropsItr = ePropsOb.keys(); + while( ePropsItr.hasNext() ){ + String pKey = ePropsItr.next(); + tmpE.property(pKey, ePropsOb.get(pKey)); + } + // For History add start-ts and source-of-truth + tmpE.property("start-ts", currentTs); + tmpE.property("source-of-truth", "N/A"); + } + } + } + } + catch ( Exception e ){ + String msg = " -- " + passInfo + " failure adding edge for: original vtxId = " + + originalVid + ". ErrorMsg = [" +e.getMessage() + "]"; + LOGGER.debug( " -- " + msg ); + LOGGER.debug(" -- now going to return/bail out of processEdgesForVtx" ); + return(" >> " + msg ); + } + + return ""; + } + + +} + + diff --git a/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java b/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java index 3afd295..22ca0a2 100644 --- a/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java +++ b/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java @@ -26,8 +26,10 @@ import java.util.concurrent.Callable; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; import org.janusgraph.core.JanusGraph; - -import com.att.eelf.configuration.EELFLogger; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.logging.ErrorLogHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -35,7 +37,7 @@ import com.google.gson.JsonParser; public class PartialVertexLoader implements Callable<HashMap<String,String>>{ - private EELFLogger LOGGER; + private Logger LOGGER; private JanusGraph jg; private String fName; @@ -45,7 +47,7 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{ private int maxAllowedErrors; public PartialVertexLoader (JanusGraph graph, String fn, Long vertDelay, Long failurePause, - Long retryDelay, int maxErrors, EELFLogger elfLog ){ + Long retryDelay, int maxErrors, Logger elfLog ){ jg = graph; fName = fn; vertAddDelayMs = vertDelay; @@ -137,7 +139,8 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{ LOGGER.debug(" --- Failed in the main loop for Buffered-Reader item # " + entryCount + ", fName = " + fName ); LOGGER.debug(" --- msg = " + e.getMessage() ); - e.printStackTrace(); + AAIException ae = new AAIException("AAI_6128", e , "Failed in the main loop for Buffered-Reader item"); + ErrorLogHelper.logException(ae); throw e; } @@ -165,8 +168,9 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{ LOGGER.debug(" -- addVertex FAILED for RETRY for vtxId = " + failedVidStr + ", label = [" + failedLabel + "]. ErrorMsg = [" +e.getMessage() + "]" ); - e.printStackTrace(); - if( retryFailureCount > maxAllowedErrors ) { + AAIException ae = new AAIException("AAI_6128", e , "addVertex FAILED for RETRY"); + ErrorLogHelper.logException(ae); + if( retryFailureCount > maxAllowedErrors ) { LOGGER.debug(">>> Abandoning PartialVertexLoader() because " + "Max Allowed Error count was exceeded for this thread. (max = " + maxAllowedErrors + ". "); @@ -189,7 +193,8 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{ + ", label = [" + failedLabel + "]. ErrorMsg = [" + e.getMessage() + "]. This vertex will not be tried again. "); - e.printStackTrace(); + AAIException ae = new AAIException("AAI_6128", e , "--POSSIBLE ERROR-- COMMIT FAILED for RETRY"); + ErrorLogHelper.logException(ae); if( retryFailureCount > maxAllowedErrors ) { LOGGER.debug(">>> Abandoning PartialVertexLoader() because " + "Max Allowed Error count was exceeded for this thread. (max = " + @@ -205,7 +210,8 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{ } catch ( Exception e ){ LOGGER.debug(" -- error in RETRY block. ErrorMsg = [" +e.getMessage() + "]" ); - e.printStackTrace(); + AAIException ae = new AAIException("AAI_6128", e , " -- error in RETRY block."); + ErrorLogHelper.logException(ae); throw e; } diff --git a/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java b/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java index 493678b..12b3099 100644 --- a/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java +++ b/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java @@ -22,100 +22,149 @@ package org.onap.aai.datasnapshot; import java.io.FileOutputStream; import java.util.ArrayList; import java.util.Iterator; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.apache.tinkerpop.gremlin.structure.io.IoCore; import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; +import org.apache.tinkerpop.gremlin.structure.io.IoCore; import org.janusgraph.core.JanusGraph; +import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.logging.ErrorLogHelper; +import org.onap.logging.filter.base.ONAPComponents; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PrintVertexDetails implements Runnable{ + private Logger LOGGER; + private JanusGraph jg; private String fname; - private ArrayList<Vertex> vtxList; + private ArrayList<Long> vtxIdList; private Boolean debugOn; private long debugDelayMs; private String snapshotType; static final byte[] newLineBytes = "\n".getBytes(); + + private AaiScheduledTaskAuditLog auditLog; + - public PrintVertexDetails (JanusGraph graph, String fn, ArrayList<Vertex> vL, Boolean debugFlag, long debugAddDelayTime, String snapshotType){ + public PrintVertexDetails (JanusGraph graph, String fn, ArrayList<Long> vIdL, Boolean debugFlag, + long debugAddDelayTime, String snapshotType, Logger elfLog){ jg = graph; fname = fn; - vtxList = vL; + vtxIdList = vIdL; debugOn = debugFlag; debugDelayMs = debugAddDelayTime; this.snapshotType = snapshotType; + LOGGER = elfLog; + this.auditLog = new AaiScheduledTaskAuditLog(); } - - public void run(){ - if( debugOn ){ - // This is much slower, but sometimes we need to find out which single line is causing a failure - try{ - int okCount = 0; - int failCount = 0; - Long debugDelayMsL = new Long(debugDelayMs); - FileOutputStream subFileStr = new FileOutputStream(fname); - Iterator <Vertex> vSubItr = vtxList.iterator(); - GraphWriter graphWriter = null; - if("gryo".equalsIgnoreCase(snapshotType)){ - graphWriter = jg.io(IoCore.gryo()).writer().create(); - } else { - graphWriter = jg.io(IoCore.graphson()).writer().create(); - } - while( vSubItr.hasNext() ){ - Long vertexIdL = 0L; - String aaiNodeType = ""; - String aaiUri = ""; - String aaiUuid = ""; - try { - Vertex tmpV = vSubItr.next(); - vertexIdL = (Long) tmpV.id(); - aaiNodeType = (String) tmpV.property("aai-node-type").orElse(null); - aaiUri = (String) tmpV.property("aai-uri").orElse(null); - aaiUuid = (String) tmpV.property("aai-uuid").orElse(null); - - Thread.sleep(debugDelayMsL); // Make sure it doesn't bump into itself - graphWriter.writeVertex(subFileStr, tmpV, Direction.BOTH); - subFileStr.write(newLineBytes); - okCount++; + + + public void run(){ + LOGGER = LoggerFactory.getLogger(PrintVertexDetails.class); + auditLog.logBefore("printVertexDetails", ONAPComponents.AAI.toString()); + try { + if (debugOn) { + // This is much slower, but sometimes we need to find out which single line is + // causing a failure + try { + int okCount = 0; + int failCount = 0; + Long debugDelayMsL = new Long(debugDelayMs); + FileOutputStream subFileStr = new FileOutputStream(fname); + + GraphWriter graphWriter = null; + if ("gryo".equalsIgnoreCase(snapshotType)) { + graphWriter = jg.io(IoCore.gryo()).writer().create(); + } else { + graphWriter = jg.io(IoCore.graphson()).writer().create(); } - catch(Exception e) { - failCount++; - System.out.println(" >> DEBUG MODE >> Failed at: VertexId = [" + vertexIdL + - "], aai-node-type = [" + aaiNodeType + - "], aai-uuid = [" + aaiUuid + - "], aai-uri = [" + aaiUri + "]. " ); - e.printStackTrace(); + + GraphTraversalSource gts = jg.traversal(); + ArrayList<Vertex> vtxList = new ArrayList<Vertex> (); + GraphTraversal<Vertex, Vertex> gt = gts.V(vtxIdList); + while( gt.hasNext() ) { + vtxList.add(gt.next()); } + Iterator<Vertex> vSubItr = vtxList.iterator(); + while (vSubItr.hasNext()) { + Long vertexIdL = 0L; + String aaiNodeType = ""; + String aaiUri = ""; + String aaiUuid = ""; + try { + Vertex tmpV = vSubItr.next(); + vertexIdL = (Long) tmpV.id(); + aaiNodeType = (String) tmpV.property("aai-node-type").orElse(null); + aaiUri = (String) tmpV.property("aai-uri").orElse(null); + aaiUuid = (String) tmpV.property("aai-uuid").orElse(null); + + Thread.sleep(debugDelayMsL); // Make sure it doesn't bump into itself + graphWriter.writeVertex(subFileStr, tmpV, Direction.BOTH); + subFileStr.write(newLineBytes); + okCount++; + } catch (Exception e) { + failCount++; + String fmsg = " >> DEBUG MODE >> Failed at: VertexId = [" + vertexIdL + + "], aai-node-type = [" + aaiNodeType + "], aai-uuid = [" + aaiUuid + + "], aai-uri = [" + aaiUri + "]. "; + System.out.println(fmsg); + LOGGER.debug(" PrintVertexDetails " + fmsg); + // e.printStackTrace(); + } + } + System.out.println(" -- Printed " + okCount + " vertexes out to " + fname + ", with " + failCount + + " failed."); + subFileStr.close(); + } catch (Exception e) { + AAIException ae = new AAIException("AAI_6128", e , "Error running PrintVertexDetails in debugon"); + ErrorLogHelper.logException(ae); } - System.out.println(" -- Printed " + okCount + " vertexes out to " + fname + - ", with " + failCount + " failed."); - subFileStr.close(); - } - catch(Exception e){ - e.printStackTrace(); - } - } - else { - // Not in DEBUG mode, so we'll do all the nodes in one group - try{ - int count = vtxList.size(); - Iterator <Vertex> vSubItr = vtxList.iterator(); - FileOutputStream subFileStr = new FileOutputStream(fname); - if ("gryo".equalsIgnoreCase(snapshotType)) { - jg.io(IoCore.gryo()).writer().create().writeVertices(subFileStr, vSubItr, Direction.BOTH); - } else { - jg.io(IoCore.graphson()).writer().create().writeVertices(subFileStr, vSubItr, Direction.BOTH); + } else { + // Not in DEBUG mode, so we'll do all the nodes in one group + GraphTraversalSource gts = jg.traversal(); + ArrayList<Vertex> vtxList = new ArrayList<Vertex> (); + GraphTraversal<Vertex, Vertex> gt = gts.V(vtxIdList); + while( gt.hasNext() ) { + vtxList.add(gt.next()); + } + + try { + int count = vtxList.size(); + Iterator<Vertex> vSubItr = vtxList.iterator(); + FileOutputStream subFileStr = new FileOutputStream(fname); + if ("gryo".equalsIgnoreCase(snapshotType)) { + jg.io(IoCore.gryo()).writer().create().writeVertices(subFileStr, vSubItr, Direction.BOTH); + } else { + jg.io(IoCore.graphson()).writer().create().writeVertices(subFileStr, vSubItr, Direction.BOTH); + } + subFileStr.close(); + String pmsg = " -- Printed " + count + " vertexes out to " + fname; + System.out.println(pmsg); + LOGGER.debug(" PrintVertexDetails " + pmsg); + } catch (Exception e) { + AAIException ae = new AAIException("AAI_6128", e , "Error running PrintVertexDetails in else"); + ErrorLogHelper.logException(ae); } - subFileStr.close(); - System.out.println(" -- Printed " + count + " vertexes out to " + fname); } - catch(Exception e){ - e.printStackTrace(); - } } + catch(Exception e){ + AAIException ae = new AAIException("AAI_6128", e , "Error running PrintVertexDetails"); + ErrorLogHelper.logException(ae); + } + finally { + // Make sure the transaction this thread was using is freed up. + jg.tx().commit(); + jg.tx().close(); + } + auditLog.logAfter(); } }
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/db/schema/AuditOXM.java b/src/main/java/org/onap/aai/db/schema/AuditOXM.java index d7f7314..e976bed 100644 --- a/src/main/java/org/onap/aai/db/schema/AuditOXM.java +++ b/src/main/java/org/onap/aai/db/schema/AuditOXM.java @@ -19,8 +19,8 @@ */ package org.onap.aai.db.schema; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Multimap; import org.janusgraph.core.Cardinality; import org.janusgraph.core.Multiplicity; @@ -43,7 +43,7 @@ import java.util.stream.Collectors; public class AuditOXM extends Auditor { - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(AuditOXM.class); + private static final Logger LOGGER = LoggerFactory.getLogger(AuditOXM.class); private Set<Introspector> allObjects; private EdgeIngestor ingestor; diff --git a/src/main/java/org/onap/aai/db/schema/ScriptDriver.java b/src/main/java/org/onap/aai/db/schema/ScriptDriver.java index 6b36adc..a634e0b 100644 --- a/src/main/java/org/onap/aai/db/schema/ScriptDriver.java +++ b/src/main/java/org/onap/aai/db/schema/ScriptDriver.java @@ -33,8 +33,6 @@ import org.onap.aai.edges.EdgeIngestor; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.logging.ErrorObjectFormatException; -import org.onap.aai.logging.LoggingContext; -import org.onap.aai.logging.LoggingContext.StatusCode; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.ExceptionTranslator; @@ -56,15 +54,6 @@ public class ScriptDriver { public static void main (String[] args) throws AAIException, IOException, ConfigurationException, ErrorObjectFormatException { CommandLineArgs cArgs = new CommandLineArgs(); - LoggingContext.init(); - LoggingContext.component("DBSchemaScriptDriver"); - LoggingContext.partnerName("NA"); - LoggingContext.targetEntity("AAI"); - LoggingContext.requestId(UUID.randomUUID().toString()); - LoggingContext.serviceName("AAI"); - LoggingContext.targetServiceName("main"); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); ErrorLogHelper.loadProperties(); new JCommander(cArgs, args); @@ -84,8 +73,6 @@ public class ScriptDriver { } catch (Exception e) { AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); - LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } @@ -136,4 +123,4 @@ class CommandLineArgs { public String type = "graph"; -} +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/dbgen/DupeTool.java b/src/main/java/org/onap/aai/dbgen/DupeTool.java index 800e9a4..4164ee8 100644 --- a/src/main/java/org/onap/aai/dbgen/DupeTool.java +++ b/src/main/java/org/onap/aai/dbgen/DupeTool.java @@ -20,8 +20,8 @@ package org.onap.aai.dbgen; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; @@ -40,8 +40,6 @@ import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.introspection.ModelType; import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.logging.LogFormatTools; -import org.onap.aai.logging.LoggingContext; -import org.onap.aai.logging.LoggingContext.StatusCode; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; @@ -57,7 +55,7 @@ import java.util.Map.Entry; public class DupeTool { - private static final EELFLogger logger = EELFManager.getInstance().getLogger(DupeTool.class.getSimpleName()); + private static final Logger logger = LoggerFactory.getLogger(DupeTool.class.getSimpleName()); private static final String FROMAPPID = "AAI-DB"; private static final String TRANSID = UUID.randomUUID().toString(); @@ -87,14 +85,12 @@ public class DupeTool { public void execute(String[] args){ - String defVersion = "v16"; + String defVersion = "v18"; try { defVersion = AAIConfig.get(AAIConstants.AAI_DEFAULT_API_VERSION_PROP); } catch (AAIException ae) { String emsg = "Error trying to get default API Version property \n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(emsg); exit(0); } @@ -104,8 +100,6 @@ public class DupeTool { try { loader = loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion()); } catch (Exception ex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); logger.error("ERROR - Could not do the moxyMod.init() " + LogFormatTools.getStackTop(ex)); exit(1); } @@ -155,8 +149,6 @@ public class DupeTool { if (thisArg.equals("-nodeType")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" No value passed with -nodeType option. "); exit(0); } @@ -165,8 +157,6 @@ public class DupeTool { } else if (thisArg.equals("-sleepMinutes")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error("No value passed with -sleepMinutes option."); exit(0); } @@ -174,8 +164,6 @@ public class DupeTool { try { sleepMinutes = Integer.parseInt(nextArg); } catch (Exception e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error("Bad value passed with -sleepMinutes option: [" + nextArg + "]"); exit(0); @@ -184,8 +172,6 @@ public class DupeTool { } else if (thisArg.equals("-maxFix")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error("No value passed with -maxFix option."); exit(0); } @@ -193,8 +179,6 @@ public class DupeTool { try { maxRecordsToFix = Integer.parseInt(nextArg); } catch (Exception e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error("Bad value passed with -maxFix option: [" + nextArg + "]"); exit(0); @@ -203,8 +187,6 @@ public class DupeTool { } else if (thisArg.equals("-timeWindowMinutes")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error("No value passed with -timeWindowMinutes option."); exit(0); } @@ -212,8 +194,6 @@ public class DupeTool { try { timeWindowMinutes = Integer.parseInt(nextArg); } catch (Exception e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error("Bad value passed with -timeWindowMinutes option: [" + nextArg + "]"); exit(0); @@ -228,8 +208,6 @@ public class DupeTool { } else if (thisArg.equals("-userId")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" No value passed with -userId option. "); exit(0); } @@ -238,16 +216,12 @@ public class DupeTool { } else if (thisArg.equals("-params4Collect")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" No value passed with -params4Collect option. "); exit(0); } filterParams = args[i]; argStr4Msg = argStr4Msg + " " + filterParams; } else { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" Unrecognized argument passed to DupeTool: [" + thisArg + "]. "); logger.error(" Valid values are: -action -userId -vertexId -edgeId -overRideProtection "); @@ -260,8 +234,6 @@ public class DupeTool { if ((userIdVal.length() < 6) || userIdVal.toUpperCase().equals("AAIADMIN")) { String emsg = "userId parameter is required. [" + userIdVal + "] passed to DupeTool(). userId must be not empty and not aaiadmin \n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(emsg); exit(0); } @@ -270,8 +242,6 @@ public class DupeTool { if (nodeTypeVal.equals("")) { String emsg = " nodeType is a required parameter for DupeTool().\n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(emsg); exit(0); } else { @@ -287,7 +257,7 @@ public class DupeTool { String msg = ""; msg = "DupeTool called with these params: [" + argStr4Msg + "]"; System.out.println(msg); - logger.info(msg); + logger.debug(msg); // Determine what the key fields are for this nodeType (and we want them ordered) ArrayList<String> keyPropNamesArr = new ArrayList<String>(obj.getKeys()); @@ -311,8 +281,6 @@ public class DupeTool { } catch (AAIException ae) { String emsg = "Error trying to get initial set of nodes to check. \n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(emsg); exit(0); } @@ -321,13 +289,13 @@ public class DupeTool { msg = " No vertices found to check. Used nodeType = [" + nodeTypeVal + "], windowMinutes = " + timeWindowMinutes + ", filterData = [" + filterParams + "]."; - logger.info(msg); + logger.debug(msg); System.out.println(msg); exit(0); } else { msg = " Found " + verts2Check.size() + " nodes of type " + nodeTypeVal + " to check using passed filterParams and windowStartTime. "; - logger.info(msg); + logger.debug(msg); System.out.println(msg); } @@ -349,15 +317,15 @@ public class DupeTool { } msg = " Found " + firstPassDupeSets.size() + " sets of duplicates for this request. "; - logger.info(msg); + logger.debug(msg); System.out.println(msg); if (firstPassDupeSets.size() > 0) { msg = " Here is what they look like: "; - logger.info(msg); + logger.debug(msg); System.out.println(msg); for (int x = 0; x < firstPassDupeSets.size(); x++) { msg = " Set " + x + ": [" + firstPassDupeSets.get(x) + "] "; - logger.info(msg); + logger.debug(msg); System.out.println(msg); showNodeDetailsForADupeSet(gt1, firstPassDupeSets.get(x), logger); } @@ -367,7 +335,7 @@ public class DupeTool { ArrayList<String> dupeSetsToFix = new ArrayList<String>(); if (autoFix && firstPassDupeSets.size() == 0) { msg = "AutoFix option is on, but no dupes were found on the first pass. Nothing to fix."; - logger.info(msg); + logger.debug(msg); System.out.println(msg); } else if (autoFix) { // We will try to fix any dupes that we can - but only after sleeping for a @@ -375,13 +343,13 @@ public class DupeTool { try { msg = "\n\n----------- About to sleep for " + sleepMinutes + " minutes." + " -----------\n\n"; - logger.info(msg); + logger.debug(msg); System.out.println(msg); int sleepMsec = sleepMinutes * 60 * 1000; Thread.sleep(sleepMsec); } catch (InterruptedException ie) { msg = "\n >>> Sleep Thread has been Interrupted <<< "; - logger.info(msg); + logger.debug(msg); System.out.println(msg); exit(0); } @@ -401,16 +369,16 @@ public class DupeTool { dupeSetsToFix = figureWhichDupesStillNeedFixing(firstPassDupeSets, secondPassDupeSets, logger); msg = "\nAfter running a second pass, there were " + dupeSetsToFix.size() + " sets of duplicates that we think can be deleted. "; - logger.info(msg); + logger.debug(msg); System.out.println(msg); if (dupeSetsToFix.size() > 0) { msg = " Here is what the sets look like: "; - logger.info(msg); + logger.debug(msg); System.out.println(msg); for (int x = 0; x < dupeSetsToFix.size(); x++) { msg = " Set " + x + ": [" + dupeSetsToFix.get(x) + "] "; - logger.info(msg); + logger.debug(msg); System.out.println(msg); showNodeDetailsForADupeSet(gt2, dupeSetsToFix.get(x), logger); } @@ -425,7 +393,7 @@ public class DupeTool { + ". No nodes will be deleted. (use the" + " -maxFix option to override this limit.)"; System.out.println(infMsg); - logger.info(infMsg); + logger.debug(infMsg); } else { // Call the routine that fixes known dupes didSomeDeletesFlag = deleteNonKeepers(gt2, dupeSetsToFix, logger); @@ -500,15 +468,6 @@ public class DupeTool { props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); MDC.put("logFilenameAppender", DupeTool.class.getSimpleName()); - LoggingContext.init(); - LoggingContext.partnerName(FROMAPPID); - LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.component("dupeTool"); - LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.targetServiceName("main"); - LoggingContext.requestId(TRANSID); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); @@ -522,8 +481,6 @@ public class DupeTool { } catch (Exception e) { AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); logger.error("Problems running DupeTool "+aai.getMessage()); - LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } @@ -550,7 +507,7 @@ public class DupeTool { String fromAppId, Graph g, String version, String nType, ArrayList<Vertex> passedVertList, ArrayList<String> keyPropNamesArr, - Boolean specialTenantRule, Loader loader, EELFLogger logger) { + Boolean specialTenantRule, Loader loader, Logger logger) { ArrayList<String> returnList = new ArrayList<String>(); @@ -643,14 +600,14 @@ public class DupeTool { * @param dbMaps the db maps * @param keyPropNamesArr Array (ordered) of keyProperty names * @param specialTenantRule flag - * @param EELFLogger the logger + * @param Logger the logger * @return the array list */ private ArrayList<String> getDupeSets4DependentNodes(String transId, String fromAppId, Graph g, String version, String nType, ArrayList<Vertex> passedVertList, ArrayList<String> keyPropNamesArr, Loader loader, - Boolean specialTenantRule, EELFLogger logger) { + Boolean specialTenantRule, Logger logger) { // This is for nodeTypes that DEPEND ON A PARENT NODE FOR UNIQUNESS @@ -766,15 +723,13 @@ public class DupeTool { }// End of getDupeSets4DependentNodes() - private Graph getGraphTransaction(JanusGraph graph, EELFLogger logger) { + private Graph getGraphTransaction(JanusGraph graph, Logger logger) { Graph gt = null; try { if (graph == null) { String emsg = "could not get graph object in DupeTool. \n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); logger.error(emsg); exit(0); } @@ -787,15 +742,11 @@ public class DupeTool { } catch (AAIException e1) { String msg = e1.getErrorObject().toString(); System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(msg); exit(0); } catch (Exception e2) { String msg = e2.toString(); System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); logger.error(msg); exit(0); } @@ -805,39 +756,35 @@ public class DupeTool { }// End of getGraphTransaction() - public void showNodeInfo(EELFLogger logger, Vertex tVert, Boolean displayAllVidsFlag) { + public void showNodeInfo(Logger logger, Vertex tVert, Boolean displayAllVidsFlag) { try { Iterator<VertexProperty<Object>> pI = tVert.properties(); String infStr = ">>> Found Vertex with VertexId = " + tVert.id() + ", properties: "; System.out.println(infStr); - logger.info(infStr); + logger.debug(infStr); while (pI.hasNext()) { VertexProperty<Object> tp = pI.next(); infStr = " [" + tp.key() + "|" + tp.value() + "] "; System.out.println(infStr); - logger.info(infStr); + logger.debug(infStr); } ArrayList<String> retArr = collectEdgeInfoForNode(logger, tVert, displayAllVidsFlag); for (String infoStr : retArr) { System.out.println(infoStr); - logger.info(infoStr); + logger.debug(infoStr); } } catch (Exception e) { String warnMsg = " -- Error -- trying to display edge info. [" + e.getMessage() + "]"; System.out.println(warnMsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); logger.warn(warnMsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); } }// End of showNodeInfo() - public ArrayList<String> collectEdgeInfoForNode(EELFLogger logger, Vertex tVert, boolean displayAllVidsFlag) { + public ArrayList<String> collectEdgeInfoForNode(Logger logger, Vertex tVert, boolean displayAllVidsFlag) { ArrayList<String> retArr = new ArrayList<String>(); Direction dir = Direction.OUT; for (int i = 0; i <= 1; i++) { @@ -909,7 +856,7 @@ public class DupeTool { * @throws AAIException the AAI exception */ public ArrayList<Vertex> getNodeJustUsingKeyParams(String transId, String fromAppId, Graph graph, String nodeType, - HashMap<String, Object> keyPropsHash, String apiVersion, EELFLogger logger) throws AAIException { + HashMap<String, Object> keyPropsHash, String apiVersion, Logger logger) throws AAIException { ArrayList<Vertex> retVertList = new ArrayList<Vertex>(); @@ -956,11 +903,7 @@ public class DupeTool { throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n"); } } catch (Exception ex) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(" ERROR trying to get node for: [" + propsAndValuesForMsg + "] " + LogFormatTools.getStackTop(ex)); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); } if (verts != null) { @@ -995,7 +938,7 @@ public class DupeTool { */ public ArrayList<Vertex> figureOutNodes2Check(String transId, String fromAppId, Graph graph, String nodeType, long windowStartTime, - String propsString, EELFLogger logger) throws AAIException { + String propsString, Logger logger) throws AAIException { ArrayList<Vertex> retVertList = new ArrayList<Vertex>(); String msg = ""; @@ -1008,8 +951,6 @@ public class DupeTool { if (firstPipeLoc <= 0) { msg = "Bad props4Collect passed: [" + propsString + "]. \n Expecting a format like, 'propName1|propVal1,propName2|propVal2'"; System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(msg); exit(0); } @@ -1021,8 +962,6 @@ public class DupeTool { if (pipeLoc <= 0) { msg = "Bad propsString passed: [" + propsString + "]. \n Expecting a format like, 'propName1|propVal1,propName2|propVal2'"; System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(msg); exit(0); } else { @@ -1037,8 +976,6 @@ public class DupeTool { if (tgQ == null) { msg = "Bad JanusGraphQuery object. "; System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); logger.error(msg); exit(0); } else { @@ -1081,14 +1018,14 @@ public class DupeTool { * @param g the g * @param dupeVertexList the dupe vertex list * @param ver the ver - * @param EELFLogger the logger + * @param Logger the logger * @return Vertex * @throws AAIException the AAI exception */ public Vertex getPreferredDupe(String transId, String fromAppId, Graph g, ArrayList<Vertex> dupeVertexList, String ver, - Boolean specialTenantRule, Loader loader, EELFLogger logger) + Boolean specialTenantRule, Loader loader, Logger logger) throws AAIException { // This method assumes that it is being passed a List of @@ -1168,13 +1105,13 @@ public class DupeTool { * @param vtxB the vtx B * @param ver the ver * @param boolean specialTenantRuleFlag flag - * @param EELFLogger the logger + * @param Logger the logger * @return Vertex * @throws AAIException the AAI exception */ public Vertex pickOneOfTwoDupes(String transId, String fromAppId, GraphTraversalSource gts, Vertex vtxA, - Vertex vtxB, String ver, Boolean specialTenantRule, Loader loader, EELFLogger logger) throws AAIException { + Vertex vtxB, String ver, Boolean specialTenantRule, Loader loader, Logger logger) throws AAIException { Vertex nullVtx = null; Vertex preferredVtx = null; @@ -1352,13 +1289,13 @@ public class DupeTool { String infMsg = " WARNING >>> we are using the special tenant rule to choose to " + " delete tenant vtxId = " + vidA + ", and keep tenant vtxId = " + vidB; System.out.println(infMsg); - logger.info(infMsg); + logger.debug(infMsg); preferredVtx = vtxB; } else if (nodeTypesConn2B.containsKey("vserver") && nodeTypesConn2A.containsKey("service-subscription")) { String infMsg = " WARNING >>> we are using the special tenant rule to choose to " + " delete tenant vtxId = " + vidB + ", and keep tenant vtxId = " + vidA; System.out.println(infMsg); - logger.info(infMsg); + logger.debug(infMsg); preferredVtx = vtxA; } } @@ -1474,11 +1411,11 @@ public class DupeTool { * * @param g the g * @param dupeInfoList the dupe info string - * @param logger the EELFLogger + * @param logger the Logger * @return the boolean */ private Boolean deleteNonKeepers(Graph g, - ArrayList<String> dupeInfoList, EELFLogger logger) { + ArrayList<String> dupeInfoList, Logger logger) { // This assumes that each dupeInfoString is in the format of // pipe-delimited vid's followed by either "keepVid=xyz" or "keepVid=UNDETERMINED" @@ -1501,11 +1438,11 @@ public class DupeTool { * * @param g the g * @param dupeSetStr the dupe string - * @param logger the EELFLogger + * @param logger the Logger * @return the boolean */ private Boolean deleteNonKeeperForOneSet(Graph g, - String dupeInfoString, EELFLogger logger) { + String dupeInfoString, Logger logger) { Boolean deletedSomething = false; // This assumes that each dupeInfoString is in the format of @@ -1536,11 +1473,7 @@ public class DupeTool { if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) { String emsg = "Bad format. Expecting KeepVid=999999"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(emsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); return false; } else { String keepVidStr = prefArr[1]; @@ -1556,21 +1489,17 @@ public class DupeTool { Vertex vtx = g.traversal().V(longVertId).next(); String msg = "--->>> We will delete node with VID = " + thisVid + " <<<---"; System.out.println(msg); - logger.info(msg); + logger.debug(msg); vtx.remove(); } catch (Exception e) { okFlag = false; String emsg = "ERROR trying to delete VID = " + thisVid + ", [" + e + "]"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(emsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); } if (okFlag) { String infMsg = " DELETED VID = " + thisVid; - logger.info(infMsg); + logger.debug(infMsg); System.out.println(infMsg); deletedSomething = true; } @@ -1578,11 +1507,7 @@ public class DupeTool { } else { String emsg = "ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = [" + dupeInfoString + "]"; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(emsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); System.out.println(emsg); return false; } @@ -1601,11 +1526,11 @@ public class DupeTool { * * @param tvx the vertex to pull the properties from * @param keyPropertyNames ArrayList (ordered) of key prop names - * @param logger the EELFLogger + * @param logger the Logger * @return a hashMap of the propertyNames/values */ private HashMap<String, Object> getNodeKeyVals(Vertex tvx, - ArrayList<String> keyPropNamesArr, EELFLogger logger) { + ArrayList<String> keyPropNamesArr, Logger logger) { HashMap<String, Object> retHash = new HashMap<String, Object>(); Iterator<String> propItr = keyPropNamesArr.iterator(); @@ -1629,11 +1554,11 @@ public class DupeTool { * @param fromAppId the from app id * @param graph the graph * @param vtx - * @param EELFLogger + * @param Logger * @return true if aai-uri is populated and the aai-uri-index points to this vtx * @throws AAIException the AAI exception */ - private Boolean checkAaiUriOk( GraphTraversalSource graph, Vertex origVtx, EELFLogger eLogger ) + private Boolean checkAaiUriOk( GraphTraversalSource graph, Vertex origVtx, Logger eLogger ) throws AAIException{ String aaiUriStr = ""; try { @@ -1677,8 +1602,6 @@ public class DupeTool { } } catch( Exception ex ){ - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); eLogger.error(" ERROR trying to get node with aai-uri: [" + aaiUriStr + "]" + LogFormatTools.getStackTop(ex)); } return true; @@ -1691,11 +1614,11 @@ public class DupeTool { * * @param tvx the vertex to pull the properties from * @param keyPropertyNames collection of key prop names - * @param logger the EELFLogger + * @param logger the Logger * @return a String of concatenated values */ private String getNodeKeyValString(Vertex tvx, - ArrayList<String> keyPropNamesArr, EELFLogger logger) { + ArrayList<String> keyPropNamesArr, Logger logger) { // -- NOTE -- for what we're using this for, we would need to // guarantee that the properties are always in the same order @@ -1719,11 +1642,11 @@ public class DupeTool { * * @param firstPassDupeSets from the first pass * @param secondPassDupeSets from the second pass - * @param EELFLogger logger + * @param Logger logger * @return commonDupeSets that are common to both passes and have a determined keeper */ private ArrayList<String> figureWhichDupesStillNeedFixing(ArrayList<String> firstPassDupeSets, - ArrayList<String> secondPassDupeSets, EELFLogger logger) { + ArrayList<String> secondPassDupeSets, Logger logger) { ArrayList<String> common2BothSet = new ArrayList<String>(); @@ -1815,7 +1738,7 @@ public class DupeTool { private HashMap<String, ArrayList<String>> makeKeeperHashOfDupeStrings(ArrayList<String> dupeSets, - ArrayList<String> excludeSets, EELFLogger logger) { + ArrayList<String> excludeSets, Logger logger) { HashMap<String, ArrayList<String>> keeperHash = new HashMap<String, ArrayList<String>>(); @@ -1852,7 +1775,7 @@ public class DupeTool { String infMsg = "Bad format in figureWhichDupesStillNeedFixing(). Expecting " + " KeepVid=999999 but string looks like: [" + tmpSetStr + "]"; System.out.println(infMsg); - logger.info(infMsg); + logger.debug(infMsg); } else { keeperHash.put(prefArr[0], delIdArr); } @@ -1871,10 +1794,10 @@ public class DupeTool { * * @param g the g * @param dupeInfoString - * @param logger the EELFLogger + * @param logger the Logger * @return void */ - private void showNodeDetailsForADupeSet(Graph g, String dupeInfoString, EELFLogger logger) { + private void showNodeDetailsForADupeSet(Graph g, String dupeInfoString, Logger logger) { // dang... parsing this string once again... @@ -1894,7 +1817,7 @@ public class DupeTool { if (prefString.equals("KeepVid=UNDETERMINED")) { String msg = " Our algorithm cannot choose from among these, so they will all be kept. -------\n"; System.out.println(msg); - logger.info(msg); + logger.debug(msg); } else { // If we know which to keep, then the prefString should look // like, "KeepVid=12345" @@ -1902,16 +1825,12 @@ public class DupeTool { if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) { String emsg = "Bad format. Expecting KeepVid=999999"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(emsg); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); } else { String keepVidStr = prefArr[1]; String msg = " vid = " + keepVidStr + " is the one that we would KEEP. ------\n"; System.out.println(msg); - logger.info(msg); + logger.debug(msg); } } } @@ -1921,7 +1840,7 @@ public class DupeTool { private int graphIndex = 1; - public JanusGraph setupGraph(EELFLogger logger) { + public JanusGraph setupGraph(Logger logger) { JanusGraph JanusGraph = null; @@ -1945,7 +1864,7 @@ public class DupeTool { return JanusGraph; } - public void closeGraph(JanusGraph graph, EELFLogger logger) { + public void closeGraph(JanusGraph graph, Logger logger) { try { if ("inmemory".equals(graphType)) { diff --git a/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java b/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java index dec6a85..2e1bc4b 100644 --- a/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java +++ b/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java @@ -1,906 +1,899 @@ -/**
- * ============LICENSE_START=======================================================
- * org.onap.aai
- * ================================================================================
- * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END=========================================================
- */
-package org.onap.aai.dbgen;
-
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import org.apache.tinkerpop.gremlin.process.traversal.P;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
-import org.apache.tinkerpop.gremlin.structure.Element;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.node.ObjectNode;
-import org.codehaus.jackson.type.TypeReference;
-import org.onap.aai.config.PropertyPasswordConfiguration;
-import org.onap.aai.db.props.AAIProperties;
-import org.onap.aai.dbmap.DBConnectionType;
-import org.onap.aai.dbmap.InMemoryGraph;
-import org.onap.aai.edges.EdgeIngestor;
-import org.onap.aai.edges.EdgeRule;
-import org.onap.aai.edges.EdgeRuleQuery;
-import org.onap.aai.edges.enums.AAIDirection;
-import org.onap.aai.edges.enums.EdgeType;
-import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException;
-import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException;
-import org.onap.aai.exceptions.AAIException;
-import org.onap.aai.introspection.Introspector;
-import org.onap.aai.introspection.Loader;
-import org.onap.aai.introspection.LoaderFactory;
-import org.onap.aai.introspection.ModelType;
-import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
-import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.logging.LogFormatTools;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.parsers.uri.URIToObject;
-import org.onap.aai.serialization.db.DBSerializer;
-import org.onap.aai.serialization.engines.InMemoryDBEngine;
-import org.onap.aai.serialization.engines.QueryStyle;
-import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
-import org.onap.aai.setup.SchemaVersion;
-import org.onap.aai.setup.SchemaVersions;
-import org.onap.aai.util.AAIConfig;
-import org.onap.aai.util.AAIConstants;
-import org.onap.aai.util.AAISystemExitUtil;
-import org.onap.aai.util.ExceptionTranslator;
-import org.slf4j.MDC;
-import org.springframework.context.annotation.AnnotationConfigApplicationContext;
-
-import java.io.*;
-import java.net.URI;
-import java.nio.file.Files;
-import java.nio.file.InvalidPathException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/*
- * The Class ListEndpoints.
- */
-public class DynamicPayloadGenerator {
-
- /*
- * Create a Dynamic memory graph instance which should not affect the
- * AAIGraph
- */
- private InMemoryGraph inMemGraph = null;
-
- private InMemoryDBEngine dbEngine;
- private InputStream sequenceInputStreams;
- /*
- * Loader, QueryStyle, ConnectionType for the Serializer
- */
- private Loader loader;
- private String urlBase;
- private BufferedWriter bw = null;
- private boolean exitFlag = true;
- private CommandLineArgs cArgs;
-
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DynamicPayloadGenerator.class);
-
- private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL;
- private static final DBConnectionType type = DBConnectionType.CACHED;
- private static final ModelType introspectorFactoryType = ModelType.MOXY;
- private final LoaderFactory loaderFactory;
- private final EdgeIngestor edgeRules;
- private final SchemaVersions schemaVersions;
- private final SchemaVersion version;
-
- public DynamicPayloadGenerator(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions){
- this.loaderFactory = loaderFactory;
- this.edgeRules = edgeIngestor;
- this.schemaVersions = schemaVersions;
- this.version = schemaVersions.getDefaultVersion();
- }
-
- /**
- * The run method.
- *
- * @param args
- * the arguments
- * @param exitFlag true if running from a shell script to call system exit, false if running from scheduled task
- * @throws AAIException
- * @throws Exception
- */
-
- public static void run (LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions, String[] args, boolean isSystemExit) {
- //
- MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName());
- DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator(loaderFactory, edgeIngestor, schemaVersions);
- payloadgen.exitFlag = isSystemExit;
- try {
- payloadgen.init(args);
-
- payloadgen.generatePayloads();
- } catch (AAIException e) {
- LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
- } catch (IOException e) {
- LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
- }
- if ( isSystemExit ) {
- AAISystemExitUtil.systemExitCloseAAIGraph(1);
- }
- else {
- AAISystemExitUtil.systemExitCloseAAIGraph(0);
- }
-
- }
- public static void main(String[] args) throws AAIException {
- AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
- PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
- initializer.initialize(ctx);
- try {
- ctx.scan(
- "org.onap.aai.config",
- "org.onap.aai.setup"
- );
- ctx.refresh();
- } catch (Exception e) {
- AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
- LOGGER.error("Problems running tool "+aai.getMessage());
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
- throw aai;
-
- }
- LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
- EdgeIngestor edgeIngestor = ctx.getBean(EdgeIngestor.class);
- SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions");
- run (loaderFactory, edgeIngestor, schemaVersions, args, true);
- }
-
-
- public void taskExit() {
- if ( this.exitFlag ) {
- AAISystemExitUtil.systemExitCloseAAIGraph(1);
- }
- else {
- AAISystemExitUtil.systemExitCloseAAIGraph(0);
- }
- }
- public void init(String[] args) throws AAIException {
- cArgs = new CommandLineArgs();
- JCommander jCommander = new JCommander(cArgs, args);
- jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName());
- LOGGER.info("Snapshot file " + cArgs.dataSnapshot);
-
-
- // TODO- How to add dynamic.properties
-
- LOGGER.info("output file " + cArgs.output);
- LOGGER.info("format file " + cArgs.format);
- LOGGER.info("schema enabled " + cArgs.schemaEnabled);
- LOGGER.info("Multiple snapshots " + cArgs.isMultipleSnapshot);
- LOGGER.info("Is Partial Graph " + cArgs.isPartialGraph);
-
- if (cArgs.config.isEmpty())
- cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties";
-
- LOGGER.info("config file " + cArgs.config);
- if (cArgs.nodePropertyFile.isEmpty())
- cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json";
- LOGGER.info("nodePropertyFile file " + cArgs.nodePropertyFile);
-
- if (cArgs.inputFilterPropertyFile.isEmpty())
- cArgs.inputFilterPropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/inputFilters.json";
- LOGGER.info("inputFilterPropertyFile file " + cArgs.inputFilterPropertyFile);
-
- if (cArgs.isPartialGraph)
- cArgs.dataSnapshot = cArgs.dataSnapshot+".partial";
-
- if (!cArgs.isMultipleSnapshot) {
- validateFile(cArgs.dataSnapshot);
- } else {
- // for multiple snapshots dataSnapshot + ".P" is the prefix of the
- // files
- sequenceInputStreams = validateMultipleSnapshots(cArgs.dataSnapshot);
- }
-
- LOGGER.info("Datasnapshot file " + cArgs.dataSnapshot);
- AAIConfig.init();
-
- urlBase = AAIConfig.get("aai.server.url.base", "");
-
- }
-
- public void generatePayloads() throws AAIException, IOException {
-
- List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile);
- /*
- * Read the inputFilters which will include for each node-type the regex that needs to be
- * applied and the filtered-node-type
- * For eg: complex --> apply regex on cloud-region and then traverse to complex
- * complex --> filtered-node-type: cloud-region, filters: include regex on cloud-region
- */
- /*
- * Example:
- * { "cloud-region" :
- * {"filtered-node-type":"cloud-region",
- * "filters": [ { "property": "cloud-owner", "regex": "att-aic" },
- * { "property": "cloud-region-id", "regex": "M*" },
- * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" }
- * ] },
- * "complex" : {
- * "filtered-node-type":"cloud-region",
- * "filters": [ { "property": "cloud-owner", "regex": "att-aic" },
- * { "property": "cloud-region-id", "regex": "M*" },
- * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" }
- * ] },
- *
- * } }
- */
- Map<String, Map<String, String>> inputFilters = readInputFilterPropertyFile(cArgs.inputFilterPropertyFile);
- Map<String, String> filteredNodeTypes = findFilteredNodeTypes(cArgs.inputFilterPropertyFile);
- // Read the input filter criteria
- LOGGER.info("Load the Graph");
-
- this.loadGraph();
- LOGGER.info("Generate payload");
- this.generatePayload(nodeFilters, inputFilters, filteredNodeTypes);
- LOGGER.info("Close graph");
- this.closeGraph();
-
- }
-
- private List<Map<String, List<String>>> readFile(String inputFile) throws IOException {
-
- // validate that we can read the inputFile
- validateFile(inputFile);
-
- InputStream is = new FileInputStream(inputFile);
- Scanner scanner = new Scanner(is);
- String jsonFile = scanner.useDelimiter("\\Z").next();
- scanner.close();
-
- List<Map<String, List<String>>> allNodes = new ArrayList<>();
- Map<String, List<String>> filterCousins = new HashMap<>();
- Map<String, List<String>> filterParents = new HashMap<>();
-
- ObjectMapper mapper = new ObjectMapper();
-
- JsonNode rootNode = mapper.readTree(jsonFile);
-
- Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
-
- while (nodeFields.hasNext()) {
- Entry<String, JsonNode> entry = nodeFields.next();
- String nodeType = entry.getKey();
- JsonNode nodeProperty = entry.getValue();
-
- JsonNode cousinFilter = nodeProperty.path("cousins");
- JsonNode parentFilter = nodeProperty.path("parents");
- List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(),
- new TypeReference<ArrayList<String>>() {
- });
-
- List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(),
- new TypeReference<ArrayList<String>>() {
- });
- for (String cousin : cousins) {
- LOGGER.info("Cousins-Filtered " + cousin);
- }
- for (String parent : parents) {
- LOGGER.info("Parents-Filtered " + parent);
- }
- filterCousins.put(nodeType, cousins);
- filterParents.put(nodeType, parents);
-
- }
-
- allNodes.add(filterCousins);
- allNodes.add(filterParents);
- return allNodes;
-
- }
-
- /* Example:
-{
- "cloud-region" : {
- "filtered-node-type" :"cloud-region",
- "filters": [
- {
- "property": "cloud-owner",
- "regex": "att-aic"
- },
- {
- "property": "cloud-region-id",
- "regex": "M*"
- },
- {
- "property": "cloud-region-version",
- "regex": "aic2.5|aic3.0"
- }
- ]
- },
- "complex" : {
- "filters":[
- ]
-
- }
-}
-*/
- private Map<String, Map<String, String>> readInputFilterPropertyFile(String inputFile) throws IOException {
-
- validateFile(inputFile);
-
- InputStream is = new FileInputStream(inputFile);
- Scanner scanner = new Scanner(is);
- String jsonFile = scanner.useDelimiter("\\Z").next();
- scanner.close();
-
- Map<String, Map<String, String>> propToRegex = new HashMap<String, Map<String, String>>();
-
- ObjectMapper mapper = new ObjectMapper();
-
- JsonNode rootNode = mapper.readTree(jsonFile);
-
- Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
-
- while (nodeFields.hasNext()) {
- Entry<String, JsonNode> entry = nodeFields.next();
- String nodeType = entry.getKey();
- JsonNode nodeProperty = entry.getValue();
-
- JsonNode filter = nodeProperty.path("filters");
- List<JsonNode> filterMap = new ObjectMapper().readValue(filter.traverse(),
- new TypeReference<ArrayList<JsonNode>>() {
- });
- HashMap<String, String> filterMaps = new HashMap<String, String>();
- for (JsonNode n : filterMap) {
- filterMaps.put(n.get("property").asText(), n.get("regex").asText());
- }
-
- propToRegex.put(nodeType, filterMaps);
- }
- return (propToRegex);
- }
-
- private Map<String, String> findFilteredNodeTypes(String inputFile) throws IOException {
-
- validateFile(inputFile);
-
- InputStream is = new FileInputStream(inputFile);
- Scanner scanner = new Scanner(is);
- String jsonFile = scanner.useDelimiter("\\Z").next();
- scanner.close();
-
- Map<String, String> filteredNodeTypes = new HashMap<String, String>();
-
- ObjectMapper mapper = new ObjectMapper();
-
- JsonNode rootNode = mapper.readTree(jsonFile);
-
- Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
-
- while (nodeFields.hasNext()) {
- Entry<String, JsonNode> entry = nodeFields.next();
- String nodeType = entry.getKey();
- JsonNode nodeProperty = entry.getValue();
-
- JsonNode filter = nodeProperty.path("filtered-node-type");
-
- filteredNodeTypes.put(nodeType, filter.asText());
- }
- return (filteredNodeTypes);
- }
-
- public void loadGraph() throws IOException {
-
- loadGraphIntoMemory();
- buildDbEngine();
-
- }
-
- private void loadGraphIntoMemory() throws IOException {
- if (!(cArgs.isMultipleSnapshot)) {
- inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled,
- cArgs.isPartialGraph);
- } else {
- inMemGraph = new InMemoryGraph.Builder().build(sequenceInputStreams, cArgs.config, cArgs.schemaEnabled,
- cArgs.isPartialGraph);
- }
- }
-
- private void buildDbEngine() {
- // TODO : parametrise version
- loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
-
- dbEngine = new InMemoryDBEngine(queryStyle, type, loader, inMemGraph.getGraph());
- dbEngine.startTransaction();
- }
-
- private void generatePayload(List<Map<String, List<String>>> nodeFilters,
- Map<String, Map<String, String>> inputFilters, Map<String, String> filteredNodeTypes)
- throws AAIException, IOException {
-
- Map<String, List<String>> filterCousinsMap = nodeFilters.get(0);
- Map<String, List<String>> filterParentsMap = nodeFilters.get(1);
- Set<String> nodeTypes = filterCousinsMap.keySet();
-
- for (String nodeType : nodeTypes) {
- if ("DMAAP-MR".equals(cArgs.format)) {
- bw = createFile(nodeType + ".json");
- }
- List<String> filterCousins = filterCousinsMap.get(nodeType);
- List<String> filterParents = filterParentsMap.get(nodeType);
- Map<String, String> nodeInputFilterMap = inputFilters.get(nodeType);
- String filteredNodeType = nodeType;
- if(filteredNodeTypes.get(nodeType) != null && !filteredNodeTypes.get(nodeType).isEmpty())
- filteredNodeType = filteredNodeTypes.get(nodeType);
- readVertices(nodeType, filterCousins, filterParents, nodeInputFilterMap, filteredNodeType);
- if(bw != null)
- bw.close();
- LOGGER.info("All Done-" + nodeType);
- }
-
- }
-
- private BufferedWriter createFile(String outfileName) throws IOException {
- // FileLocation
- String fileName = outfileName;
- File outFile = new File(fileName);
- FileWriter fw = null;
- LOGGER.info(" Will write to " + fileName);
- try {
- fw = new FileWriter(outFile.getAbsoluteFile());
- } catch (IOException i) {
- String emsg = "Unable to write to " + fileName + " Exception = " + i.getMessage();
- LOGGER.error(emsg);
- System.out.println(emsg);
- throw i;
- }
- return new BufferedWriter(fw);
- }
-
- private void createDirectory(String dirName) throws IOException {
- // FileLocation
- Path pathDir = null;
- try {
- pathDir = Paths.get(dirName);
- } catch (InvalidPathException i) {
- String emsg = "Directory " + dirName + " could not be found.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- try {
- Files.createDirectories(pathDir);
- } catch (Exception e) {
- String emsg = "Directory " + dirName + " could not be created. Exception = " + e.getMessage();
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- }
-
- public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents,
- Map<String, String> nodeInputFilters, String filteredNodeType) throws AAIException, IOException {
-
- DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
-
- /*
- * Start with nodeType you need to filter and then traverse to the actual nodeType
- */
- GraphTraversal<Vertex, Vertex> gtraversal = inMemGraph.getGraph().traversal().V().has("aai-node-type",
- filteredNodeType);
-
-
- // input regex
- if (nodeInputFilters != null && (!nodeInputFilters.isEmpty())) {
- for (Map.Entry<String, String> entry : nodeInputFilters.entrySet()) {
- String property = entry.getKey();
- String regex = entry.getValue();
- Pattern pa = Pattern.compile(regex);
-
- gtraversal = gtraversal.has(property, P.test((t, p) -> {
- Matcher m = ((Pattern) p).matcher((CharSequence) t);
- boolean b = m.matches();
- return b;
- }, pa));
- }
- }
-
- /*
- * Tenant, AZ, Complex, Zone, pserver come here
- */
- if (!filteredNodeType.equals(nodeType)) {
-
- EdgeRuleQuery treeEdgeRuleQuery = new EdgeRuleQuery
- .Builder(filteredNodeType, nodeType)
- .edgeType(EdgeType.TREE)
- .build();
-
- EdgeRuleQuery cousinEdgeQuery = new EdgeRuleQuery
- .Builder(filteredNodeType, nodeType)
- .edgeType(EdgeType.COUSIN)
- .build();
-
- EdgeRule rule = null;
- boolean hasTreeEdgeRule = true;
-
- try {
- rule = edgeRules.getRule(treeEdgeRuleQuery);
- } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
- hasTreeEdgeRule = false;
- }
-
- if(!hasTreeEdgeRule) {
- try {
- rule = edgeRules.getRule(cousinEdgeQuery);
- } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
- LOGGER.error("Unable to get a tree or cousin edge between {} and {}", filteredNodeType, nodeType);
- return;
- }
- }
-
- if (rule.getDirection().toString().equals(AAIDirection.OUT.toString())) {
- gtraversal.out(rule.getLabel()).has("aai-node-type", nodeType);
- } else {
- gtraversal.in(rule.getLabel()).has("aai-node-type", nodeType);
- }
-
- }
-
- String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP;
- createDirectory(dirName);
- // TODO: Formatter
-
- if ("DMAAP-MR".equals(cArgs.format)) {
- while (gtraversal.hasNext()) {
- if (bw != null)
- bw = createFile(nodeType + ".json");
- Vertex node = gtraversal.next();
- Introspector nodeObj = serializer.getLatestVersionView(node);
- createPayloadForDmaap(node, nodeObj);
- }
- } else {
- if ("PAYLOAD".equals(cArgs.format)) {
- int counter = 0;
- while (gtraversal.hasNext()) {
- Vertex node = gtraversal.next();
- try {
- counter++;
- String filename = dirName + counter + "-" + nodeType + ".json";
- bw = createFile(filename);
- Introspector obj = loader.introspectorFromName(nodeType);
- Set<Vertex> seen = new HashSet<>();
- int depth = AAIProperties.MAXIMUM_DEPTH;
- boolean nodeOnly = false;
-
- Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly);
- TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree);
- serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins,
- filterParents);
- createPayloadForPut(obj);
- if(bw != null)
- bw.close();
-
- URI uri = serializer.getURIForVertex(node);
- String filenameWithUri = dirName + counter + "-" + nodeType + ".txt";
- bw = createFile(filenameWithUri);
- bw.write(uri.toString());
- bw.newLine();
- bw.close();
- } catch (Exception e) {
- String emsg = "Caught exception while processing [" + counter + "-" + nodeType + "] continuing";
- System.out.println(emsg);
- LOGGER.error(emsg);
-
- }
- }
- }
- }
-
- }
-
- public void createPayloadForPut(Introspector nodeObj) throws IOException {
-
- String entityJson = nodeObj.marshal(false);
- ObjectMapper mapper = new ObjectMapper();
-
- ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson);
- rootNode.remove("resource-version");
-
- bw.newLine();
- bw.write(rootNode.toString());
- bw.newLine();
- }
-
- public void createPayloadForDmaap(Vertex node, Introspector nodeObj)
- throws AAIException, UnsupportedEncodingException {
-
- DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
-
- URI uri = serializer.getURIForVertex(node);
-
- String sourceOfTruth = "";
- HashMap<String, Introspector> relatedVertices = new HashMap<>();
- List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node);
-
- for (Vertex vertex : vertexChain) {
- try {
-
- Introspector vertexObj = serializer.getVertexProperties(vertex);
-
- relatedVertices.put(vertexObj.getObjectId(), vertexObj);
- } catch (AAIUnknownObjectException e) {
- LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
- }
-
- }
-
- String transactionId = "TXID";
- createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices);
-
- }
-
- public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj,
- Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException {
-
- String action = "CREATE";
- final Introspector notificationEvent = loader.introspectorFromName("notification-event");
-
- try {
- Introspector eventHeader = loader.introspectorFromName("notification-event-header");
- URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects);
-
- String entityLink = urlBase + version + uri;
-
- notificationEvent.setValue("cambria-partition", "AAI");
-
- eventHeader.setValue("entity-link", entityLink);
- eventHeader.setValue("action", action);
- eventHeader.setValue("entity-type", obj.getDbName());
- eventHeader.setValue("top-entity-type", parser.getTopEntityName());
- eventHeader.setValue("source-name", sourceOfTruth);
- eventHeader.setValue("version", version.toString());
- eventHeader.setValue("id", transactionId);
- eventHeader.setValue("event-type", "AAI-BASELINE");
- if (eventHeader.getValue("domain") == null) {
- eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK"));
- }
-
- if (eventHeader.getValue("sequence-number") == null) {
- eventHeader.setValue("sequence-number",
- AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK"));
- }
-
- if (eventHeader.getValue("severity") == null) {
- eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK"));
- }
-
- if (eventHeader.getValue("id") == null) {
- eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString());
-
- }
-
- if (eventHeader.getValue("timestamp") == null) {
- eventHeader.setValue("timestamp", genDate());
- }
-
- List<Object> parentList = parser.getParentList();
- parentList.clear();
-
- if (!parser.getTopEntity().equals(parser.getEntity())) {
- Introspector child;
- String json = obj.marshal(false);
- child = parser.getLoader().unmarshal(parser.getEntity().getName(), json);
- parentList.add(child.getUnderlyingObject());
- }
-
- final Introspector eventObject;
-
- String json = "";
- if (parser.getTopEntity().equals(parser.getEntity())) {
- json = obj.marshal(false);
- eventObject = loader.unmarshal(obj.getName(), json);
- } else {
- json = parser.getTopEntity().marshal(false);
-
- eventObject = loader.unmarshal(parser.getTopEntity().getName(), json);
- }
- notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject());
- notificationEvent.setValue("entity", eventObject.getUnderlyingObject());
-
- String entityJson = notificationEvent.marshal(false);
-
- bw.newLine();
- bw.write(entityJson);
-
- } catch (AAIUnknownObjectException e) {
- LOGGER.error("Fatal error - notification-event-header object not found!");
- } catch (Exception e) {
- LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e));
- }
- }
-
- private void closeGraph() {
- inMemGraph.getGraph().tx().rollback();
- inMemGraph.getGraph().close();
- }
-
- public static String genDate() {
- Date date = new Date();
- DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS");
- return formatter.format(date);
- }
-
- public static String genDate2() {
- Date date = new Date();
- DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
- return formatter.format(date);
- }
-
- private void validateFile(String filename) {
- File f = new File(filename);
- if (!f.exists()) {
- String emsg = "File " + filename + " could not be found.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- } else if (!f.canRead()) {
- String emsg = "File " + filename + " could not be read.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- } else if (f.length() == 0) {
- String emsg = "File " + filename + " had no data.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- }
-
- private InputStream validateMultipleSnapshots(String filenamePrefix) {
- if (filenamePrefix == null || filenamePrefix.length() == 0) {
- String emsg = "No snapshot path was provided.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- String targetDir = ".";
- int lastSeparator = filenamePrefix.lastIndexOf(File.separator);
-
- LOGGER.info("File separator=[" + File.separator + "] lastSeparator=" + lastSeparator + " filenamePrefix="
- + filenamePrefix);
- if (lastSeparator >= 0) {
- targetDir = filenamePrefix.substring(0, lastSeparator);
- LOGGER.info("targetDir=" + targetDir);
- }
- if (targetDir.length() == 0) {
- String emsg = "No snapshot directory was found in path:" + filenamePrefix;
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- String prefix = filenamePrefix.substring(lastSeparator + 1);
- if (prefix == null || prefix.length() == 0) {
- String emsg = "No snapshot file prefix was provided.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- long timeA = System.nanoTime();
-
- ArrayList<File> snapFilesArr = new ArrayList<File>();
- String thisSnapPrefix = prefix + ".P";
- File fDir = new File(targetDir); // Snapshot directory
- File[] allFilesArr = fDir.listFiles();
- for (File snapFile : allFilesArr) {
- String snapFName = snapFile.getName();
- if (snapFName.startsWith(thisSnapPrefix)) {
- snapFilesArr.add(snapFile);
- }
- }
-
- if (snapFilesArr.isEmpty()) {
- String fullFName = targetDir + AAIConstants.AAI_FILESEP + thisSnapPrefix;
- String emsg = "Snapshot files " + fullFName + "* could not be found.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
-
- int fCount = snapFilesArr.size();
- Iterator<File> fItr = snapFilesArr.iterator();
- Vector<InputStream> inputStreamsV = new Vector<>();
- for (int i = 0; i < fCount; i++) {
- File f = snapFilesArr.get(i);
- String fname = f.getName();
- if (!f.canRead()) {
- String emsg = "Snapshot file " + fname + " could not be read.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- } else if (f.length() == 0) {
- String emsg = "Snapshot file " + fname + " had no data.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- String fullFName = targetDir + AAIConstants.AAI_FILESEP + fname;
- InputStream fis = null;
- try {
- fis = new FileInputStream(fullFName);
- } catch (FileNotFoundException e) {
- // should not happen at this point
- String emsg = "Snapshot file " + fullFName + " could not be found";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- inputStreamsV.add(fis);
- }
- // Now add inputStreams.elements() to the Vector,
- InputStream sis = new SequenceInputStream(inputStreamsV.elements());
- return (sis);
- }
-
- public InMemoryGraph getInMemGraph() {
- return inMemGraph;
- }
-
- public void setInMemGraph(InMemoryGraph inMemGraph) {
- this.inMemGraph = inMemGraph;
- }
-}
-
-class CommandLineArgs {
-
- @Parameter(names = "--help", help = true)
- public boolean help;
-
- @Parameter(names = "-d", description = "snapshot file to be loaded", required = true)
- public String dataSnapshot;
-
- @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1)
- public boolean schemaEnabled = true;
-
- @Parameter(names = "-c", description = "location of configuration file")
- public String config = "";
-
- @Parameter(names = "-o", description = "output location")
- public String output = "";
-
- @Parameter(names = "-f", description = "format of output")
- public String format = "PAYLOAD";
-
- @Parameter(names = "-n", description = "Node input file")
- public String nodePropertyFile = "";
-
- @Parameter(names = "-m", description = "multipe snapshots or not", arity = 1)
- public boolean isMultipleSnapshot = false;
-
- @Parameter(names = "-i", description = "input filter configuration file")
- public String inputFilterPropertyFile = "";
-
- @Parameter(names = "-p", description = "Use the partial graph", arity = 1)
- public boolean isPartialGraph = true;
-
-}
+/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.dbgen; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.node.ObjectNode; +import org.codehaus.jackson.type.TypeReference; +import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.db.props.AAIProperties; +import org.onap.aai.dbmap.InMemoryGraph; +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.edges.EdgeRule; +import org.onap.aai.edges.EdgeRuleQuery; +import org.onap.aai.edges.enums.AAIDirection; +import org.onap.aai.edges.enums.EdgeType; +import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException; +import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; +import org.onap.aai.introspection.exceptions.AAIUnknownObjectException; +import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; +import org.onap.aai.parsers.uri.URIToObject; +import org.onap.aai.serialization.db.DBSerializer; +import org.onap.aai.serialization.engines.InMemoryDBEngine; +import org.onap.aai.serialization.engines.QueryStyle; +import org.onap.aai.serialization.tinkerpop.TreeBackedVertex; +import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; +import org.onap.aai.util.AAIConstants; +import org.onap.aai.util.AAISystemExitUtil; +import org.onap.aai.util.ExceptionTranslator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import java.io.*; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/* + * The Class ListEndpoints. + */ +public class DynamicPayloadGenerator { + + /* + * Create a Dynamic memory graph instance which should not affect the + * AAIGraph + */ + private InMemoryGraph inMemGraph = null; + + private InMemoryDBEngine dbEngine; + private InputStream sequenceInputStreams; + /* + * Loader, QueryStyle, ConnectionType for the Serializer + */ + private Loader loader; + private String urlBase; + private BufferedWriter bw = null; + private boolean exitFlag = true; + private CommandLineArgs cArgs; + + private static final Logger LOGGER = LoggerFactory.getLogger(DynamicPayloadGenerator.class); + + private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL; + private static final ModelType introspectorFactoryType = ModelType.MOXY; + private final LoaderFactory loaderFactory; + private final EdgeIngestor edgeRules; + private final SchemaVersions schemaVersions; + private final SchemaVersion version; + + public DynamicPayloadGenerator(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions){ + this.loaderFactory = loaderFactory; + this.edgeRules = edgeIngestor; + this.schemaVersions = schemaVersions; + this.version = schemaVersions.getDefaultVersion(); + } + + /** + * The run method. + * + * @param args + * the arguments + * @param exitFlag true if running from a shell script to call system exit, false if running from scheduled task + * @throws AAIException + * @throws Exception + */ + + public static void run (LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions, String[] args, boolean isSystemExit) { + // + MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName()); + DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator(loaderFactory, edgeIngestor, schemaVersions); + payloadgen.exitFlag = isSystemExit; + try { + payloadgen.init(args); + + payloadgen.generatePayloads(); + } catch (AAIException | IOException e) { + LOGGER.error("Exception {}", LogFormatTools.getStackTop(e)); + } + if ( isSystemExit ) { + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + else { + AAISystemExitUtil.systemExitCloseAAIGraph(0); + } + + } + public static void main(String[] args) throws AAIException { + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); + PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); + initializer.initialize(ctx); + try { + ctx.scan( + "org.onap.aai.config", + "org.onap.aai.setup" + ); + ctx.refresh(); + } catch (Exception e) { + AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); + LOGGER.error("Problems running tool {}", aai.getMessage()); + ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); + throw aai; + + } + LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); + EdgeIngestor edgeIngestor = ctx.getBean(EdgeIngestor.class); + SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions"); + run (loaderFactory, edgeIngestor, schemaVersions, args, true); + } + + + public void taskExit() { + if ( this.exitFlag ) { + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + else { + AAISystemExitUtil.systemExitCloseAAIGraph(0); + } + } + public void init(String[] args) throws AAIException { + cArgs = new CommandLineArgs(); + JCommander jCommander = new JCommander(cArgs, args); + jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName()); + LOGGER.debug("Snapshot file " + cArgs.dataSnapshot); + + + // TODO- How to add dynamic.properties + + LOGGER.debug("output file " + cArgs.output); + LOGGER.debug("format file " + cArgs.format); + LOGGER.debug("schema enabled " + cArgs.schemaEnabled); + LOGGER.debug("Multiple snapshots " + cArgs.isMultipleSnapshot); + LOGGER.debug("Is Partial Graph " + cArgs.isPartialGraph); + + if (cArgs.config.isEmpty()) + cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties"; + + LOGGER.debug("config file " + cArgs.config); + if (cArgs.nodePropertyFile.isEmpty()) + cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json"; + LOGGER.debug("nodePropertyFile file " + cArgs.nodePropertyFile); + + if (cArgs.inputFilterPropertyFile.isEmpty()) + cArgs.inputFilterPropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/inputFilters.json"; + LOGGER.debug("inputFilterPropertyFile file " + cArgs.inputFilterPropertyFile); + + if (cArgs.isPartialGraph) + cArgs.dataSnapshot = cArgs.dataSnapshot+".partial"; + + if (!cArgs.isMultipleSnapshot) { + validateFile(cArgs.dataSnapshot); + } else { + // for multiple snapshots dataSnapshot + ".P" is the prefix of the + // files + sequenceInputStreams = validateMultipleSnapshots(cArgs.dataSnapshot); + } + + LOGGER.debug("Datasnapshot file " + cArgs.dataSnapshot); + AAIConfig.init(); + + urlBase = AAIConfig.get("aai.server.url.base", ""); + + } + + public void generatePayloads() throws AAIException, IOException { + + List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile); + /* + * Read the inputFilters which will include for each node-type the regex that needs to be + * applied and the filtered-node-type + * For eg: complex --> apply regex on cloud-region and then traverse to complex + * complex --> filtered-node-type: cloud-region, filters: include regex on cloud-region + */ + /* + * Example: + * { "cloud-region" : + * {"filtered-node-type":"cloud-region", + * "filters": [ { "property": "cloud-owner", "regex": "att-aic" }, + * { "property": "cloud-region-id", "regex": "M*" }, + * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" } + * ] }, + * "complex" : { + * "filtered-node-type":"cloud-region", + * "filters": [ { "property": "cloud-owner", "regex": "att-aic" }, + * { "property": "cloud-region-id", "regex": "M*" }, + * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" } + * ] }, + * + * } } + */ + Map<String, Map<String, String>> inputFilters = readInputFilterPropertyFile(cArgs.inputFilterPropertyFile); + Map<String, String> filteredNodeTypes = findFilteredNodeTypes(cArgs.inputFilterPropertyFile); + // Read the input filter criteria + LOGGER.debug("Load the Graph"); + + this.loadGraph(); + LOGGER.debug("Generate payload"); + this.generatePayload(nodeFilters, inputFilters, filteredNodeTypes); + LOGGER.debug("Close graph"); + this.closeGraph(); + + } + + private List<Map<String, List<String>>> readFile(String inputFile) throws IOException { + + // validate that we can read the inputFile + validateFile(inputFile); + + InputStream is = new FileInputStream(inputFile); + Scanner scanner = new Scanner(is); + String jsonFile = scanner.useDelimiter("\\Z").next(); + scanner.close(); + + List<Map<String, List<String>>> allNodes = new ArrayList<>(); + Map<String, List<String>> filterCousins = new HashMap<>(); + Map<String, List<String>> filterParents = new HashMap<>(); + + ObjectMapper mapper = new ObjectMapper(); + + JsonNode rootNode = mapper.readTree(jsonFile); + + Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields(); + + while (nodeFields.hasNext()) { + Entry<String, JsonNode> entry = nodeFields.next(); + String nodeType = entry.getKey(); + JsonNode nodeProperty = entry.getValue(); + + JsonNode cousinFilter = nodeProperty.path("cousins"); + JsonNode parentFilter = nodeProperty.path("parents"); + List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(), + new TypeReference<ArrayList<String>>() { + }); + + List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(), + new TypeReference<ArrayList<String>>() { + }); + for (String cousin : cousins) { + LOGGER.debug("Cousins-Filtered " + cousin); + } + for (String parent : parents) { + LOGGER.debug("Parents-Filtered " + parent); + } + filterCousins.put(nodeType, cousins); + filterParents.put(nodeType, parents); + + } + + allNodes.add(filterCousins); + allNodes.add(filterParents); + return allNodes; + + } + + /* Example: +{ + "cloud-region" : { + "filtered-node-type" :"cloud-region", + "filters": [ + { + "property": "cloud-owner", + "regex": "att-aic" + }, + { + "property": "cloud-region-id", + "regex": "M*" + }, + { + "property": "cloud-region-version", + "regex": "aic2.5|aic3.0" + } + ] + }, + "complex" : { + "filters":[ + ] + + } +} +*/ + private Map<String, Map<String, String>> readInputFilterPropertyFile(String inputFile) throws IOException { + + validateFile(inputFile); + + InputStream is = new FileInputStream(inputFile); + Scanner scanner = new Scanner(is); + String jsonFile = scanner.useDelimiter("\\Z").next(); + scanner.close(); + + Map<String, Map<String, String>> propToRegex = new HashMap<String, Map<String, String>>(); + + ObjectMapper mapper = new ObjectMapper(); + + JsonNode rootNode = mapper.readTree(jsonFile); + + Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields(); + + while (nodeFields.hasNext()) { + Entry<String, JsonNode> entry = nodeFields.next(); + String nodeType = entry.getKey(); + JsonNode nodeProperty = entry.getValue(); + + JsonNode filter = nodeProperty.path("filters"); + List<JsonNode> filterMap = new ObjectMapper().readValue(filter.traverse(), + new TypeReference<ArrayList<JsonNode>>() { + }); + HashMap<String, String> filterMaps = new HashMap<String, String>(); + for (JsonNode n : filterMap) { + filterMaps.put(n.get("property").asText(), n.get("regex").asText()); + } + + propToRegex.put(nodeType, filterMaps); + } + return (propToRegex); + } + + private Map<String, String> findFilteredNodeTypes(String inputFile) throws IOException { + + validateFile(inputFile); + + InputStream is = new FileInputStream(inputFile); + Scanner scanner = new Scanner(is); + String jsonFile = scanner.useDelimiter("\\Z").next(); + scanner.close(); + + Map<String, String> filteredNodeTypes = new HashMap<String, String>(); + + ObjectMapper mapper = new ObjectMapper(); + + JsonNode rootNode = mapper.readTree(jsonFile); + + Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields(); + + while (nodeFields.hasNext()) { + Entry<String, JsonNode> entry = nodeFields.next(); + String nodeType = entry.getKey(); + JsonNode nodeProperty = entry.getValue(); + + JsonNode filter = nodeProperty.path("filtered-node-type"); + + filteredNodeTypes.put(nodeType, filter.asText()); + } + return (filteredNodeTypes); + } + + public void loadGraph() throws IOException { + + loadGraphIntoMemory(); + buildDbEngine(); + + } + + private void loadGraphIntoMemory() throws IOException { + if (!(cArgs.isMultipleSnapshot)) { + inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled, + cArgs.isPartialGraph); + } else { + inMemGraph = new InMemoryGraph.Builder().build(sequenceInputStreams, cArgs.config, cArgs.schemaEnabled, + cArgs.isPartialGraph); + } + } + + private void buildDbEngine() { + // TODO : parametrise version + loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); + + dbEngine = new InMemoryDBEngine(queryStyle, loader, inMemGraph.getGraph()); + dbEngine.startTransaction(); + } + + private void generatePayload(List<Map<String, List<String>>> nodeFilters, + Map<String, Map<String, String>> inputFilters, Map<String, String> filteredNodeTypes) + throws AAIException, IOException { + + Map<String, List<String>> filterCousinsMap = nodeFilters.get(0); + Map<String, List<String>> filterParentsMap = nodeFilters.get(1); + Set<String> nodeTypes = filterCousinsMap.keySet(); + + for (String nodeType : nodeTypes) { + if ("DMAAP-MR".equals(cArgs.format)) { + bw = createFile(nodeType + ".json"); + } + List<String> filterCousins = filterCousinsMap.get(nodeType); + List<String> filterParents = filterParentsMap.get(nodeType); + Map<String, String> nodeInputFilterMap = inputFilters.get(nodeType); + String filteredNodeType = nodeType; + if(filteredNodeTypes.get(nodeType) != null && !filteredNodeTypes.get(nodeType).isEmpty()) + filteredNodeType = filteredNodeTypes.get(nodeType); + readVertices(nodeType, filterCousins, filterParents, nodeInputFilterMap, filteredNodeType); + if(bw != null) + bw.close(); + LOGGER.debug("All Done-" + nodeType); + } + + } + + private BufferedWriter createFile(String outfileName) throws IOException { + // FileLocation + String fileName = outfileName; + File outFile = new File(fileName); + FileWriter fw = null; + LOGGER.debug(" Will write to " + fileName); + try { + fw = new FileWriter(outFile.getAbsoluteFile()); + } catch (IOException i) { + String emsg = "Unable to write to " + fileName + " Exception = " + i.getMessage(); + LOGGER.error(emsg); + System.out.println(emsg); + throw i; + } + return new BufferedWriter(fw); + } + + private void createDirectory(String dirName) throws IOException { + // FileLocation + Path pathDir = null; + try { + pathDir = Paths.get(dirName); + } catch (InvalidPathException i) { + String emsg = "Directory " + dirName + " could not be found."; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } + try { + Files.createDirectories(pathDir); + } catch (Exception e) { + String emsg = "Directory " + dirName + " could not be created. Exception = " + e.getMessage(); + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } + } + + public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents, + Map<String, String> nodeInputFilters, String filteredNodeType) throws AAIException, IOException { + + DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth"); + + /* + * Start with nodeType you need to filter and then traverse to the actual nodeType + */ + GraphTraversal<Vertex, Vertex> gtraversal = inMemGraph.getGraph().traversal().V().has("aai-node-type", + filteredNodeType); + + + // input regex + if (nodeInputFilters != null && (!nodeInputFilters.isEmpty())) { + for (Map.Entry<String, String> entry : nodeInputFilters.entrySet()) { + String property = entry.getKey(); + String regex = entry.getValue(); + Pattern pa = Pattern.compile(regex); + + gtraversal = gtraversal.has(property, P.test((t, p) -> { + Matcher m = ((Pattern) p).matcher((CharSequence) t); + boolean b = m.matches(); + return b; + }, pa)); + } + } + + /* + * Tenant, AZ, Complex, Zone, pserver come here + */ + if (!filteredNodeType.equals(nodeType)) { + + EdgeRuleQuery treeEdgeRuleQuery = new EdgeRuleQuery + .Builder(filteredNodeType, nodeType) + .edgeType(EdgeType.TREE) + .build(); + + EdgeRuleQuery cousinEdgeQuery = new EdgeRuleQuery + .Builder(filteredNodeType, nodeType) + .edgeType(EdgeType.COUSIN) + .build(); + + EdgeRule rule = null; + boolean hasTreeEdgeRule = true; + + try { + rule = edgeRules.getRule(treeEdgeRuleQuery); + } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) { + hasTreeEdgeRule = false; + } + + if(!hasTreeEdgeRule) { + try { + rule = edgeRules.getRule(cousinEdgeQuery); + } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) { + LOGGER.error("Unable to get a tree or cousin edge between {} and {}", filteredNodeType, nodeType); + return; + } + } + + if (rule.getDirection().toString().equals(AAIDirection.OUT.toString())) { + gtraversal.out(rule.getLabel()).has("aai-node-type", nodeType); + } else { + gtraversal.in(rule.getLabel()).has("aai-node-type", nodeType); + } + + } + + String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP; + createDirectory(dirName); + // TODO: Formatter + + if ("DMAAP-MR".equals(cArgs.format)) { + while (gtraversal.hasNext()) { + if (bw != null) + bw = createFile(nodeType + ".json"); + Vertex node = gtraversal.next(); + Introspector nodeObj = serializer.getLatestVersionView(node); + createPayloadForDmaap(node, nodeObj); + } + } else { + if ("PAYLOAD".equals(cArgs.format)) { + int counter = 0; + while (gtraversal.hasNext()) { + Vertex node = gtraversal.next(); + try { + counter++; + String filename = dirName + counter + "-" + nodeType + ".json"; + bw = createFile(filename); + Introspector obj = loader.introspectorFromName(nodeType); + Set<Vertex> seen = new HashSet<>(); + int depth = AAIProperties.MAXIMUM_DEPTH; + boolean nodeOnly = false; + + Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly); + TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree); + serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins, + filterParents); + createPayloadForPut(obj); + if(bw != null) + bw.close(); + + URI uri = serializer.getURIForVertex(node); + String filenameWithUri = dirName + counter + "-" + nodeType + ".txt"; + bw = createFile(filenameWithUri); + bw.write(uri.toString()); + bw.newLine(); + bw.close(); + } catch (Exception e) { + String emsg = "Caught exception while processing [" + counter + "-" + nodeType + "] continuing"; + System.out.println(emsg); + LOGGER.error(emsg); + + } + } + } + } + + } + + public void createPayloadForPut(Introspector nodeObj) throws IOException { + + String entityJson = nodeObj.marshal(false); + ObjectMapper mapper = new ObjectMapper(); + + ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson); + rootNode.remove("resource-version"); + + bw.newLine(); + bw.write(rootNode.toString()); + bw.newLine(); + } + + public void createPayloadForDmaap(Vertex node, Introspector nodeObj) + throws AAIException, UnsupportedEncodingException { + + DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth"); + + URI uri = serializer.getURIForVertex(node); + + String sourceOfTruth = ""; + HashMap<String, Introspector> relatedVertices = new HashMap<>(); + List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node); + + for (Vertex vertex : vertexChain) { + try { + + Introspector vertexObj = serializer.getVertexProperties(vertex); + + relatedVertices.put(vertexObj.getObjectId(), vertexObj); + } catch (AAIUnknownObjectException e) { + LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned"); + } + + } + + String transactionId = "TXID"; + createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices); + + } + + public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj, + Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException { + + String action = "CREATE"; + final Introspector notificationEvent = loader.introspectorFromName("notification-event"); + + try { + Introspector eventHeader = loader.introspectorFromName("notification-event-header"); + URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects); + + String entityLink = urlBase + version + uri; + + notificationEvent.setValue("cambria-partition", "AAI"); + + eventHeader.setValue("entity-link", entityLink); + eventHeader.setValue("action", action); + eventHeader.setValue("entity-type", obj.getDbName()); + eventHeader.setValue("top-entity-type", parser.getTopEntityName()); + eventHeader.setValue("source-name", sourceOfTruth); + eventHeader.setValue("version", version.toString()); + eventHeader.setValue("id", transactionId); + eventHeader.setValue("event-type", "AAI-BASELINE"); + if (eventHeader.getValue("domain") == null) { + eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK")); + } + + if (eventHeader.getValue("sequence-number") == null) { + eventHeader.setValue("sequence-number", + AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK")); + } + + if (eventHeader.getValue("severity") == null) { + eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK")); + } + + if (eventHeader.getValue("id") == null) { + eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString()); + + } + + if (eventHeader.getValue("timestamp") == null) { + eventHeader.setValue("timestamp", genDate()); + } + + List<Object> parentList = parser.getParentList(); + parentList.clear(); + + if (!parser.getTopEntity().equals(parser.getEntity())) { + Introspector child; + String json = obj.marshal(false); + child = parser.getLoader().unmarshal(parser.getEntity().getName(), json); + parentList.add(child.getUnderlyingObject()); + } + + final Introspector eventObject; + + String json = ""; + if (parser.getTopEntity().equals(parser.getEntity())) { + json = obj.marshal(false); + eventObject = loader.unmarshal(obj.getName(), json); + } else { + json = parser.getTopEntity().marshal(false); + + eventObject = loader.unmarshal(parser.getTopEntity().getName(), json); + } + notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject()); + notificationEvent.setValue("entity", eventObject.getUnderlyingObject()); + + String entityJson = notificationEvent.marshal(false); + + bw.newLine(); + bw.write(entityJson); + + } catch (AAIUnknownObjectException e) { + LOGGER.error("Fatal error - notification-event-header object not found!"); + } catch (Exception e) { + LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e)); + } + } + + private void closeGraph() { + inMemGraph.getGraph().tx().rollback(); + inMemGraph.getGraph().close(); + } + + public static String genDate() { + Date date = new Date(); + DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS"); + return formatter.format(date); + } + + public static String genDate2() { + Date date = new Date(); + DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss"); + return formatter.format(date); + } + + private void validateFile(String filename) { + File f = new File(filename); + if (!f.exists()) { + String emsg = "File " + filename + " could not be found."; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } else if (!f.canRead()) { + String emsg = "File " + filename + " could not be read."; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } else if (f.length() == 0) { + String emsg = "File " + filename + " had no data."; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } + } + + private InputStream validateMultipleSnapshots(String filenamePrefix) { + if (filenamePrefix == null || filenamePrefix.length() == 0) { + String emsg = "No snapshot path was provided."; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } + String targetDir = "."; + int lastSeparator = filenamePrefix.lastIndexOf(File.separator); + + LOGGER.debug("File separator=[" + File.separator + "] lastSeparator=" + lastSeparator + " filenamePrefix=" + + filenamePrefix); + if (lastSeparator >= 0) { + targetDir = filenamePrefix.substring(0, lastSeparator); + LOGGER.debug("targetDir=" + targetDir); + } + if (targetDir.length() == 0) { + String emsg = "No snapshot directory was found in path:" + filenamePrefix; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } + String prefix = filenamePrefix.substring(lastSeparator + 1); + if (prefix == null || prefix.length() == 0) { + String emsg = "No snapshot file prefix was provided."; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } + long timeA = System.nanoTime(); + + ArrayList<File> snapFilesArr = new ArrayList<File>(); + String thisSnapPrefix = prefix + ".P"; + File fDir = new File(targetDir); // Snapshot directory + File[] allFilesArr = fDir.listFiles(); + for (File snapFile : allFilesArr) { + String snapFName = snapFile.getName(); + if (snapFName.startsWith(thisSnapPrefix)) { + snapFilesArr.add(snapFile); + } + } + + if (snapFilesArr.isEmpty()) { + String fullFName = targetDir + AAIConstants.AAI_FILESEP + thisSnapPrefix; + String emsg = "Snapshot files " + fullFName + "* could not be found."; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } + + int fCount = snapFilesArr.size(); + Iterator<File> fItr = snapFilesArr.iterator(); + Vector<InputStream> inputStreamsV = new Vector<>(); + for (int i = 0; i < fCount; i++) { + File f = snapFilesArr.get(i); + String fname = f.getName(); + if (!f.canRead()) { + String emsg = "Snapshot file " + fname + " could not be read."; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } else if (f.length() == 0) { + String emsg = "Snapshot file " + fname + " had no data."; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } + String fullFName = targetDir + AAIConstants.AAI_FILESEP + fname; + InputStream fis = null; + try { + fis = new FileInputStream(fullFName); + } catch (FileNotFoundException e) { + // should not happen at this point + String emsg = "Snapshot file " + fullFName + " could not be found"; + LOGGER.error(emsg); + System.out.println(emsg); + taskExit(); + } + inputStreamsV.add(fis); + } + // Now add inputStreams.elements() to the Vector, + InputStream sis = new SequenceInputStream(inputStreamsV.elements()); + return (sis); + } + + public InMemoryGraph getInMemGraph() { + return inMemGraph; + } + + public void setInMemGraph(InMemoryGraph inMemGraph) { + this.inMemGraph = inMemGraph; + } +} + +class CommandLineArgs { + + @Parameter(names = "--help", help = true) + public boolean help; + + @Parameter(names = "-d", description = "snapshot file to be loaded", required = true) + public String dataSnapshot; + + @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1) + public boolean schemaEnabled = true; + + @Parameter(names = "-c", description = "location of configuration file") + public String config = ""; + + @Parameter(names = "-o", description = "output location") + public String output = ""; + + @Parameter(names = "-f", description = "format of output") + public String format = "PAYLOAD"; + + @Parameter(names = "-n", description = "Node input file") + public String nodePropertyFile = ""; + + @Parameter(names = "-m", description = "multipe snapshots or not", arity = 1) + public boolean isMultipleSnapshot = false; + + @Parameter(names = "-i", description = "input filter configuration file") + public String inputFilterPropertyFile = ""; + + @Parameter(names = "-p", description = "Use the partial graph", arity = 1) + public boolean isPartialGraph = true; + +} diff --git a/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java b/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java index 790bfa1..c12f3f3 100644 --- a/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java +++ b/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java @@ -38,8 +38,6 @@ import org.onap.aai.dbmap.AAIGraphConfig; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.LogFormatTools; -import org.onap.aai.logging.LoggingContext; -import org.onap.aai.logging.LoggingContext.StatusCode; import org.onap.aai.edges.enums.AAIDirection; import org.onap.aai.edges.enums.EdgeProperty; import org.onap.aai.util.AAIConfig; @@ -47,8 +45,8 @@ import org.onap.aai.util.AAIConstants; import org.slf4j.MDC; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.janusgraph.core.JanusGraphFactory; import org.janusgraph.core.JanusGraph; @@ -85,19 +83,9 @@ public class ForceDeleteTool { Properties props = System.getProperties(); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_FORCE_DELETE_LOGBACK_PROPS); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); - EELFLogger logger = EELFManager.getInstance().getLogger(ForceDeleteTool.class.getSimpleName()); + Logger logger = LoggerFactory.getLogger(ForceDeleteTool.class.getSimpleName()); MDC.put("logFilenameAppender", ForceDeleteTool.class.getSimpleName()); - LoggingContext.init(); - LoggingContext.partnerName(FROMAPPID); - LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.component("forceDeleteTool"); - LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.targetServiceName("main"); - LoggingContext.requestId(TRANSID); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - String actionVal = ""; String userIdVal = ""; String dataString = ""; @@ -116,8 +104,6 @@ public class ForceDeleteTool { if (thisArg.equals("-action")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" No value passed with -action option. "); exit(0); } @@ -127,8 +113,6 @@ public class ForceDeleteTool { else if (thisArg.equals("-userId")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" No value passed with -userId option. "); exit(0); } @@ -144,8 +128,6 @@ public class ForceDeleteTool { else if (thisArg.equals("-vertexId")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" No value passed with -vertexId option. "); exit(0); } @@ -154,8 +136,6 @@ public class ForceDeleteTool { try { vertexIdLong = Long.parseLong(nextArg); } catch (Exception e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error("Bad value passed with -vertexId option: [" + nextArg + "]"); exit(0); @@ -164,8 +144,6 @@ public class ForceDeleteTool { else if (thisArg.equals("-params4Collect")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" No value passed with -params4Collect option. "); exit(0); } @@ -175,8 +153,6 @@ public class ForceDeleteTool { else if (thisArg.equals("-edgeId")) { i++; if (i >= args.length) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" No value passed with -edgeId option. "); exit(0); } @@ -185,8 +161,6 @@ public class ForceDeleteTool { edgeIdStr = nextArg; } else { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(" Unrecognized argument passed to ForceDeleteTool: [" + thisArg + "]. "); logger.error(" Valid values are: -action -userId -vertexId -edgeId -overRideProtection -params4Collect -DISPLAY_ALL_VIDS"); @@ -198,8 +172,6 @@ public class ForceDeleteTool { if( !actionVal.equals("COLLECT_DATA") && !actionVal.equals("DELETE_NODE") && !actionVal.equals("DELETE_EDGE")){ String emsg = "Bad action parameter [" + actionVal + "] passed to ForceDeleteTool(). Valid values = COLLECT_DATA or DELETE_NODE or DELETE_EDGE\n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(emsg); exit(0); } @@ -207,16 +179,12 @@ public class ForceDeleteTool { if( actionVal.equals("DELETE_NODE") && vertexIdLong == 0 ){ String emsg = "ERROR: No vertex ID passed on DELETE_NODE request. \n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(emsg); exit(0); } else if( actionVal.equals("DELETE_EDGE") && edgeIdStr.equals("")){ String emsg = "ERROR: No edge ID passed on DELETE_EDGE request. \n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(emsg); exit(0); } @@ -226,8 +194,6 @@ public class ForceDeleteTool { if( (userIdVal.length() < 6) || userIdVal.toUpperCase().equals("AAIADMIN") ){ String emsg = "Bad userId parameter [" + userIdVal + "] passed to ForceDeleteTool(). must be not empty and not aaiadmin \n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(emsg); exit(0); } @@ -241,8 +207,6 @@ public class ForceDeleteTool { if( graph == null ){ String emsg = "could not get graph object in ForceDeleteTool() \n"; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); logger.error(emsg); exit(0); } @@ -250,23 +214,19 @@ public class ForceDeleteTool { catch (AAIException e1) { msg = e1.getErrorObject().toString(); System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); logger.error(msg); exit(0); } catch (Exception e2) { msg = e2.toString(); System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); logger.error(msg); exit(0); } msg = "ForceDelete called by: userId [" + userIdVal + "] with these params: [" + argStr4Msg + "]"; System.out.println(msg); - logger.info(msg); + logger.debug(msg); ForceDelete fd = new ForceDelete(graph); if( actionVal.equals("COLLECT_DATA") ){ @@ -293,8 +253,6 @@ public class ForceDeleteTool { if( firstPipeLoc <= 0 ){ msg = "Must use the -params4Collect option when collecting data with data string in a format like: 'propName1|propVal1,propName2|propVal2'"; System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(msg); exit(0); } @@ -307,8 +265,6 @@ public class ForceDeleteTool { if( pipeLoc <= 0 ){ msg = "Must use the -params4Collect option when collecting data with data string in a format like: 'propName1|propVal1,propName2|propVal2'"; System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(msg); exit(0); } @@ -329,14 +285,12 @@ public class ForceDeleteTool { int descendantCount = fd.countDescendants( logger, v, 0 ); String infMsg = " Found " + descendantCount + " descendant nodes \n"; System.out.println( infMsg ); - logger.info( infMsg ); + logger.debug( infMsg ); } } else { msg = "Bad JanusGraphQuery object. "; System.out.println(msg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.error(msg); exit(0); } @@ -344,7 +298,7 @@ public class ForceDeleteTool { String infMsg = "\n\n Found: " + resCount + " nodes for this query: [" + qStringForMsg + "]\n"; System.out.println( infMsg ); - logger.info( infMsg ); + logger.debug( infMsg ); } else if( actionVal.equals("DELETE_NODE") ){ Iterator <Vertex> vtxItr = graph.vertices( vertexIdLong ); @@ -355,31 +309,31 @@ public class ForceDeleteTool { String infMsg = " Found " + descendantCount + " descendant nodes. Note - forceDelete does not cascade to " + " child nodes, but they may become unreachable after the delete. \n"; System.out.println( infMsg ); - logger.info( infMsg ); + logger.debug( infMsg ); int edgeCount = fd.countEdges( logger, vtx ); infMsg = " Found total of " + edgeCount + " edges incident on this node. \n"; System.out.println( infMsg ); - logger.info( infMsg ); + logger.debug( infMsg ); if( fd.getNodeDelConfirmation(logger, userIdVal, vtx, descendantCount, edgeCount, overRideProtection) ){ vtx.remove(); graph.tx().commit(); infMsg = ">>>>>>>>>> Removed node with vertexId = " + vertexIdLong; - logger.info( infMsg ); + logger.debug( infMsg ); System.out.println(infMsg); } else { infMsg = " Delete Cancelled. "; System.out.println(infMsg); - logger.info( infMsg ); + logger.debug( infMsg ); } } else { String infMsg = ">>>>>>>>>> Vertex with vertexId = " + vertexIdLong + " not found."; System.out.println( infMsg ); - logger.info( infMsg ); + logger.debug( infMsg ); } } else if( actionVal.equals("DELETE_EDGE") ){ @@ -391,7 +345,7 @@ public class ForceDeleteTool { if( thisEdge == null ){ String infMsg = ">>>>>>>>>> Edge with edgeId = " + edgeIdStr + " not found."; - logger.info( infMsg ); + logger.debug( infMsg ); System.out.println(infMsg); exit(0); } @@ -400,20 +354,20 @@ public class ForceDeleteTool { thisEdge.remove(); graph.tx().commit(); String infMsg = ">>>>>>>>>> Removed edge with edgeId = " + edgeIdStr; - logger.info( infMsg ); + logger.debug( infMsg ); System.out.println(infMsg); } else { String infMsg = " Delete Cancelled. "; System.out.println(infMsg); - logger.info( infMsg ); + logger.debug( infMsg ); } exit(0); } else { String emsg = "Unknown action parameter [" + actionVal + "] passed to ForceDeleteTool(). Valid values = COLLECT_DATA, DELETE_NODE or DELETE_EDGE \n"; System.out.println(emsg); - logger.info( emsg ); + logger.debug( emsg ); exit(0); } @@ -429,44 +383,41 @@ public class ForceDeleteTool { public ForceDelete(JanusGraph graph) { this.graph = graph; } - public void showNodeInfo(EELFLogger logger, Vertex tVert, Boolean displayAllVidsFlag ){ + public void showNodeInfo(Logger logger, Vertex tVert, Boolean displayAllVidsFlag ){ try { Iterator<VertexProperty<Object>> pI = tVert.properties(); String infStr = ">>> Found Vertex with VertexId = " + tVert.id() + ", properties: "; System.out.println( infStr ); - logger.info(infStr); + logger.debug(infStr); while( pI.hasNext() ){ VertexProperty<Object> tp = pI.next(); infStr = " [" + tp.key() + "|" + tp.value() + "] "; System.out.println( infStr ); - logger.info(infStr); + logger.debug(infStr); } ArrayList <String> retArr = collectEdgeInfoForNode( logger, tVert, displayAllVidsFlag ); for( String infoStr : retArr ){ System.out.println( infoStr ); - logger.info(infoStr); + logger.debug(infoStr); } } catch (Exception e){ String warnMsg = " -- Error -- trying to display edge info. [" + e.getMessage() + "]"; System.out.println( warnMsg ); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.warn(warnMsg); - LoggingContext.successStatusFields(); } }// End of showNodeInfo() - public void showPropertiesForEdge( EELFLogger logger, Edge tEd ){ + public void showPropertiesForEdge( Logger logger, Edge tEd ){ String infMsg = ""; if( tEd == null ){ infMsg = "null Edge object passed to showPropertiesForEdge()"; System.out.print(infMsg); - logger.info(infMsg); + logger.debug(infMsg); return; } @@ -474,82 +425,82 @@ public class ForceDeleteTool { try { infMsg =" Label for this Edge = [" + tEd.label() + "] "; System.out.print(infMsg); - logger.info(infMsg); + logger.debug(infMsg); infMsg =" EDGE Properties for edgeId = " + tEd.id() + ": "; System.out.print(infMsg); - logger.info(infMsg); + logger.debug(infMsg); Iterator <String> pI = tEd.keys().iterator(); while( pI.hasNext() ){ String propKey = pI.next(); infMsg = "Prop: [" + propKey + "], val = [" + tEd.property(propKey) + "] "; System.out.print(infMsg); - logger.info(infMsg); + logger.debug(infMsg); } } catch( Exception ex ){ infMsg = " Could not retrieve properties for this edge. exMsg = [" + ex.getMessage() + "] "; System.out.println( infMsg ); - logger.info(infMsg); + logger.debug(infMsg); } // Try to show what's connected to the IN side of this Edge try { infMsg = " Looking for the Vertex on the IN side of the edge: "; System.out.print(infMsg); - logger.info(infMsg); + logger.debug(infMsg); Vertex inVtx = tEd.inVertex(); Iterator<VertexProperty<Object>> pI = inVtx.properties(); String infStr = ">>> Found Vertex with VertexId = " + inVtx.id() + ", properties: "; System.out.println( infStr ); - logger.info(infStr); + logger.debug(infStr); while( pI.hasNext() ){ VertexProperty<Object> tp = pI.next(); infStr = " [" + tp.key() + "|" + tp.value() + "] "; System.out.println( infStr ); - logger.info(infStr); + logger.debug(infStr); } } catch( Exception ex ){ infMsg = " Could not retrieve vertex data for the IN side of " + "the edge. exMsg = [" + ex.getMessage() + "] "; System.out.println( infMsg ); - logger.info(infMsg); + logger.debug(infMsg); } // Try to show what's connected to the OUT side of this Edge try { infMsg = " Looking for the Vertex on the OUT side of the edge: "; System.out.print(infMsg); - logger.info(infMsg); + logger.debug(infMsg); Vertex outVtx = tEd.outVertex(); Iterator<VertexProperty<Object>> pI = outVtx.properties(); String infStr = ">>> Found Vertex with VertexId = " + outVtx.id() + ", properties: "; System.out.println( infStr ); - logger.info(infStr); + logger.debug(infStr); while( pI.hasNext() ){ VertexProperty<Object> tp = pI.next(); infStr = " [" + tp.key() + "|" + tp.value() + "] "; System.out.println( infStr ); - logger.info(infStr); + logger.debug(infStr); } } catch( Exception ex ){ infMsg = " Could not retrieve vertex data for the OUT side of " + "the edge. exMsg = [" + ex.getMessage() + "] "; System.out.println( infMsg ); - logger.info(infMsg); + logger.debug(infMsg); } }// end showPropertiesForEdge() - public ArrayList <String> collectEdgeInfoForNode( EELFLogger logger, Vertex tVert, boolean displayAllVidsFlag ){ + public ArrayList <String> collectEdgeInfoForNode( Logger logger, Vertex tVert, boolean displayAllVidsFlag ){ ArrayList <String> retArr = new ArrayList <String> (); Direction dir = Direction.OUT; for ( int i = 0; i <= 1; i++ ){ @@ -597,7 +548,7 @@ public class ForceDeleteTool { }// end of collectEdgeInfoForNode() - public int countEdges( EELFLogger logger, Vertex vtx ){ + public int countEdges( Logger logger, Vertex vtx ){ int edgeCount = 0; try { Iterator<Edge> edgesItr = vtx.edges(Direction.BOTH); @@ -609,25 +560,20 @@ public class ForceDeleteTool { catch (Exception e) { String wMsg = "-- ERROR -- Stopping the counting of edges because of Exception [" + e.getMessage() + "]"; System.out.println( wMsg ); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.warn( wMsg ); - LoggingContext.successStatusFields(); } return edgeCount; }// end of countEdges() - public int countDescendants(EELFLogger logger, Vertex vtx, int levelVal ){ + public int countDescendants(Logger logger, Vertex vtx, int levelVal ){ int totalCount = 0; int thisLevel = levelVal + 1; if( thisLevel > MAXDESCENDENTDEPTH ){ String wMsg = "Warning -- Stopping the counting of descendents because we reached the max depth of " + MAXDESCENDENTDEPTH; System.out.println( wMsg ); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.warn( wMsg ); return totalCount; } @@ -643,10 +589,7 @@ public class ForceDeleteTool { catch (Exception e) { String wMsg = "Error -- Stopping the counting of descendents because of Exception [" + e.getMessage() + "]"; System.out.println( wMsg ); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.warn( wMsg ); - LoggingContext.successStatusFields(); } @@ -654,7 +597,7 @@ public class ForceDeleteTool { }// end of countDescendants() - public boolean getEdgeDelConfirmation( EELFLogger logger, String uid, Edge ed, + public boolean getEdgeDelConfirmation( Logger logger, String uid, Edge ed, Boolean overRideProtection ) { showPropertiesForEdge( logger, ed ); @@ -667,20 +610,20 @@ public class ForceDeleteTool { if (!confirm.equalsIgnoreCase("y")) { String infMsg = " User [" + uid + "] has chosen to abandon this delete request. "; System.out.println("\n" + infMsg); - logger.info(infMsg); + logger.debug(infMsg); return false; } else { String infMsg = " User [" + uid + "] has confirmed this delete request. "; System.out.println("\n" + infMsg); - logger.info(infMsg); + logger.debug(infMsg); return true; } } // End of getEdgeDelConfirmation() - public boolean getNodeDelConfirmation( EELFLogger logger, String uid, Vertex vtx, int edgeCount, + public boolean getNodeDelConfirmation( Logger logger, String uid, Vertex vtx, int edgeCount, int descendantCount, Boolean overRideProtection ) { String thisNodeType = ""; try { @@ -690,10 +633,7 @@ public class ForceDeleteTool { // Let the user know something is going on - but they can confirm the delete if they want to. String infMsg = " -- WARNING -- could not get an aai-node-type for this vertex. -- WARNING -- "; System.out.println( infMsg ); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.warn( infMsg ); - LoggingContext.successStatusFields(); } String ntListString = ""; @@ -714,10 +654,7 @@ public class ForceDeleteTool { // Don't worry, we will use default values String infMsg = "-- WARNING -- could not get aai.forceDel.protected values from aaiconfig.properties -- will use default values. "; System.out.println( infMsg ); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logger.warn( infMsg ); - LoggingContext.successStatusFields(); } if( maxDescString != null && !maxDescString.equals("") ){ @@ -752,7 +689,7 @@ public class ForceDeleteTool { String infMsg = " >> WARNING >> This node has more descendant edges than the max ProtectedDescendantCount: " + edgeCount + ". Max = " + maxEdgeCount + ". It can be DANGEROUS to delete one of these. << WARNING << "; System.out.println(infMsg); - logger.info(infMsg); + logger.debug(infMsg); if( ! overRideProtection ){ // They cannot delete this kind of node without using the override option giveProtErrorMsg = true; @@ -767,7 +704,7 @@ public class ForceDeleteTool { String infMsg = " >> WARNING >> This node has more edges than the max ProtectedEdgeCount: " + edgeCount + ". Max = " + maxEdgeCount + ". It can be DANGEROUS to delete one of these. << WARNING << "; System.out.println(infMsg); - logger.info(infMsg); + logger.debug(infMsg); if( ! overRideProtection ){ // They cannot delete this kind of node without using the override option giveProtErrorMsg = true; @@ -782,7 +719,7 @@ public class ForceDeleteTool { String infMsg = " >> WARNING >> This node is a PROTECTED NODE-TYPE (" + thisNodeType + "). " + " It can be DANGEROUS to delete one of these. << WARNING << "; System.out.println(infMsg); - logger.info(infMsg); + logger.debug(infMsg); if( ! overRideProtection ){ // They cannot delete this kind of node without using the override option giveProtErrorMsg = true; @@ -795,15 +732,12 @@ public class ForceDeleteTool { if( giveProtOverRideMsg ){ String infMsg = " !!>> WARNING >>!! you are using the overRideProtection parameter which will let you do this potentially dangerous delete."; System.out.println("\n" + infMsg); - logger.info(infMsg); + logger.debug(infMsg); } else if( giveProtErrorMsg ) { String errMsg = " ERROR >> this kind of node can only be deleted if you pass the overRideProtection parameter."; System.out.println("\n" + errMsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logger.error(errMsg); - LoggingContext.successStatusFields(); return false; } @@ -816,20 +750,20 @@ public class ForceDeleteTool { if (!confirm.equalsIgnoreCase("y")) { String infMsg = " User [" + uid + "] has chosen to abandon this delete request. "; System.out.println("\n" + infMsg); - logger.info(infMsg); + logger.debug(infMsg); return false; } else { String infMsg = " User [" + uid + "] has confirmed this delete request. "; System.out.println("\n" + infMsg); - logger.info(infMsg); + logger.debug(infMsg); return true; } } // End of getNodeDelConfirmation() } - public static JanusGraph setupGraph(EELFLogger logger){ + public static JanusGraph setupGraph(Logger logger){ JanusGraph janusGraph = null; @@ -856,7 +790,7 @@ public class ForceDeleteTool { return janusGraph; } - public static void closeGraph(JanusGraph graph, EELFLogger logger){ + public static void closeGraph(JanusGraph graph, Logger logger){ try { if("inmemory".equals(graphType)) { diff --git a/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java b/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java index ebe2180..a170453 100644 --- a/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java +++ b/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java @@ -45,8 +45,8 @@ import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper; import org.apache.tinkerpop.shaded.jackson.databind.node.JsonNodeType; import org.onap.aai.dbmap.InMemoryGraph; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -76,7 +76,7 @@ public final class GraphSONPartialReader implements GraphReader { private boolean unwrapAdjacencyList = false; private final GraphSONReader reader; - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(InMemoryGraph.class); + private static final Logger LOGGER = LoggerFactory.getLogger(InMemoryGraph.class); final TypeReference<Map<String, Object>> mapTypeReference = new TypeReference<Map<String, Object>>() { }; diff --git a/src/main/java/org/onap/aai/dbgen/UpdatePropertyTool.java b/src/main/java/org/onap/aai/dbgen/UpdatePropertyTool.java new file mode 100644 index 0000000..032fc9e --- /dev/null +++ b/src/main/java/org/onap/aai/dbgen/UpdatePropertyTool.java @@ -0,0 +1,79 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.dbgen; + +import org.janusgraph.core.JanusGraph; +import org.onap.aai.util.AAIConstants; +import com.att.eelf.configuration.Configuration; +import org.slf4j.MDC; + +import java.util.Properties; +import java.util.UUID; + +public class UpdatePropertyTool { + + private static final String FROMAPPID = "AAI-DB"; + private static final String TRANSID = UUID.randomUUID().toString(); + + public static boolean SHOULD_EXIT_VM = true; + + public static int EXIT_VM_STATUS_CODE = -1; + public static int EXIT_VM_STATUS_CODE_SUCCESS = 0; + public static int EXIT_VM_STATUS_CODE_FAILURE = 1; + public static final String PROPERTY_LOGGING_FILE_NAME = "updatePropertyTool-logback.xml"; + + public static void exit(int statusCode){ + if(SHOULD_EXIT_VM){ + System.exit(statusCode); + } + EXIT_VM_STATUS_CODE = statusCode; + } + + /** + * The main method. + * + * @param args the arguments + */ + public static void main(String[] args) + { + System.setProperty("aai.service.name", UpdatePropertyTool.class.getSimpleName()); + // Set the logging file properties to be used by EELFManager + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, PROPERTY_LOGGING_FILE_NAME); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); + MDC.put("logFilenameAppender", UpdatePropertyTool.class.getSimpleName()); + + + UpdatePropertyToolInternal updatePropertyToolInternal = new UpdatePropertyToolInternal(); + JanusGraph graph = updatePropertyToolInternal.openGraph(AAIConstants.REALTIME_DB_CONFIG); + + try { + EXIT_VM_STATUS_CODE = updatePropertyToolInternal.run(graph, args) ? EXIT_VM_STATUS_CODE_SUCCESS : EXIT_VM_STATUS_CODE_FAILURE; + } catch (Exception e) { + e.printStackTrace(); + EXIT_VM_STATUS_CODE = EXIT_VM_STATUS_CODE_FAILURE; + } finally { + updatePropertyToolInternal.closeGraph(graph); + } + + exit(EXIT_VM_STATUS_CODE); + } +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/dbgen/UpdatePropertyToolInternal.java b/src/main/java/org/onap/aai/dbgen/UpdatePropertyToolInternal.java new file mode 100644 index 0000000..79b988d --- /dev/null +++ b/src/main/java/org/onap/aai/dbgen/UpdatePropertyToolInternal.java @@ -0,0 +1,383 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.dbgen; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.JanusGraphFactory; +import org.janusgraph.core.JanusGraphTransaction; +import org.onap.aai.dbmap.AAIGraphConfig; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.util.AAIConfig; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class UpdatePropertyToolInternal { + private final Logger LOGGER; + + @Parameter(names = {"--filename", "-f"}, description = "Enter path to the file name containing a list of your vertex IDs separated by new lines") + String filename; + @Parameter(names = {"--vertexId", "-v"}, description = "Vertex id(s) used to update the node(s). Flag can be used multiple times to enter a list of vertex ids") + private List<String> vertexIds = new ArrayList<>(); + @Parameter(names = {"--property", "-p"}, description = "Property to be updated within the given node(s)", required = true) + String property; + @Parameter(names = {"-h", "--help"}, + description = "Help/Usage", + help = true) + boolean help; + + public UpdatePropertyToolInternal(){ + LOGGER = LoggerFactory.getLogger(UpdatePropertyToolInternal.class.getSimpleName()); + } + + /** + * Run method for the main method of the update property tool. Returns boolean to indicate success or failure + * @param graph + * @param args + * @return + */ + public boolean run(JanusGraph graph, String[] args) { + logAndPrint("===============Start UpdatePropertyTool==============="); + boolean isSuccessful = true; + try { + // Error check args + if (!processCommandLineArguments(args)) { + isSuccessful = false; + } + + // Aggregate all ids into one list + List<String> vertexIdList = aggregateVertexIdList(filename, vertexIds); + + // Attempt update on given vertex ids + if (!processUpdateTransaction(graph, vertexIdList, property)) { + isSuccessful = false; + } + + } catch (Exception e) { + isSuccessful = false; + logErrorAndPrint("ERROR exception thrown in run() method of UpdatePropertyTool", e); + } + logAndPrint("===============End UpdatePropertyTool==============="); + return isSuccessful; + } + + /** + * Use JCommander to process the provided command-line arguments. + * This method is an instance method (not static) to allow JCommander + * to use this instance's JCommander-annotated fields. + * + * @param args Command-line arguments. + */ + private boolean processCommandLineArguments(final String[] args) { + logAndPrint("Start of processCommandLineArguments()"); + final JCommander commander = new JCommander(); + commander.addObject(this); + commander.setVerbose(1); + commander.parse(args); + commander.setProgramName(UpdatePropertyTool.class.getSimpleName()); + boolean filenameExists = false; + boolean vertexIdExists = false; + boolean isValidArgs = true; + + // check for help flag + if (help) { + commander.usage(); + } + + // Check for property field + if (property != null && !property.isEmpty()) { + logAndPrint("The property provided is: " + property); + } else { + logAndPrint("ERROR: No property argument was entered. Property argument is required."); + isValidArgs = false; + } + + // Check for file name + if (filename != null && !filename.isEmpty()) { + logAndPrint("The filename provided is: " + filename); + filenameExists = true; + } + + // Check for vertex Ids + if (vertexIds != null && !vertexIds.isEmpty()) { + logAndPrint("The vertex id(s) provided: ".concat(String.join("|", vertexIds))); + vertexIdExists = true; + } + + // Fail and exit if there are no vertexIds to work with + if (!filenameExists && !vertexIdExists) { + isValidArgs = false; + logAndPrint("ERROR: Cannot execute UpdatePropertyTools without any given vertex Ids."); + } + logAndPrint("End of processCommandLineArguments()"); + + return isValidArgs; + } + + /** + * Executes Gremlin queries to obtain vertices by their ids and updates the property defined by the given property parameter + * Reads a list of vertex IDs, and uses the prop parameter to indicate which property has corrupt index + * Uses gremlin console to get the property value and use that value returned to set the property and update. + * + * @param vIdList + * @param propertyKey property value to be updated + */ + private boolean processUpdateTransaction(JanusGraph graph, List<String> vIdList, String propertyKey) { + logAndPrint("Start of processUpdateTransaction()"); + boolean isValidTransaction = true; + + if (graph == null) { + logAndPrint("JanusGraph graph object is null. Stopping processUpdateTransaction()"); + return false; + } + + if (vIdList == null || vIdList.isEmpty()) { + logAndPrint("Vertex Id list is null or empty. Stopping processUpdateTransaction()"); + return false; + } + + if (propertyKey == null || propertyKey.isEmpty()) { + logAndPrint("propertyKey is null or empty. Stopping processUpdateTransaction()"); + return false; + } + + // if AAIConfig.init() fails, exit application + if (!setUpAAIConfig(graph)) { + isValidTransaction = false; + } + + // Obtain the vertex objects using the given vertex ids + JanusGraphTransaction transaction = graph.newTransaction(); + + try { + GraphTraversalSource g = transaction.traversal(); + boolean isCommitUpdateValid = false; + for (String vertexId: vIdList) { + /* + * Query the vertex using the vertex id from the graph + * Check if the query obtained a vertex + * Get the property value from the vertex itself + * Update the property using the value obtained from the query + */ + GraphTraversal<Vertex, Vertex> query = g.V(vertexId); + if (query.hasNext()) { + Vertex vertex = query.next(); + Object propertyValue = vertex.property(propertyKey).orElse(null); + if (propertyValue != null) { + vertex.property(propertyKey, propertyValue); + isCommitUpdateValid = true; + logAndPrint("Updated vertex with property: '" + propertyKey + "'. With value of: '" + propertyValue.toString() + "'"); + } else { + logAndPrint("Could not update the value for property '" + propertyKey + "'. Value was empty."); + } + } else { + logAndPrint("Vertex not found for id: " + vertexId); + } + } + + // If a transaction to update a property has occurred, commit the transaction(s) + if (isCommitUpdateValid) { + transaction.commit(); + logAndPrint("Successful update transaction has occurred. Committing update to graph."); + } else { + transaction.rollback(); + logAndPrint("Unsuccessful update transaction. Rolling back graph"); + } + } catch (Exception e) { + if (transaction != null) { + transaction.rollback(); + } else { + logAndPrint("ERROR: JanusGraphTransaction object is null"); + } + logErrorAndPrint("ERROR: Could not properly query and update vertex.", e); + isValidTransaction = false; + } finally { + // close the transaction -- note: JanusGraph graph object will be closed in the main method. + if (transaction != null) { + transaction.close(); + } else { + logAndPrint("ERROR: JanusGraphTransaction object is null. Cannot close the transaction."); + } + } + + logAndPrint("End of processUpdateTransaction()"); + return isValidTransaction; + } + + /** + * Combine the vertex ids from the file and list of ids given + * @param filePath + * @param vertexIds + * @return + */ + private List<String> aggregateVertexIdList(String filePath, List<String> vertexIds) { + List<String> allVertexIds = new ArrayList<>(); + + if (filePath != null && !filePath.isEmpty()) { + // Add vertex Ids listed from the given file name + try { + logAndPrint("Loading file at: " + filePath); + BufferedReader br = new BufferedReader(new FileReader(filePath)); + StringBuilder sb = new StringBuilder(); + String nextLine = br.readLine(); + + while (nextLine != null) { + if (!nextLine.matches("[0-9]+")) { + logAndPrint("Invalid vertex id: " + nextLine); + continue; + } + allVertexIds.add(nextLine); + sb.append(nextLine); + sb.append(System.lineSeparator()); + nextLine = br.readLine(); + } + String allVertexIdsString = sb.toString(); + logAndPrint("All vertex IDs from file " + filePath + ":\n" + allVertexIdsString); + br.close(); + } catch (IOException ioe) { + logErrorAndPrint("ERROR reading in text file failed.", ioe); + } + } + + // Add all vertex Ids input one at a time from args parameter + StringBuilder sb = new StringBuilder(); + for (String vId : vertexIds) { + if (!vId.matches("[0-9]+")) { + logAndPrint("Invalid vertex id: " + vId); + continue; + } + allVertexIds.add(vId); + sb.append(vId); + sb.append(System.lineSeparator()); + } + logAndPrint("Vertex IDs from --vertexId args:\n" + sb.toString()); + + return allVertexIds; + } + + /** + * Set up AAIConfig object + * + * @return + */ + private boolean setUpAAIConfig(JanusGraph graph) { + String msg = ""; + try { + AAIConfig.init(); + if (graph == null) { + String emsg = "Graph is null. Could not get graph object. \n"; + logAndPrint(emsg); + return false; + } + } catch (AAIException e1) { + msg = e1.getErrorObject().toString(); + logErrorAndPrint("ERROR: AAIConfig set up failed. ", e1); + return false; + } catch (Exception e2) { + msg = e2.toString(); + logErrorAndPrint("ERROR: AAIConfig set up failed. ", e2); + return false; + } + return true; + } + + /** + * Set up and return and open JanusGraph Object + * + * @return + */ + public JanusGraph openGraph(String configPath) { + logAndPrint("Setting up Janus Graph..."); + JanusGraph janusGraph = null; + + try { + janusGraph = JanusGraphFactory.open( + new AAIGraphConfig.Builder(configPath) + .forService(UpdatePropertyTool.class.getSimpleName()) + .withGraphType("AAITools-" + UpdatePropertyTool.class.getSimpleName()) + .buildConfiguration() + ); + } catch (Exception e) { + logErrorAndPrint("Unable to open the graph. ", e); + } + + return janusGraph; + } + + /** + * Closes the given JanusGraph object + * + * @param graph + */ + public void closeGraph(JanusGraph graph) { + + try { + if (graph != null && graph.isOpen()) { + graph.tx().close(); + graph.close(); + } + } catch (Exception ex) { + // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed + logErrorAndPrint("WARNING from final graph.shutdown(): ", ex); + } + } + + /** + * Log and print. + * + * @param msg the msg + */ + protected void logAndPrint(String msg) { + System.out.println(msg); + LOGGER.error(msg); + } + + /** + * Log error and print. + * + * @param msg the msg + */ + protected void logErrorAndPrint(String msg, Exception e) { + System.out.println(msg); + System.out.println(e.getCause() + " - " + e.getMessage()); + LOGGER.error(msg, e); + } + + /** + * Log error and print. + * + * @param msg the msg + */ + protected void logErrorAndPrint(String msg) { + System.out.println(msg); + LOGGER.error(msg); + } +} diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java index d1c283f..15b4c81 100644 --- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java +++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java @@ -20,16 +20,12 @@ package org.onap.aai.dbgen.schemamod; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; import org.onap.aai.config.PropertyPasswordConfiguration; -import org.onap.aai.dbmap.DBConnectionType; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Loader; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.introspection.ModelType; import org.onap.aai.logging.ErrorLogHelper; -import org.onap.aai.logging.LoggingContext; import org.onap.aai.serialization.engines.JanusGraphDBEngine; import org.onap.aai.serialization.engines.QueryStyle; import org.onap.aai.serialization.engines.TransactionalGraphEngine; @@ -38,7 +34,9 @@ import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConfig; import org.onap.aai.util.AAIConstants; import org.onap.aai.util.ExceptionTranslator; -import org.onap.aai.util.UniquePropertyCheck; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.slf4j.MDC; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @@ -49,6 +47,11 @@ public class SchemaMod { private final LoaderFactory loaderFactory; private final SchemaVersions schemaVersions; + + private static boolean historyEnabled; + + private Logger logger = LoggerFactory.getLogger(SchemaMod.class.getSimpleName()); + public SchemaMod(LoaderFactory loaderFactory, SchemaVersions schemaVersions){ this.loaderFactory = loaderFactory; @@ -57,33 +60,37 @@ public class SchemaMod { public void execute(String[] args) { - // Set the logging file properties to be used by EELFManager - Properties props = System.getProperties(); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_SCHEMA_MOD_LOGBACK_PROPS); - props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); - - EELFLogger logger = EELFManager.getInstance().getLogger(UniquePropertyCheck.class.getSimpleName()); - MDC.put("logFilenameAppender", SchemaMod.class.getSimpleName()); - + logger = LoggerFactory.getLogger(SchemaMod.class.getSimpleName()); + // NOTE -- We're just working with properties that are used for NODES // for now. String propName = ""; String targetDataType = ""; String targetIndexInfo = ""; String preserveDataFlag = ""; + String commitBlockSizeStr = ""; + long commitBlockSize = 120000; - String usageString = "Usage: SchemaMod propertyName targetDataType targetIndexInfo preserveDataFlag \n"; - if (args.length != 4) { - String emsg = "Four Parameters are required. \n" + usageString; - logAndPrint(logger, emsg); - System.exit(1); - } else { + String usageString = "Usage: SchemaMod propertyName targetDataType targetIndexInfo preserveDataFlag [blockSize] \n"; + + if (args.length == 4) { propName = args[0]; targetDataType = args[1]; targetIndexInfo = args[2]; preserveDataFlag = args[3]; } - + else if (args.length == 5) { + propName = args[0]; + targetDataType = args[1]; + targetIndexInfo = args[2]; + preserveDataFlag = args[3]; + commitBlockSizeStr = args[4]; + } + else { + String emsg = "Incorrect number of Parameters passed. \n" + usageString; + logAndPrint(logger, emsg); + System.exit(1); + } if (propName.equals("")) { String emsg = "Bad parameter - propertyName cannot be empty. \n" + usageString; logAndPrint(logger, emsg); @@ -102,6 +109,17 @@ public class SchemaMod { logAndPrint(logger, emsg); System.exit(1); } + + try { + if( !commitBlockSizeStr.equals("")) { + // They're over-riding the commitBlockSize + commitBlockSize = Long.parseLong(commitBlockSizeStr); + } + } catch (NumberFormatException nfe) { + String emsg = "NumberFormatException - Bad block size passed in: [" + commitBlockSizeStr + "]. "; + logAndPrint(logger, emsg ); + System.exit(1); + } try { AAIConfig.init(); @@ -112,15 +130,6 @@ public class SchemaMod { System.exit(1); } - // Give a big warning if the DbMaps.PropertyDataTypeMap value does not - // agree with what we're doing - String warningMsg = ""; - - if (!warningMsg.equals("")) { - logAndPrint(logger, "\n>>> WARNING <<<< "); - logAndPrint(logger, ">>> " + warningMsg + " <<<"); - } - logAndPrint(logger, ">>> Processing will begin in 5 seconds (unless interrupted). <<<"); try { // Give them a chance to back out of this @@ -138,8 +147,8 @@ public class SchemaMod { Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); TransactionalGraphEngine engine = null; try { - engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader); - SchemaModInternal internal = new SchemaModInternal(engine, logger, propName, targetDataType, targetIndexInfo, new Boolean(preserveDataFlag)); + engine = new JanusGraphDBEngine(queryStyle, loader); + SchemaModInternalBatch internal = new SchemaModInternalBatch(engine, logger, propName, targetDataType, targetIndexInfo, Boolean.parseBoolean(preserveDataFlag), commitBlockSize); internal.execute(); engine.startTransaction(); engine.tx().close(); @@ -157,13 +166,19 @@ public class SchemaMod { * @param logger the logger * @param msg the msg */ - protected void logAndPrint(EELFLogger logger, String msg) { + protected void logAndPrint(Logger logger, String msg) { System.out.println(msg); - logger.info(msg); + logger.debug(msg); } public static void main(String[] args) throws AAIException { + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_SCHEMA_MOD_LOGBACK_PROPS); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); + + MDC.put("logFilenameAppender", SchemaMod.class.getSimpleName()); + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); initializer.initialize(ctx); @@ -176,11 +191,17 @@ public class SchemaMod { } catch (Exception e) { AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); System.out.println("Problems running SchemaMod "+aai.getMessage()); - LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } + + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); + if( historyEnabled ) { + String emsg = "Regular SchemaMod may not be used when history.enabled=true. "; + System.out.println(emsg); + throw new AAIException("AAI-4005",emsg); + } + LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions"); SchemaMod schemaMod = new SchemaMod(loaderFactory, schemaVersions); diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java new file mode 100644 index 0000000..837f8a8 --- /dev/null +++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java @@ -0,0 +1,200 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.dbgen.schemamod; + +import com.att.eelf.configuration.Configuration; +import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; +import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.serialization.engines.JanusGraphDBEngine; +import org.onap.aai.serialization.engines.QueryStyle; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.setup.SchemaVersions; +import org.onap.aai.util.AAIConfig; +import org.onap.aai.util.AAIConstants; +import org.onap.aai.util.ExceptionTranslator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import java.util.Properties; + +public class SchemaMod4Hist { + + private final LoaderFactory loaderFactory; + + private final SchemaVersions schemaVersions; + + private static boolean historyEnabled; + + public SchemaMod4Hist(LoaderFactory loaderFactory, SchemaVersions schemaVersions){ + this.loaderFactory = loaderFactory; + this.schemaVersions = schemaVersions; + } + + public void execute(String[] args) { + + // Set the logging file properties to be used by EELFManager + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_SCHEMA_MOD_LOGBACK_PROPS); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); + + Logger logger = LoggerFactory.getLogger(SchemaMod4Hist.class.getSimpleName()); + MDC.put("logFilenameAppender", SchemaMod4Hist.class.getSimpleName()); + + + // NOTE -- We're just working with properties that are used for NODES + // for now. + String propName = ""; + String targetDataType = ""; + String targetIndexInfo = ""; + String preserveDataFlag = ""; + + String usageString = "Usage: SchemaMod4Hist propertyName targetDataType targetIndexInfo preserveDataFlag \n"; + if (args.length != 4) { + String emsg = "Four Parameters are required. \n" + usageString; + logAndPrint(logger, emsg); + System.exit(1); + } else { + propName = args[0]; + targetDataType = args[1]; + targetIndexInfo = args[2]; + preserveDataFlag = args[3]; // Note - even if they pass in "false", History will preserve the data + } + + if (propName.equals("")) { + String emsg = "Bad parameter - propertyName cannot be empty. \n" + usageString; + logAndPrint(logger, emsg); + System.exit(1); + } else if (!targetDataType.equals("String") && !targetDataType.equals("Set<String>") + && !targetDataType.equals("Integer") && !targetDataType.equals("Long") + && !targetDataType.equals("Boolean")) { + String emsg = "Unsupported targetDataType. We only support String, Set<String>, Integer, Long or Boolean for now.\n" + + usageString; + logAndPrint(logger, emsg); + System.exit(1); + } else if (!targetIndexInfo.equals("index") && !targetIndexInfo.equals("noIndex")) { + String emsg = "Unsupported IndexInfo. We only support: 'index' or 'noIndex' for History db.\n" + + usageString; + logAndPrint(logger, emsg); + System.exit(1); + } + + try { + AAIConfig.init(); + ErrorLogHelper.loadProperties(); + } catch (Exception ae) { + String emsg = "Problem with either AAIConfig.init() or ErrorLogHelper.LoadProperties(). "; + logAndPrint(logger, emsg + "[" + ae.getMessage() + "]"); + System.exit(1); + } + + // Give a big warning if the DbMaps.PropertyDataTypeMap value does not + // agree with what we're doing + String warningMsg = ""; + + if (!warningMsg.equals("")) { + logAndPrint(logger, "\n>>> WARNING <<<< "); + logAndPrint(logger, ">>> " + warningMsg + " <<<"); + } + + logAndPrint(logger, ">>> Processing will begin in 5 seconds (unless interrupted). <<<"); + try { + // Give them a chance to back out of this + Thread.sleep(5000); + } catch (java.lang.InterruptedException ie) { + logAndPrint(logger, " DB Schema Update has been aborted. "); + System.exit(1); + } + + logAndPrint(logger, " ---- NOTE --- about to open graph (takes a little while)\n"); + + SchemaVersion version = schemaVersions.getDefaultVersion(); + QueryStyle queryStyle = QueryStyle.TRAVERSAL; + ModelType introspectorFactoryType = ModelType.MOXY; + Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); + TransactionalGraphEngine engine = null; + try { + engine = new JanusGraphDBEngine(queryStyle, loader); + //NOTE - no matter what they passed in, we're passing in "true" for the + // preserve-data-flag last parameter since this is for HISTORY. + SchemaModInternal4Hist internal = new SchemaModInternal4Hist(engine, logger, propName, targetDataType, targetIndexInfo, true); + internal.execute(); + engine.startTransaction(); + engine.tx().close(); + logAndPrint(logger, "------ Completed the SchemaMod -------- "); + } catch (Exception e) { + String emsg = "Not able to complete the requested SchemaMod4Hist \n"; + logAndPrint(logger, e.getMessage()); + logAndPrint(logger, emsg); + System.exit(1); + } + } + /** + * Log and print. + * + * @param logger the logger + * @param msg the msg + */ + protected void logAndPrint(Logger logger, String msg) { + System.out.println(msg); + logger.debug(msg); + } + + public static void main(String[] args) throws AAIException { + + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); + PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); + initializer.initialize(ctx); + try { + ctx.scan( + "org.onap.aai.config", + "org.onap.aai.setup" + ); + ctx.refresh(); + } catch (Exception e) { + AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); + System.out.println("Problems running SchemaMod4Hist "+aai.getMessage()); + ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); + throw aai; + } + + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); + if( !historyEnabled ) { + String emsg = "SchemaMod4Hist may only be used when history.enabled=true. "; + System.out.println(emsg); + throw new AAIException("AAI-4005",emsg); + } + + LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class); + SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions"); + SchemaMod4Hist schemaMod4H = new SchemaMod4Hist(loaderFactory, schemaVersions); + schemaMod4H.execute(args); + + System.exit(0); + } + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java deleted file mode 100644 index b5ce16b..0000000 --- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java +++ /dev/null @@ -1,317 +0,0 @@ -/** - * ============LICENSE_START======================================================= - * org.onap.aai - * ================================================================================ - * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END========================================================= - */ -package org.onap.aai.dbgen.schemamod; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.UUID; - -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; -import org.apache.tinkerpop.gremlin.structure.Direction; -import org.apache.tinkerpop.gremlin.structure.Edge; -import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.apache.tinkerpop.gremlin.structure.VertexProperty; -import org.onap.aai.serialization.engines.TransactionalGraphEngine; -import org.onap.aai.util.FormatDate; -import org.onap.aai.util.UniquePropertyCheck; - -import com.att.eelf.configuration.EELFLogger; -import org.janusgraph.core.Cardinality; -import org.janusgraph.core.PropertyKey; -import org.janusgraph.core.schema.JanusGraphManagement; - -public class SchemaModInternal { - private static final String FROMAPPID = "AAI-UTILS"; - private final String TRANSID = UUID.randomUUID().toString(); - private final TransactionalGraphEngine engine; - private final String propName; - private final Class<?> type; - private final String indexType; - private final boolean preserveData; - private final Cardinality cardinality; - private final EELFLogger logger; - - public SchemaModInternal(TransactionalGraphEngine engine, EELFLogger logger, String propName, String type, String indexType, boolean preserveData) { - this.engine = engine; - this.propName = propName; - this.type = determineClass(type); - this.indexType = indexType; - this.preserveData = preserveData; - this.cardinality = determineCardinality(type); - this.logger = logger; - } - - - private Class<?> determineClass(String type) { - final Class<?> result; - if (type.equals("String")) { - result = String.class; - } else if (type.equals("Set<String>")) { - result = String.class; - } else if (type.equals("Integer")) { - result = Integer.class; - } else if (type.equals("Boolean")) { - result = Boolean.class; - } else if (type.equals("Character")) { - result = Character.class; - } else if (type.equals("Long")) { - result = Long.class; - } else if (type.equals("Float")) { - result = Float.class; - } else if (type.equals("Double")) { - result = Double.class; - } else { - String emsg = "Not able translate the targetDataType [" + type + "] to a Class variable.\n"; - logAndPrint(logger, emsg); - throw new RuntimeException(emsg); - } - - return result; - } - private Cardinality determineCardinality(String type) { - if (type.equals("Set<String>")) { - return Cardinality.SET; - } else { - return Cardinality.SINGLE; - } - } - public void execute() { - JanusGraphManagement graphMgt = null; - boolean success = false; - try { - // Make sure this property is in the DB. - graphMgt = engine.asAdmin().getManagementSystem(); - if (graphMgt == null) { - String emsg = "Not able to get a graph Management object in SchemaMod.java\n"; - logAndPrint(logger, emsg); - System.exit(1); - } - PropertyKey origPropKey = graphMgt.getPropertyKey(propName); - if (origPropKey == null) { - String emsg = "The propName = [" + propName + "] is not defined in our graph. "; - logAndPrint(logger, emsg); - System.exit(1); - } - - if (indexType.equals("uniqueIndex")) { - // Make sure the data in the property being changed can have a - // unique-index put on it. - // Ie. if there are duplicate values, we will not be able to - // migrate the data back into the property. - - - Graph grTmp = engine.tx(); - if( grTmp == null ){ - grTmp = engine.startTransaction(); - } - // This is good to know in the logs - logAndPrint(logger, "-- Starting UniquePropertyCheck. (this may take a loooong time) --"); - - Boolean foundDupesFlag = UniquePropertyCheck.runTheCheckForUniqueness(TRANSID, FROMAPPID, - grTmp, propName, logger); - if (foundDupesFlag) { - logAndPrint(logger, - "\n\n!!!!!! >> Cannot add a uniqueIndex for the property: [" + propName - + "] because duplicate values were found. See the log for details on which" - + " nodes have this value. \nThey will need to be resolved (by updating those values to new" - + " values or deleting unneeded nodes) using the standard REST-API \n"); - System.exit(1); - } - logAndPrint(logger, "-- Finished UniquePropertyCheck. "); // This is good to know in the logs - } - - - // ---- If we made it to here - we must be OK with making this change - - // Rename this property to a backup name (old name with "retired_" - // appended plus a dateStr) - FormatDate fd = new FormatDate("MMddHHmm", "GMT"); - String dteStr= fd.getDateTime(); - - String retiredName = propName + "-" + dteStr + "-RETIRED"; - graphMgt.changeName(origPropKey, retiredName); - - // Create a new property using the original property name and the - // targetDataType - PropertyKey freshPropKey = graphMgt.makePropertyKey(propName).dataType(type) - .cardinality(cardinality).make(); - - // Create the appropriate index (if any) - if (indexType.equals("uniqueIndex")) { - String freshIndexName = propName + dteStr; - graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).unique().buildCompositeIndex(); - } else if (indexType.equals("index")) { - String freshIndexName = propName + dteStr; - graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).buildCompositeIndex(); - } - - logAndPrint(logger, "Committing schema changes with graphMgt.commit()"); - graphMgt.commit(); - engine.commit(); - Graph grTmp2 = engine.startTransaction(); - - - // For each node that has this property, update the new from the old - // and then remove the - // old property from that node - Iterator<Vertex> verts = grTmp2.traversal().V().has(retiredName); - int vtxCount = 0; - ArrayList<String> alreadySeenVals = new ArrayList<String>(); - while (verts.hasNext()) { - vtxCount++; - Vertex tmpVtx = verts.next(); - String tmpVid = tmpVtx.id().toString(); - Object origVal = tmpVtx.<Object> property(retiredName).orElse(null); - if (preserveData) { - tmpVtx.property(propName, origVal); - if (indexType.equals("uniqueIndex")) { - // We're working on a property that is being used as a - // unique index - String origValStr = ""; - if (origVal != null) { - origValStr = origVal.toString(); - } - if (alreadySeenVals.contains(origValStr)) { - // This property is supposed to be unique, but we've - // already seen this value in this loop - // This should have been caught up in the first part - // of SchemaMod, but since it wasn't, we - // will just log the problem. - logAndPrint(logger, - "\n\n ---------- ERROR - could not migrate the old data [" + origValStr - + "] for propertyName [" + propName - + "] because this property is having a unique index put on it."); - showPropertiesAndEdges(TRANSID, FROMAPPID, tmpVtx, logger); - logAndPrint(logger, "-----------------------------------\n"); - } else { - // Ok to add this prop in as a unique value - tmpVtx.property(propName, origVal); - logAndPrint(logger, - "INFO -- just did the add of the freshPropertyKey and updated it with the orig value (" - + origValStr + ")"); - } - alreadySeenVals.add(origValStr); - } else { - // We are not working with a unique index - tmpVtx.property(propName, origVal); - logAndPrint(logger, - "INFO -- just did the add of the freshPropertyKey and updated it with the orig value (" - + origVal.toString() + ")"); - } - } else { - // existing nodes just won't have that property anymore - // Not sure if we'd ever actually want to do this -- maybe - // we'd do this if the new - // data type was not compatible with the old? - } - tmpVtx.property(retiredName).remove(); - logAndPrint(logger, "INFO -- just did the remove of the " + retiredName + " from this vertex. (vid=" - + tmpVid + ")"); - } - - success = true; - } catch (Exception ex) { - logAndPrint(logger, "Threw a regular Exception: "); - logAndPrint(logger, ex.getMessage()); - } finally { - if (graphMgt != null && graphMgt.isOpen()) { - // Any changes that worked correctly should have already done - // their commits. - graphMgt.rollback(); - } - if (engine != null) { - if (success) { - engine.commit(); - } else { - engine.rollback(); - } - } - } - } - - /** - * Show properties and edges. - * - * @param transId the trans id - * @param fromAppId the from app id - * @param tVert the t vert - * @param logger the logger - */ - private static void showPropertiesAndEdges(String transId, String fromAppId, Vertex tVert, EELFLogger logger) { - - if (tVert == null) { - logAndPrint(logger, "Null node passed to showPropertiesAndEdges."); - } else { - String nodeType = ""; - Object ob = tVert.<String> property("aai-node-type"); - if (ob == null) { - nodeType = "null"; - } else { - nodeType = ob.toString(); - } - - logAndPrint(logger, " AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]"); - logAndPrint(logger, " Property Detail: "); - Iterator<VertexProperty<Object>> pI = tVert.properties(); - while (pI.hasNext()) { - VertexProperty<Object> tp = pI.next(); - Object val = tp.value(); - logAndPrint(logger, "Prop: [" + tp.key() + "], val = [" + val + "] "); - } - - Iterator<Edge> eI = tVert.edges(Direction.BOTH); - if (!eI.hasNext()) { - logAndPrint(logger, "No edges were found for this vertex. "); - } - while (eI.hasNext()) { - Edge ed = eI.next(); - String lab = ed.label(); - Vertex vtx; - if (tVert.equals(ed.inVertex())) { - vtx = ed.outVertex(); - } else { - vtx = ed.inVertex(); - } - if (vtx == null) { - logAndPrint(logger, - " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< "); - } else { - String nType = vtx.<String> property("aai-node-type").orElse(null); - String vid = vtx.id().toString(); - logAndPrint(logger, "Found an edge (" + lab + ") from this vertex to a [" + nType - + "] node with VtxId = " + vid); - } - } - } - } // End of showPropertiesAndEdges() - - /** - * Log and print. - * - * @param logger the logger - * @param msg the msg - */ - protected static void logAndPrint(EELFLogger logger, String msg) { - System.out.println(msg); - logger.info(msg); - } - -} diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java new file mode 100644 index 0000000..f656b9f --- /dev/null +++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java @@ -0,0 +1,181 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.dbgen.schemamod; + +import java.util.Iterator; +import java.util.UUID; + +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.util.FormatDate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.janusgraph.core.Cardinality; +import org.janusgraph.core.PropertyKey; +import org.janusgraph.core.schema.JanusGraphManagement; + +public class SchemaModInternal4Hist { + private final TransactionalGraphEngine engine; + private final String propName; + private final Class<?> type; + private final String indexType; + private final boolean preserveData; + private final Cardinality cardinality; + private final Logger logger; + + public SchemaModInternal4Hist(TransactionalGraphEngine engine, Logger logger, String propName, String type, String indexType, boolean preserveData) { + this.engine = engine; + this.propName = propName; + this.type = determineClass(type); + this.indexType = indexType; + this.preserveData = preserveData; + this.cardinality = Cardinality.LIST; // Always use this for History + this.logger = logger; + } + + private Class<?> determineClass(String type) { + final Class<?> result; + if (type.equals("String")) { + result = String.class; + } else if (type.equals("Set<String>")) { + result = String.class; + } else if (type.equals("Integer")) { + result = Integer.class; + } else if (type.equals("Boolean")) { + result = Boolean.class; + } else if (type.equals("Character")) { + result = Character.class; + } else if (type.equals("Long")) { + result = Long.class; + } else if (type.equals("Float")) { + result = Float.class; + } else if (type.equals("Double")) { + result = Double.class; + } else { + String emsg = "Not able translate the targetDataType [" + type + "] to a Class variable.\n"; + logAndPrint(logger, emsg); + throw new RuntimeException(emsg); + } + + return result; + } + + public void execute() { + JanusGraphManagement graphMgt = null; + boolean success = false; + try { + // Make sure this property is in the DB. + graphMgt = engine.asAdmin().getManagementSystem(); + if (graphMgt == null) { + String emsg = "Not able to get a graph Management object in SchemaModInternal4Hist.java\n"; + logAndPrint(logger, emsg); + System.exit(1); + } + PropertyKey origPropKey = graphMgt.getPropertyKey(propName); + if (origPropKey == null) { + String emsg = "The propName = [" + propName + "] is not defined in our graph. "; + logAndPrint(logger, emsg); + System.exit(1); + } + + // Rename this property to a backup name (old name with "retired_" + // appended plus a dateStr) + FormatDate fd = new FormatDate("MMddHHmm", "GMT"); + String dteStr= fd.getDateTime(); + + String retiredName = propName + "-" + dteStr + "-RETIRED"; + graphMgt.changeName(origPropKey, retiredName); + + // Create a new property using the original property name and the + // targetDataType + PropertyKey freshPropKey = graphMgt.makePropertyKey(propName).dataType(type) + .cardinality(cardinality).make(); + + // Create an index if needed (regular index will be used instead of unique for history) + if (indexType.equals("index") || indexType.equals("uniqueIndex")) { + String freshIndexName = propName + dteStr; + graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).buildCompositeIndex(); + } + + logAndPrint(logger, "Committing schema changes with graphMgt.commit()"); + graphMgt.commit(); + engine.commit(); + Graph grTmp2 = engine.startTransaction(); + + + // For each node that has this property, update the new from the old + // and then remove the + // old property from that node + Iterator<Vertex> verts = grTmp2.traversal().V().has(retiredName); + int vtxCount = 0; + while (verts.hasNext()) { + vtxCount++; + Vertex tmpVtx = verts.next(); + String tmpVid = tmpVtx.id().toString(); + Object origVal = tmpVtx.<Object> property(retiredName).orElse(null); + if (preserveData) { + tmpVtx.property(propName, origVal); + logAndPrint(logger, + "INFO -- just did the add of the freshPropertyKey and updated it with the orig value (" + + origVal.toString() + ")"); + } else { + // existing nodes just won't have that property anymore + // Not sure if we'd ever actually want to do this -- maybe + // we'd do this if the new + // data type was not compatible with the old? + } + tmpVtx.property(retiredName).remove(); + logAndPrint(logger, "INFO -- just did the remove of the " + retiredName + " from this vertex. (vid=" + + tmpVid + ")"); + } + + success = true; + } catch (Exception ex) { + logAndPrint(logger, "Threw a regular Exception: "); + logAndPrint(logger, ex.getMessage()); + } finally { + if (graphMgt != null && graphMgt.isOpen()) { + // Any changes that worked correctly should have already done + // their commits. + graphMgt.rollback(); + } + if (engine != null) { + if (success) { + engine.commit(); + } else { + engine.rollback(); + } + } + } + } + + /** + * Log and print. + * + * @param logger the logger + * @param msg the msg + */ + protected static void logAndPrint(Logger logger, String msg) { + System.out.println(msg); + logger.debug(msg); + } + +} diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternalBatch.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternalBatch.java new file mode 100644 index 0000000..e88e2bf --- /dev/null +++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternalBatch.java @@ -0,0 +1,451 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.dbgen.schemamod; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.util.FormatDate; + +import org.slf4j.Logger; +import org.janusgraph.core.Cardinality; +import org.janusgraph.core.PropertyKey; +import org.janusgraph.core.schema.JanusGraphManagement; + +public class SchemaModInternalBatch { + private static final String FROMAPPID = "AAI-UTILS"; + private final String TRANSID = UUID.randomUUID().toString(); + private final TransactionalGraphEngine engine; + private final String propName; + private final Class<?> type; + private final String indexType; + private final boolean preserveData; + private final Cardinality cardinality; + private final long commitBlockSize; + private final Logger logger; + + public SchemaModInternalBatch(TransactionalGraphEngine engine, Logger logger, String propName, + String type, String indexType, boolean preserveData, long commitBlockSize) { + this.engine = engine; + this.propName = propName; + this.type = determineClass(type); + this.indexType = indexType; + this.preserveData = preserveData; + this.cardinality = determineCardinality(type); + this.commitBlockSize = commitBlockSize; + this.logger = logger; + } + + + private Class<?> determineClass(String type) { + final Class<?> result; + if (type.equals("String")) { + result = String.class; + } else if (type.equals("Set<String>")) { + result = String.class; + } else if (type.equals("Integer")) { + result = Integer.class; + } else if (type.equals("Boolean")) { + result = Boolean.class; + } else if (type.equals("Character")) { + result = Character.class; + } else if (type.equals("Long")) { + result = Long.class; + } else if (type.equals("Float")) { + result = Float.class; + } else if (type.equals("Double")) { + result = Double.class; + } else { + String emsg = "Not able translate the targetDataType [" + type + "] to a Class variable.\n"; + logAndPrint(logger, emsg); + throw new RuntimeException(emsg); + } + + return result; + } + + private Cardinality determineCardinality(String type) { + if (type.equals("Set<String>")) { + return Cardinality.SET; + } else { + return Cardinality.SINGLE; + } + } + + public void execute() { + JanusGraphManagement graphMgt = null; + String retiredName = ""; + boolean success = false; + long timeStart = System.nanoTime(); + int batchCt = 0; + int totalCount = 0; + + ArrayList<HashMap<String,Object>> allVerts = new ArrayList<HashMap<String,Object>>(); + HashMap<String,Object> batchVHash = new HashMap<String,Object>(); + + try { + // Make sure this property is in the DB. + graphMgt = engine.asAdmin().getManagementSystem(); + if (graphMgt == null) { + String emsg = "Not able to get a graph Management object in SchemaMod.java\n"; + logAndPrint(logger, emsg); + System.exit(1); + } + PropertyKey origPropKey = graphMgt.getPropertyKey(propName); + if (origPropKey == null) { + String emsg = "The propName = [" + propName + "] is not defined in our graph. "; + logAndPrint(logger, emsg); + System.exit(1); + } + + // Collect the data that needs to be processed and + // store as hashes of vertex-id's and the original property value + long timeA = System.nanoTime(); + int msgEveryXCount = 1000; + Graph grTmp1 = engine.startTransaction(); + Iterator<Vertex> allVtxItr = grTmp1.traversal().V().has(propName); + // Will hold these in lists that are no bigger than our + // allowed commitBatch size. + logAndPrint(logger, "Collecting the data (this takes a little while).. "); + int batchKey = 0; + int batchVCount = 0; + totalCount = 0; + int msgCount = 0; + logAndPrint(logger, "Collecting the data for batch # " + batchKey ); + Object origVal = null; + while (allVtxItr.hasNext()) { + Vertex v = allVtxItr.next(); + origVal = v.<Object>property(propName).orElse(null); + batchVHash.put(v.id().toString(), origVal); + batchVCount++; + totalCount++; + msgCount++; + if (batchVCount >= commitBlockSize ) { + // This was the last one for this batch + allVerts.add(batchKey, batchVHash); + batchKey++; + logAndPrint(logger, "Collecting the data for batch # " + batchKey ); + batchVCount = 0; + batchVHash = new HashMap<String,Object>(); + } + if( msgCount > msgEveryXCount ) { + msgCount = 0; + logAndPrint(logger, " Initial processing running... total so far = " + totalCount ); + } + } + + if( batchVCount > 0 ) { + // Add the last partial set if there is one. + allVerts.add(batchKey, batchVHash); + } + logAndPrint(logger, "Found " + totalCount + " nodes that will be affected. "); + + batchCt = batchKey +1; + + if( totalCount == 0 ) { + logAndPrint(logger, "INFO -- No data found to process. "); + System.exit(1); + } + + logAndPrint(logger, "INFO -- Total of " + totalCount + + " nodes to process. Will use " + batchCt + + " batches. " ); + + long timeB = System.nanoTime(); + long diffTime = timeB - timeA; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + logAndPrint(logger, " -- To collect all nodes took: " + + minCount + " minutes, " + secCount + " seconds " ); + + if (indexType.equals("uniqueIndex")) { + // Make sure the data in the property being changed can have a + // unique-index put on it. + // Ie. if there are duplicate values, we will not be able to + // migrate the data back into the property. + Boolean foundDupesFlag = false; + try { + foundDupesFlag = doUniquenessCheck(allVerts, propName); + } catch (Exception e) { + logAndPrint(logger, "ERROR thrown in doUniquenessCheck(): [" + + e.getMessage() + "]"); + System.exit(1); + } + if (foundDupesFlag) { + logAndPrint(logger, + "\n\n!!!!!! >> Cannot add a uniqueIndex for the property: [" + propName + + "] because duplicate values were found. See the log for details on which" + + " nodes have this value. \nThey will need to be resolved (by updating those values to" + + " new values or deleting unneeded nodes) using the standard REST-API \n"); + System.exit(1); + } + logAndPrint(logger, "-- Finished/Passed UniquePropertyCheck. "); + logAndPrint(logger, "There are " + totalCount + " nodes that have this property. "); + } + + // ---- If we made it to here - we must be OK with making this change + + // Rename this property to a backup name (old name with a dateString and + // "-RETIRED" appended) + long timeE = System.nanoTime(); + FormatDate fd = new FormatDate("MMddHHmm", "GMT"); + String dteStr= fd.getDateTime(); + retiredName = propName + "-" + dteStr + "-RETIRED"; + graphMgt.changeName(origPropKey, retiredName); + logAndPrint(logger, " -- Temporary property name will be: [" + retiredName + "]. "); + + // Create a new property using the original property name and the + // targetDataType + PropertyKey freshPropKey = graphMgt.makePropertyKey(propName).dataType(type) + .cardinality(cardinality).make(); + + // Create the appropriate index (if any) + if (indexType.equals("uniqueIndex")) { + String freshIndexName = propName + dteStr; + graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).unique().buildCompositeIndex(); + } else if (indexType.equals("index")) { + String freshIndexName = propName + dteStr; + graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).buildCompositeIndex(); + } + + logAndPrint(logger, "Committing schema changes with graphMgt.commit()"); + graphMgt.commit(); + success = true; + + long timeF = System.nanoTime(); + diffTime = timeF - timeE; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + logAndPrint(logger, " -- Temporary property Name Change took: " + + minCount + " minutes, " + secCount + " seconds " ); + + } catch (Exception ex) { + logAndPrint(logger, "Threw a regular Exception: "); + logAndPrint(logger, ex.getMessage()); + System.exit(1); + } finally { + if (graphMgt != null && graphMgt.isOpen()) { + // Any changes that worked correctly should have already done + // their commits. + graphMgt.rollback(); + } + if (engine != null) { + if (success) { + engine.commit(); + } else { + engine.rollback(); + } + } + } + + + // For each node that has this property, update the new from the old + // and then remove the old property from that node + // Note - do it in batches since there can be a LOT of updates. + + long timeE = System.nanoTime(); + ArrayList <String> emsgList = new ArrayList <String> (); + for( int batNo=0; batNo < batchCt; batNo++ ) { + try { + logAndPrint(logger, "BEGIN -- Batch # " + batNo ); + processUpdateForBatch( allVerts.get(batNo), retiredName ); + logAndPrint(logger, "Completed Batch # " + batNo ); + } catch (Exception e) { + String emsg = "ERROR -- Batch # " + batNo + + " failed to process. Please clean up manually. " + + " data in [" + retiredName + + "] will have to be moved to the original property."; + logAndPrint(logger, emsg); + emsgList.add(emsg); + } + } + long timeF = System.nanoTime(); + long diffTime = timeF - timeE; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + logAndPrint(logger, " -- Time to process all batches: " + + minCount + " minutes, " + secCount + " seconds " ); + + logAndPrint(logger, "\nINFO -- Total of " + totalCount + + " nodes processed using: " + batchCt + " batches. " ); + + if( !emsgList.isEmpty() ) { + Iterator <String> eItr = emsgList.iterator(); + logAndPrint(logger, ">>> These will need to be taken care of: "); + while( eItr.hasNext() ) { + logAndPrint(logger, (String)eItr.next()); + } + } + + long timeEnd = System.nanoTime(); + diffTime = timeEnd - timeStart; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + logAndPrint(logger, " -- Total Processing time was: " + + minCount + " minutes, " + secCount + " seconds " ); + + }// End of Execute() + + + private void processUpdateForBatch( HashMap<String,Object> vertHash, + String retiredName ) throws Exception { + + Iterator<Map.Entry<String, Object>> vertHashItr = vertHash.entrySet().iterator(); + int vtxCount = 0; + Boolean success = false; + Graph grTmpBat = engine.startTransaction(); + try { + while( vertHashItr.hasNext() ){ + Map.Entry<String, Object> entry = vertHashItr.next(); + String tmpVid = entry.getKey(); + Vertex tmpVtx = null; + + Iterator<Vertex> oneVItr = grTmpBat.traversal().V(tmpVid); + while( oneVItr.hasNext() ) { + // should never find more than one... + tmpVtx = oneVItr.next(); + Object origVal = entry.getValue(); + if (preserveData) { + tmpVtx.property(propName, origVal); + } else { + // existing nodes just won't have that property anymore + // Might want to do this if the new + // data type was not compatible with the old. + } + tmpVtx.property(retiredName).remove(); + logAndPrint(logger, "INFO -- update item: (vid= " + + tmpVid + ", val=[" + origVal + "])"); + vtxCount++; + } + } + + logAndPrint(logger, "INFO -- finished processing a batch with " + vtxCount + " nodes."); + success = true; + } catch (Exception ex) { + logAndPrint(logger, "Threw a regular Exception: "); + logAndPrint(logger, ex.getMessage()); + } finally { + if (engine != null) { + if (success) { + logAndPrint(logger, "INFO -- committing node updates for this batch."); + engine.commit(); + } else { + logAndPrint(logger, "ERROR -- rolling back node updates for this batch."); + engine.rollback(); + } + } + } + if( ! success ) { + throw new Exception ("ERROR - could not process this batch -- see the log for details."); + } + + }// end of processUpdateForBatch() + + + private Boolean doUniquenessCheck( ArrayList<HashMap<String,Object>> allVerts, + String propertyName ){ + // Note - property can be found in more than one nodetype + // our uniqueness constraints are always across the entire db - so this + // tool looks across all nodeTypes that the property is found in. + long timeStart = System.nanoTime(); + int batchCt = allVerts.size(); + HashMap <String,Object> bigSingleHash = new HashMap <String,Object> (); + + for( int batNo=0; batNo < batchCt; batNo++ ) { + bigSingleHash.putAll(allVerts.get(batNo)); + } + + ArrayList <Object> dupeValues = new ArrayList<Object> (); + int dupeCount = 0; + + Iterator bItr = bigSingleHash.entrySet().iterator(); + while( bItr.hasNext() ) { + Map.Entry pair = (Map.Entry)bItr.next(); + Object thisVal = pair.getValue(); + bItr.remove(); + if( bigSingleHash.containsValue(thisVal) ) { + // Found a dupe - because the value was still in the bigHash after + // we removed this pair from the bigHash + logAndPrint(logger, " Found a dupe node with val [" + thisVal + "]"); + if( dupeCount == 0 ) { + dupeValues.add(thisVal); + } + else if( !dupeValues.contains(thisVal) ){ + // Only record the first time we see it since we're just tracking + // the values, not the vids + dupeValues.add(thisVal); + } + dupeCount++; + } + } + + long timeEnd = System.nanoTime(); + long diffTime = timeEnd - timeStart; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + logAndPrint(logger, " -- Total Uniqueness Check took: " + + minCount + " minutes, " + secCount + " seconds " ); + + if( dupeValues.isEmpty() ){ + logAndPrint(logger, "\n ------------ No Duplicates Found -------- \n"); + } + else { + logAndPrint(logger, "\n -------------- Found " + dupeCount + + " cases of duplicate values for property [" + propertyName + "\n\n"); + logAndPrint(logger, "\n --- These values are in the db twice or more: "); + Iterator <?> dupeValItr = dupeValues.iterator(); + while( dupeValItr.hasNext() ){ + logAndPrint(logger, " value = [" + dupeValItr.next() + "]"); + } + } + + if( dupeCount > 0 ) { + return true; + }else { + return false; + } + + }// end of doUniquenessCheck() + + + + /** + * Log and print. + * + * @param logger the logger + * @param msg the msg + */ + protected static void logAndPrint(Logger logger, String msg) { + System.out.println(msg); + logger.debug(msg); + } + +} diff --git a/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java b/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java new file mode 100644 index 0000000..45b5d04 --- /dev/null +++ b/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java @@ -0,0 +1,518 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.historytruncate; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +import org.onap.aai.config.PropertyPasswordConfiguration; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Property; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.onap.aai.dbmap.AAIGraph; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; +import org.onap.aai.util.AAIConfig; +import org.onap.aai.util.AAIConstants; +import org.onap.aai.util.AAISystemExitUtil; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import com.att.eelf.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.janusgraph.core.JanusGraph; + +public class HistoryTruncate { + + private static Logger LOGGER = LoggerFactory.getLogger(HistoryTruncate.class); + + /* Using realtime d */ + private static final String REALTIME_DB = "realtime"; + + private static final String LOG_ONLY_MODE = "LOG_ONLY"; + private static final String DELETE_AND_LOG_MODE = "DELETE_AND_LOG"; + private static final String SILENT_DELETE_MODE = "SILENT_DELETE"; + static ArrayList <String> VALIDMODES = new <String> ArrayList (); + static { + VALIDMODES.add(LOG_ONLY_MODE); + VALIDMODES.add(DELETE_AND_LOG_MODE); + VALIDMODES.add(SILENT_DELETE_MODE); + } + + private static final int batchCommitSize = 500; + + private static boolean historyEnabled; + private static String defaultTruncateMode; + private static Integer defaultTruncateWindowDays; + + + /** + * The main method. + * + */ + public static void main(String[] args) { + + // Set the logging file properties to be used by EELFManager + System.setProperty("aai.service.name", HistoryTruncate.class.getSimpleName()); + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); + + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); + PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); + initializer.initialize(ctx); + try { + ctx.scan( + "org.onap.aai.config", + "org.onap.aai.setup" + ); + ctx.refresh(); + } catch (Exception e) { + LOGGER.error("Error - Could not initialize context beans for HistoryTruncate. "); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); + if( !historyEnabled ) { + String emsg = "Error - HistoryTruncate may only be used when history.enabled=true. "; + System.out.println(emsg); + LOGGER.error(emsg); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + defaultTruncateWindowDays = Integer.parseInt(ctx.getEnvironment().getProperty("history.truncate.window.days","999")); + defaultTruncateMode = ctx.getEnvironment().getProperty("history.truncate.mode",LOG_ONLY_MODE); + + HistoryTruncate histTrunc = new HistoryTruncate(); + boolean success = histTrunc.executeCommand(args); + if(success){ + AAISystemExitUtil.systemExitCloseAAIGraph(0); + } else { + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + }// End of main() + + + public boolean executeCommand(String[] args) { + boolean successStatus = true; + // If they passed in args on the command line, then we should + // use those in place of the default ones we got from environment variables. + // "-truncateMode","LOG_ONLY","-truncateWindow","999" + String truncateMode = defaultTruncateMode; + int truncateWindowDays = defaultTruncateWindowDays; + + if (args != null && args.length > 0) { + // They passed some arguments in that will affect processing + for (int i = 0; i < args.length; i++) { + String thisArg = args[i]; + if (thisArg.equals("-truncateMode")) { + i++; + if (i >= args.length) { + LOGGER.error(" No value passed with -truncateMode option. "); + return false; + } + if( !VALIDMODES.contains(args[i]) ) { + LOGGER.error(" Unrecognized -truncateMode value passed: [" + + args[i] + "]. Valid values = " + VALIDMODES.toString() ); + return false; + } + truncateMode = args[i]; + } else if (thisArg.equals("-truncateWindowDays")) { + i++; + if (i >= args.length) { + LOGGER.error("No value passed with -truncateWindowDays option."); + return false; + } + String nextArg = args[i]; + try { + truncateWindowDays = Integer.parseInt(nextArg); + } catch (Exception e) { + LOGGER.error("Bad value passed with -truncateWindowDays option: [" + + nextArg + "]"); + return false; + } + } else { + LOGGER.error(" Unrecognized argument passed to HistoryTruncate: [" + + thisArg + "]. "); + LOGGER.error(" Valid values are: -truncateMode -truncateWindowDays "); + return false; + } + } + } + + LOGGER.debug(" Running HistoryTruncate with: truncateMode = " + truncateMode + + ", truncateWindowDays = " + truncateWindowDays ); + + Long truncateEndTs = calculateTruncWindowEndTimeStamp(truncateWindowDays); + JanusGraph jgraph = null; + long scriptStartTime = System.currentTimeMillis(); + Boolean doLogging = doLoggingOrNot( truncateMode ); + Boolean doDelete = doDeleteOrNot( truncateMode ); + + try { + AAIConfig.init(); + ErrorLogHelper.loadProperties(); + + LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while) "); + verifyGraph(AAIGraph.getInstance().getGraph()); + jgraph = AAIGraph.getInstance().getGraph(); + LOGGER.debug(" ---- got the new graph instance. "); + + // Note - process edges first so they get logged as they are deleted since + // edges connected to vertices being deleted would get auto-deleted by the db. + long timeA = System.nanoTime(); + processEdges(jgraph, truncateEndTs, doLogging, doDelete); + long timeB = System.nanoTime(); + long diffTime = timeB - timeA; + long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" Took this long to process the Edges: " + + minCount + " minutes, " + secCount + " seconds " ); + + processVerts(jgraph, truncateEndTs, doLogging, doDelete); + long timeC = System.nanoTime(); + diffTime = timeC - timeB; + minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime); + secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount); + LOGGER.debug(" Took this long to process the Vertices: " + + minCount + " minutes, " + secCount + " seconds " ); + + } catch (AAIException e) { + ErrorLogHelper.logError("AAI_6128", e.getMessage()); + LOGGER.error("Encountered an exception during the historyTruncate: ", e); + e.printStackTrace(); + successStatus = false; + } catch (Exception ex) { + ErrorLogHelper.logError("AAI_6128", ex.getMessage()); + LOGGER.error("Encountered an exception during the historyTruncate: ", ex); + ex.printStackTrace(); + successStatus = false; + } finally { + if (jgraph != null ) { + // Any changes that worked correctly should have already done + // their commits. + if(!"true".equals(System.getProperty("org.onap.aai.graphadmin.started"))) { + if (jgraph.isOpen()) { + jgraph.tx().rollback(); + jgraph.close(); + } + } + } + } + + return successStatus; + } + + + public void processVerts(JanusGraph jgraph, Long truncBeforeTs, + Boolean doLogging, Boolean doDelete ) { + + Graph g = jgraph.newTransaction(); + GraphTraversalSource gts = g.traversal(); + //Iterator <Vertex> vertItr = gts.V().has(AAIProperties.END_TS, P.lt(truncBeforeTs)); + Iterator <Vertex> vertItr = gts.V().has("end-ts", P.lt(truncBeforeTs)); + ArrayList <Long> vidList = new ArrayList <Long> (); + while( vertItr.hasNext() ) { + Vertex tmpV = vertItr.next(); + Long tmpVid = Long.valueOf(tmpV.id().toString()); + vidList.add(tmpVid); + } + + int vTotalCount = vidList.size(); + int batchCount = vTotalCount / batchCommitSize; + if((batchCount * batchCommitSize) < vTotalCount){ + batchCount++; + } + + LOGGER.info( " Vertex TotalCount = " + vTotalCount + + ", we get batchCount = " + batchCount + + ", using commit size = " + batchCommitSize ); + + int vIndex = 0; + for(int batchNo=1; batchNo<=batchCount; batchNo++){ + ArrayList <Long> batchVids = new ArrayList <Long> (); + int thisBVCount = 0; + while( (thisBVCount < batchCommitSize) && (vIndex < vTotalCount) ) { + batchVids.add(vidList.get(vIndex)); + thisBVCount++; + vIndex++; + } + // now process this batch + LOGGER.info( "Process vertex batch # " + batchNo + + ", which contains " + batchVids.size() + " ids. "); + processVertBatch(jgraph, doLogging, doDelete, batchVids); + } + } + + + private void processVertBatch(JanusGraph jgraph, Boolean doLogging, + Boolean doDelete, ArrayList <Long> vidList ) { + + Graph g = jgraph.newTransaction(); + GraphTraversalSource gts = g.traversal(); + int delFailCount = 0; + int vCount = 0; + int delCount = 0; + + Iterator <Vertex> vertItr = gts.V(vidList); + while( vertItr.hasNext() ) { + vCount++; + Vertex tmpV = vertItr.next(); + String tmpVid = tmpV.id().toString(); + String tmpPropsStr = ""; + if( doLogging ) { + Iterator<VertexProperty<Object>> pI = tmpV.properties(); + while( pI.hasNext() ){ + VertexProperty<Object> tp = pI.next(); + Object val = tp.value(); + tmpPropsStr = tmpPropsStr + "[" + tp.key() + "=" + val + "]"; + } + LOGGER.info(" vid = " + tmpVid + ", props: (" + tmpPropsStr + ") " ); + } + + if( doDelete ) { + LOGGER.info("Removing vid = " + tmpVid ); + try { + tmpV.remove(); + delCount++; + } catch ( Exception e ) { + // figure out what to do + delFailCount++; + LOGGER.error("ERROR trying to delete Candidate VID = " + tmpVid + " " + LogFormatTools.getStackTop(e)); + } + } + } + + if( doDelete ) { + LOGGER.info("Calling commit on delete of Vertices." ); + try { + g.tx().commit(); + } catch ( Exception e ) { + LOGGER.error("ERROR trying to commit Vertex Deletes for this batch. " + + LogFormatTools.getStackTop(e) ); + LOGGER.info( vCount + " candidate vertices processed. " + + " vertex deletes - COMMIT FAILED. "); + return; + } + } + + if( doDelete ) { + LOGGER.info( vCount + " candidate vertices processed. " + + delFailCount + " delete attempts failed, " + + delCount + " deletes successful. "); + } + else { + LOGGER.info( vCount + " candidate vertices processed in this batch. " ); + } + } + + + public void processEdges(JanusGraph jgraph, Long truncBeforeTs, + Boolean doLogging, Boolean doDelete ) { + + Graph g = jgraph.newTransaction(); + GraphTraversalSource gts = g.traversal(); + //Iterator <Edge> edgeItr = gts.E().has(AAIProperties.END_TS, P.lt(truncBeforeTs)); + Iterator <Edge> edgeItr = gts.E().has("end-ts", P.lt(truncBeforeTs)); + ArrayList <String> eidList = new ArrayList <String> (); + while( edgeItr.hasNext() ) { + Edge tmpE = edgeItr.next(); + String tmpEid = tmpE.id().toString(); + eidList.add(tmpEid); + } + + int eTotalCount = eidList.size(); + int batchCount = eTotalCount / batchCommitSize; + if((batchCount * batchCommitSize) < eTotalCount){ + batchCount++; + } + + LOGGER.info( " Edge TotalCount = " + eTotalCount + + ", we get batchCount = " + batchCount + + ", using commit size = " + batchCommitSize ); + + int eIndex = 0; + for(int batchNo=1; batchNo<=batchCount; batchNo++){ + ArrayList <String> batchEids = new ArrayList <String> (); + int thisBECount = 0; + while( (thisBECount < batchCommitSize) && (eIndex < eTotalCount) ) { + batchEids.add(eidList.get(eIndex)); + thisBECount++; + eIndex++; + } + // now process this batch + LOGGER.info( "Process edge batch # " + batchNo + + ", which contains " + batchEids.size() + " ids. "); + processEdgeBatch(jgraph, doLogging, doDelete, batchEids); + } + } + + + private void processEdgeBatch(JanusGraph jgraph, Boolean doLogging, + Boolean doDelete, ArrayList <String> eidList ) { + + Graph g = jgraph.newTransaction(); + GraphTraversalSource gts = g.traversal(); + int delFailCount = 0; + int eCount = 0; + int delCount = 0; + + Iterator <Edge> edgeItr = gts.E(eidList); + while( edgeItr.hasNext() ) { + eCount++; + Edge tmpE = edgeItr.next(); + String tmpEid = tmpE.id().toString(); + if( doLogging ) { + String tmpEProps = ""; + Iterator<Property<Object>> epI = tmpE.properties(); + while( epI.hasNext() ){ + Property<Object> ep = epI.next(); + Object val = ep.value(); + tmpEProps = tmpEProps + "[" + ep.key() + "=" + val + "]"; + } + Iterator <Vertex> conVtxs = tmpE.bothVertices(); + String tmpConVs = ""; + while( conVtxs.hasNext() ) { + Vertex conV = conVtxs.next(); + tmpConVs = tmpConVs + "[" + conV.id().toString() + "] "; + } + LOGGER.info(" eid = " + tmpEid + + ", Connecting vids = " + tmpConVs + + ", props: (" + tmpEProps + "). " ); + } + + if( doDelete ) { + LOGGER.info("Removing Edge eid = " + tmpEid ); + try { + tmpE.remove(); + delCount++; + } catch ( Exception e ) { + delFailCount++; + LOGGER.error("ERROR trying to delete Candidate Edge with eid = " + tmpEid + " " + LogFormatTools.getStackTop(e)); + } + } + } + + if( doDelete ) { + LOGGER.info("Calling commit on delete of Edges." ); + try { + g.tx().commit(); + } catch ( Exception e ) { + LOGGER.error("ERROR trying to commit Edge Deletes for this batch. " + + LogFormatTools.getStackTop(e) ); + LOGGER.info( eCount + " candidate edges processed. " + + " edge deletes - COMMIT FAILED. "); + return; + } + } + + if( doDelete ) { + LOGGER.info( eCount + " candidate edges processed. " + + delFailCount + " delete attempts failed, " + + delCount + " deletes successful. "); + } + else { + LOGGER.info( eCount + " candidate edges processed in this batch. " ); + } + } + + + public int getCandidateVertexCount(JanusGraph jgraph, int windowDaysVal) { + Graph g = jgraph.newTransaction(); + GraphTraversalSource gts = g.traversal(); + Long truncTs = calculateTruncWindowEndTimeStamp(windowDaysVal); + //int candVCount = gts.V().has(AAIProperties.END_TS, P.lt(truncTs)).count().next().intValue(); + int candVCount = gts.V().has("end-ts", P.lt(truncTs)).count().next().intValue(); + LOGGER.info( " for the timeStamp = " + truncTs + + ", which corresponds to the passed truncateWindowDays = " + + windowDaysVal + + ", found " + candVCount + + " candidate vertices. "); + return candVCount; + } + + + public int getCandidateEdgeCount(JanusGraph jgraph, int windowDaysVal) { + Graph g = jgraph.newTransaction(); + GraphTraversalSource gts = g.traversal(); + Long truncTs = calculateTruncWindowEndTimeStamp(windowDaysVal); + //int candECount = gts.E().has(AAIProperties.END_TS, P.lt(truncTs)).count().next().intValue(); + int candECount = gts.E().has("end-ts", P.lt(truncTs)).count().next().intValue(); + LOGGER.info( " for the timeStamp = " + truncTs + + ", which corresponds to the passed truncateWindowDays = " + + windowDaysVal + + ", found " + candECount + + " candidate Edges. "); + return candECount; + } + + + public static void verifyGraph(JanusGraph graph) { + + if (graph == null) { + String emsg = "Not able to get a graph object in DataSnapshot.java\n"; + LOGGER.debug(emsg); + AAISystemExitUtil.systemExitCloseAAIGraph(1); + } + + } + + public long calculateTruncWindowEndTimeStamp( int timeWindowDays ){ + // Given a window size in days, calculate the timestamp that + // represents the early-edge of that window. + + long unixTimeNow = System.currentTimeMillis(); + if( timeWindowDays <= 0 ){ + // This just means that they want to truncate all the way up to the current time + return unixTimeNow; + } + + long windowInMillis = timeWindowDays * 24 * 60 * 60L * 1000; + long windowEdgeTimeStampInMs = unixTimeNow - windowInMillis; + return windowEdgeTimeStampInMs; + + } + + private Boolean doLoggingOrNot( String truncMode ){ + if( truncMode.equals(SILENT_DELETE_MODE) ){ + return false; + } + else { + return true; + } + } + + private Boolean doDeleteOrNot( String truncMode ){ + if( truncMode.equals(LOG_ONLY_MODE) ){ + return false; + } + else { + return true; + } + } + + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/historytruncate/HistoryTruncateTasks.java b/src/main/java/org/onap/aai/historytruncate/HistoryTruncateTasks.java new file mode 100644 index 0000000..ec6fac3 --- /dev/null +++ b/src/main/java/org/onap/aai/historytruncate/HistoryTruncateTasks.java @@ -0,0 +1,108 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.historytruncate; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.text.SimpleDateFormat; +import java.util.*; + +import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.logging.LogFormatTools; +import org.onap.aai.util.AAIConfig; +import org.onap.logging.filter.base.ONAPComponents; +import org.onap.logging.ref.slf4j.ONAPLogConstants; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +@Component +@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties") +public class HistoryTruncateTasks { + + @Autowired + private AaiScheduledTaskAuditLog auditLog; + + private static final Logger LOGGER = LoggerFactory.getLogger(HistoryTruncateTasks.class); + private final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); + + @Scheduled(cron = "${historytruncatetasks.cron}" ) + public void historyTruncateScheduleTask() throws AAIException, Exception { + + if(!"true".equals(AAIConfig.get("aai.disable.check.historytruncate.running", "false"))){ + if(checkIfHistoryTruncateIsRunning()){ + LOGGER.debug("History Truncate is already running on the system"); + return; + } + } + + auditLog.logBefore("historyTruncateTask", ONAPComponents.AAI.toString() ); + LOGGER.debug("Started cron job HistoryTruncate @ " + dateFormat.format(new Date())); + try { + if (AAIConfig.get("aai.cron.enable.historytruncate").equals("true")) { + // Until we're comfortable with how it is working, we will keep it in "LOG_ONLY" mode + String defaultTruncMode = "LOG_ONLY"; + String defaultTruncWindowDays = "999"; + String [] params = {"-truncateMode",defaultTruncMode,"-truncateWindowDays",defaultTruncWindowDays}; + HistoryTruncate.main(params); + } + } + catch (Exception e) { + ErrorLogHelper.logError("AAI_4000", "Exception running cron job for HistoryTruncate "+LogFormatTools.getStackTop(e)); + LOGGER.debug("AAI_4000", "Exception running cron job for HistoryTruncate "+LogFormatTools.getStackTop(e)); + } finally { + LOGGER.debug("Ended cron job historyTruncate @ " + dateFormat.format(new Date())); + } + auditLog.logAfter(); + + } + + private boolean checkIfHistoryTruncateIsRunning(){ + + Process process = null; + + int count = 0; + try { + process = new ProcessBuilder().command("bash", "-c", "ps -ef | grep '[H]istoryTruncate'").start(); + InputStream is = process.getInputStream(); + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); + while (br.readLine() != null){ + count++; + } + + int exitVal = process.waitFor(); + LOGGER.debug("Exit value of the historyTruncate check process: " + exitVal); + } catch (Exception e) { + LOGGER.debug("Exception in checkIfHistoryTruncateIsRunning" + LogFormatTools.getStackTop(e)); + } + + return count > 0; + } +} + + diff --git a/src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java b/src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java deleted file mode 100644 index baf28ad..0000000 --- a/src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * ============LICENSE_START======================================================= - * org.onap.aai - * ================================================================================ - * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END========================================================= - */ -package org.onap.aai.interceptors.post; - -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; -import org.onap.aai.interceptors.AAIContainerFilter; -import org.onap.aai.logging.LoggingContext; -import org.onap.aai.logging.LoggingContext.StatusCode; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.annotation.Priority; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.container.ContainerRequestContext; -import javax.ws.rs.container.ContainerResponseContext; -import javax.ws.rs.container.ContainerResponseFilter; -import javax.ws.rs.core.Response.Status; -import javax.ws.rs.core.Response.StatusType; -import java.io.IOException; - -@Priority(AAIResponseFilterPriority.RESET_LOGGING_CONTEXT) -public class ResetLoggingContext extends AAIContainerFilter implements ContainerResponseFilter { - - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(ResetLoggingContext.class); - - @Autowired - private HttpServletRequest httpServletRequest; - - @Override - public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) - throws IOException { - - this.cleanLoggingContext(responseContext); - - } - - private void cleanLoggingContext(ContainerResponseContext responseContext) { - //String url = httpServletRequest.getRequestURL().toString(); - boolean success = true; - String uri = httpServletRequest.getRequestURI(); - String queryString = httpServletRequest.getQueryString(); - - if(queryString != null && !queryString.isEmpty()){ - uri = uri + "?" + queryString; - } - // For now, we use the the HTTP status code, - // This may change, once the requirements for response codes are defined - - int httpStatusCode = responseContext.getStatus(); - if ( httpStatusCode < 100 || httpStatusCode > 599 ) { - httpStatusCode = Status.INTERNAL_SERVER_ERROR.getStatusCode(); - } - LoggingContext.responseCode(Integer.toString(httpStatusCode)); - - StatusType sType = responseContext.getStatusInfo(); - if ( sType != null ) { - Status.Family sFamily = sType.getFamily(); - if ( ! ( Status.Family.SUCCESSFUL.equals(sFamily) || - ( Status.NOT_FOUND.equals(Status.fromStatusCode(httpStatusCode)) ) ) ) { - success = false; - } - } - else { - if ( (httpStatusCode < 200 || httpStatusCode > 299) && ( ! ( Status.NOT_FOUND.equals(Status.fromStatusCode(httpStatusCode) ) ) ) ) { - success = false; - } - } - if (success) { - LoggingContext.statusCode(StatusCode.COMPLETE); - LOGGER.info(uri + " call succeeded"); - } - else { - LoggingContext.statusCode(StatusCode.ERROR); - LOGGER.error(uri + " call failed with responseCode=" + httpStatusCode); - } - LoggingContext.clear(); - - - } - -} diff --git a/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java b/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java index 547a7c8..a544a90 100644 --- a/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java +++ b/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java @@ -19,8 +19,8 @@ */ package org.onap.aai.interceptors.post; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.gson.JsonObject; import org.onap.aai.exceptions.AAIException; import org.onap.aai.interceptors.AAIContainerFilter; @@ -41,7 +41,7 @@ import java.util.Optional; @Priority(AAIResponseFilterPriority.RESPONSE_TRANS_LOGGING) public class ResponseTransactionLogging extends AAIContainerFilter implements ContainerResponseFilter { - private static final EELFLogger TRANSACTION_LOGGER = EELFManager.getInstance().getLogger(ResponseTransactionLogging.class); + private static final Logger TRANSACTION_LOGGER = LoggerFactory.getLogger(ResponseTransactionLogging.class); @Autowired private HttpServletResponse httpServletResponse; diff --git a/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java b/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java index afacf66..0182f2c 100644 --- a/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java +++ b/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java @@ -23,6 +23,8 @@ import org.onap.aai.exceptions.AAIException; import org.onap.aai.interceptors.AAIContainerFilter; import org.onap.aai.interceptors.AAIHeaderProperties; import org.onap.aai.logging.ErrorLogHelper; +import org.onap.logging.filter.base.Constants; +import org.onap.logging.ref.slf4j.ONAPLogConstants; import javax.annotation.Priority; import javax.ws.rs.container.ContainerRequestContext; @@ -36,7 +38,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.UUID; @Provider @PreMatching @@ -49,36 +50,28 @@ public class HeaderValidation extends AAIContainerFilter implements ContainerReq Optional<Response> oResp; MultivaluedMap<String, String> headersMap = requestContext.getHeaders(); - - String transId = headersMap.getFirst(AAIHeaderProperties.TRANSACTION_ID); - String fromAppId = headersMap.getFirst(AAIHeaderProperties.FROM_APP_ID); List<MediaType> acceptHeaderValues = requestContext.getAcceptableMediaTypes(); - + String fromAppId = getPartnerName(requestContext); oResp = this.validateHeaderValuePresence(fromAppId, "AAI_4009", acceptHeaderValues); if (oResp.isPresent()) { requestContext.abortWith(oResp.get()); return; } + String transId = getRequestId(requestContext); oResp = this.validateHeaderValuePresence(transId, "AAI_4010", acceptHeaderValues); if (oResp.isPresent()) { requestContext.abortWith(oResp.get()); return; } - if (!this.isValidUUID(transId)) { - transId = UUID.randomUUID().toString(); - requestContext.getHeaders().get(AAIHeaderProperties.TRANSACTION_ID).clear(); - requestContext.getHeaders().add(AAIHeaderProperties.TRANSACTION_ID, transId); - } - } private Optional<Response> validateHeaderValuePresence(String value, String errorCode, List<MediaType> acceptHeaderValues) { Response response = null; AAIException aaie; - if (value == null) { + if (value == null || value.isEmpty()) { aaie = new AAIException(errorCode); return Optional.of(Response.status(aaie.getErrorObject().getHTTPResponseCode()) .entity(ErrorLogHelper.getRESTAPIErrorResponse(acceptHeaderValues, aaie, new ArrayList<>())) @@ -87,5 +80,52 @@ public class HeaderValidation extends AAIContainerFilter implements ContainerReq return Optional.ofNullable(response); } + public String getRequestId(ContainerRequestContext requestContext) { + String requestId = requestContext.getHeaderString(ONAPLogConstants.Headers.REQUEST_ID); + if (requestId == null || requestId.isEmpty()) { + requestId = requestContext.getHeaderString(Constants.HttpHeaders.HEADER_REQUEST_ID); + if (requestId == null || requestId.isEmpty()) { + requestId = requestContext.getHeaderString(Constants.HttpHeaders.TRANSACTION_ID); + if (requestId == null || requestId.isEmpty()) { + requestId = requestContext.getHeaderString(Constants.HttpHeaders.ECOMP_REQUEST_ID); + if (requestId == null || requestId.isEmpty()) { + return requestId; + } + } + } + } + if (requestContext.getHeaders().get(ONAPLogConstants.Headers.REQUEST_ID) != null) { + requestContext.getHeaders().get(ONAPLogConstants.Headers.REQUEST_ID).clear(); + } + if (requestContext.getHeaders().get(Constants.HttpHeaders.TRANSACTION_ID) != null) { + requestContext.getHeaders().get(Constants.HttpHeaders.TRANSACTION_ID).clear(); + } + if (requestContext.getHeaders().get(Constants.HttpHeaders.HEADER_REQUEST_ID) != null) { + requestContext.getHeaders().get(Constants.HttpHeaders.HEADER_REQUEST_ID).clear(); + } + if (requestContext.getHeaders().get(Constants.HttpHeaders.ECOMP_REQUEST_ID) != null) { + requestContext.getHeaders().get(Constants.HttpHeaders.ECOMP_REQUEST_ID).clear(); + } + requestContext.getHeaders().add(Constants.HttpHeaders.TRANSACTION_ID, requestId); + + return requestId; + } + public String getPartnerName(ContainerRequestContext requestContext) { + String partnerName = requestContext.getHeaderString(ONAPLogConstants.Headers.PARTNER_NAME); + if (partnerName == null || (partnerName.isEmpty())) { + partnerName = requestContext.getHeaderString(AAIHeaderProperties.FROM_APP_ID); + if (partnerName == null || (partnerName.isEmpty())) { + return partnerName; + } + } + if (requestContext.getHeaders().get(ONAPLogConstants.Headers.PARTNER_NAME) != null) { + requestContext.getHeaders().get(ONAPLogConstants.Headers.PARTNER_NAME).clear(); + } + if (requestContext.getHeaders().get(AAIHeaderProperties.FROM_APP_ID) != null) { + requestContext.getHeaders().get(AAIHeaderProperties.FROM_APP_ID).clear(); + } + requestContext.getHeaders().add(AAIHeaderProperties.FROM_APP_ID, partnerName); + return partnerName; + } } diff --git a/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java b/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java index b770296..6aa180b 100644 --- a/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java +++ b/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java @@ -19,8 +19,8 @@ */ package org.onap.aai.interceptors.pre; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.gson.JsonObject; import org.glassfish.jersey.message.internal.ReaderWriter; import org.onap.aai.exceptions.AAIException; @@ -53,7 +53,7 @@ import java.util.UUID; @Priority(AAIRequestFilterPriority.REQUEST_TRANS_LOGGING) public class RequestTransactionLogging extends AAIContainerFilter implements ContainerRequestFilter { - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(RequestTransactionLogging.class); + private static final Logger LOGGER = LoggerFactory.getLogger(RequestTransactionLogging.class); @Autowired private HttpServletRequest httpServletRequest; diff --git a/src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java b/src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java deleted file mode 100644 index 6c3a7fc..0000000 --- a/src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * ============LICENSE_START======================================================= - * org.onap.aai - * ================================================================================ - * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END========================================================= - */ -package org.onap.aai.interceptors.pre; - -import org.onap.aai.interceptors.AAIContainerFilter; -import org.onap.aai.interceptors.AAIHeaderProperties; -import org.onap.aai.logging.LoggingContext; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; - -import javax.annotation.Priority; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.container.ContainerRequestContext; -import javax.ws.rs.container.ContainerRequestFilter; -import javax.ws.rs.container.PreMatching; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.ext.Provider; -import java.io.IOException; - -@Provider -@PreMatching -@Priority(AAIRequestFilterPriority.SET_LOGGING_CONTEXT) -public class SetLoggingContext extends AAIContainerFilter implements ContainerRequestFilter { - - @Autowired - private Environment environment; - - @Autowired - private HttpServletRequest httpServletRequest; - - @Override - public void filter(ContainerRequestContext requestContext) throws IOException { - - String uri = httpServletRequest.getRequestURI(); - String queryString = httpServletRequest.getQueryString(); - - if(queryString != null && !queryString.isEmpty()){ - uri = uri + "?" + queryString; - } - - String httpMethod = requestContext.getMethod(); - - MultivaluedMap<String, String> headersMap = requestContext.getHeaders(); - - String transId = headersMap.getFirst(AAIHeaderProperties.TRANSACTION_ID); - String fromAppId = headersMap.getFirst(AAIHeaderProperties.FROM_APP_ID); - - LoggingContext.init(); - LoggingContext.requestId(transId); - LoggingContext.partnerName(fromAppId); - LoggingContext.targetEntity(environment.getProperty("spring.application.name")); - LoggingContext.component(fromAppId); - LoggingContext.serviceName(httpMethod + " " + uri); - LoggingContext.targetServiceName(httpMethod + " " + uri); - LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE); - } - -} diff --git a/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java b/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java index 73b7877..8c973a1 100644 --- a/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java +++ b/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java @@ -19,7 +19,7 @@ */ package org.onap.aai.interceptors.pre; -import org.onap.aai.auth.AAIAuthCore; +import org.onap.aai.aaf.auth.AAIAuthCore; import org.onap.aai.exceptions.AAIException; import org.onap.aai.interceptors.AAIContainerFilter; import org.onap.aai.interceptors.AAIHeaderProperties; diff --git a/src/main/java/org/onap/aai/migration/EdgeMigrator.java b/src/main/java/org/onap/aai/migration/EdgeMigrator.java index bb8acfc..c002afd 100644 --- a/src/main/java/org/onap/aai/migration/EdgeMigrator.java +++ b/src/main/java/org/onap/aai/migration/EdgeMigrator.java @@ -124,7 +124,7 @@ public abstract class EdgeMigrator extends Migrator { e.properties().forEachRemaining(prop -> prop.remove()); edgeSerializer.addProperties(e, rule); } else { - logger.info("found vertices connected by unkwown label: out=" + out + " label=" + logger.debug("found vertices connected by unkwown label: out=" + out + " label=" + e.label() + " in=" + in); } } diff --git a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java index b3faec8..3f25119 100644 --- a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java +++ b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java @@ -106,12 +106,12 @@ public abstract class EdgeSwingMigrator extends Migrator { try { // If the old and new Vertices aren't populated, throw an exception if( oldNode == null ){ - logger.info ( "null oldNode passed to swingEdges() "); + logger.debug ( "null oldNode passed to swingEdges() "); success = false; return; } else if( newNode == null ){ - logger.info ( "null newNode passed to swingEdges() "); + logger.debug ( "null newNode passed to swingEdges() "); success = false; return; } @@ -120,7 +120,7 @@ public abstract class EdgeSwingMigrator extends Migrator { && !edgeDirRestr.equals("IN") && !edgeDirRestr.equals("OUT") ) ){ - logger.info ( "invalid direction passed to swingEdges(). valid values are BOTH/IN/OUT "); + logger.debug ( "invalid direction passed to swingEdges(). valid values are BOTH/IN/OUT "); success = false; return; } @@ -140,7 +140,7 @@ public abstract class EdgeSwingMigrator extends Migrator { // If the nodeTypes don't match, throw an error if( !oldNodeType.equals(newNodeType) ){ - logger.info ( "Can not swing edge from a [" + oldNodeType + "] node to a [" + + logger.debug ( "Can not swing edge from a [" + oldNodeType + "] node to a [" + newNodeType + "] node. "); success = false; return; @@ -182,7 +182,7 @@ public abstract class EdgeSwingMigrator extends Migrator { } String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present"; - logger.info ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is [" + logger.debug ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is [" + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. "); // remove the old edge @@ -204,7 +204,7 @@ public abstract class EdgeSwingMigrator extends Migrator { newOutE.property(pair.getKey().toString(), pair.getValue().toString() ); } }else { - logger.info("\n Edge was not swung due to Multiplicity Rule Violation..."); + logger.debug("\n Edge was not swung due to Multiplicity Rule Violation..."); } } } @@ -245,7 +245,7 @@ public abstract class EdgeSwingMigrator extends Migrator { } String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present"; - logger.info ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is [" + logger.debug ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is [" + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. "); @@ -268,7 +268,7 @@ public abstract class EdgeSwingMigrator extends Migrator { newInE.property(pair.getKey().toString(), pair.getValue().toString() ); } } else { - logger.info("\t Edge was not swung due to Multiplicity Rule Violation..."); + logger.debug("\t Edge was not swung due to Multiplicity Rule Violation..."); } } } diff --git a/src/main/java/org/onap/aai/migration/MigrationController.java b/src/main/java/org/onap/aai/migration/MigrationController.java index abd4648..8d758e3 100644 --- a/src/main/java/org/onap/aai/migration/MigrationController.java +++ b/src/main/java/org/onap/aai/migration/MigrationController.java @@ -25,8 +25,6 @@ import org.onap.aai.edges.EdgeIngestor; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.logging.ErrorLogHelper; -import org.onap.aai.logging.LoggingContext; -import org.onap.aai.logging.LoggingContext.StatusCode; import org.onap.aai.serialization.db.EdgeSerializer; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConstants; @@ -49,16 +47,6 @@ public class MigrationController { */ public static void main(String[] args) throws AAIException { - LoggingContext.init(); - LoggingContext.partnerName("Migration"); - LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.component("MigrationController"); - LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.targetServiceName("main"); - LoggingContext.requestId(UUID.randomUUID().toString()); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); initializer.initialize(ctx); @@ -71,8 +59,6 @@ public class MigrationController { } catch (Exception e) { AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); System.out.println("Problems running tool "+aai.getMessage()); - LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } diff --git a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java index 55b03fb..7cb71ca 100644 --- a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java +++ b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java @@ -20,51 +20,44 @@ package org.onap.aai.migration; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; - +import com.att.eelf.configuration.Configuration; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.io.IoCore; import org.onap.aai.datasnapshot.DataSnapshot; import org.onap.aai.db.props.AAIProperties; import org.onap.aai.dbmap.AAIGraph; -import org.onap.aai.dbmap.DBConnectionType; import org.onap.aai.edges.EdgeIngestor; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Loader; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.introspection.ModelType; import org.onap.aai.serialization.db.EdgeSerializer; -import org.onap.aai.setup.SchemaVersions; -import org.onap.aai.setup.SchemaVersion; -import org.onap.aai.logging.LoggingContext; -import org.onap.aai.logging.LoggingContext.StatusCode; -import org.onap.aai.serialization.engines.QueryStyle; import org.onap.aai.serialization.engines.JanusGraphDBEngine; +import org.onap.aai.serialization.engines.QueryStyle; import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConstants; import org.onap.aai.util.FormatDate; import org.reflections.Reflections; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; /** * Runs a series of migrations from a defined directory based on the presence of @@ -74,8 +67,7 @@ import com.beust.jcommander.Parameter; */ public class MigrationControllerInternal { - private EELFLogger logger; - private final int DANGER_ZONE = 10; + private Logger logger; public static final String VERTEX_TYPE = "migration-list-1707"; private final List<String> resultsSummary = new ArrayList<>(); private final List<NotificationHelper> notifications = new ArrayList<>(); @@ -107,11 +99,9 @@ public class MigrationControllerInternal { props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml"); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES); - logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName()); + logger = LoggerFactory.getLogger(MigrationControllerInternal.class.getSimpleName()); MDC.put("logFilenameAppender", MigrationController.class.getSimpleName()); - boolean loadSnapshot = false; - CommandLineArgs cArgs = new CommandLineArgs(); JCommander jCommander = new JCommander(cArgs, args); @@ -123,8 +113,6 @@ public class MigrationControllerInternal { try { PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config); if (config.getString("storage.backend").equals("inmemory")) { - loadSnapshot = true; -// System.setProperty("load.snapshot.file", "true"); System.setProperty("snapshot.location", cArgs.dataSnapshot); String snapshotLocation =cArgs.dataSnapshot; String snapshotDir; @@ -136,15 +124,13 @@ public class MigrationControllerInternal { snapshotFile = snapshotLocation; } else { snapshotDir = snapshotLocation.substring(0, index+1); - snapshotFile = snapshotLocation.substring(index+1, snapshotLocation.length()) ; + snapshotFile = snapshotLocation.substring(index+1) ; } String [] dataSnapShotArgs = {"-c","MULTITHREAD_RELOAD","-f", snapshotFile, "-oldFileDir",snapshotDir, "-caller","migration"}; DataSnapshot dataSnapshot = new DataSnapshot(); dataSnapshot.executeCommand(dataSnapShotArgs, true, false, null, "MULTITHREAD_RELOAD", snapshotFile); } } catch (ConfigurationException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e)); return; } @@ -160,7 +146,7 @@ public class MigrationControllerInternal { QueryStyle queryStyle = QueryStyle.TRAVERSAL; ModelType introspectorFactoryType = ModelType.MOXY; Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); - TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader); + TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, loader); if (cArgs.help) { jCommander.usage(); @@ -172,7 +158,7 @@ public class MigrationControllerInternal { List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections)); //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names if (cArgs.list) { - listMigrationWithStatus(cArgs, migratorClasses, engine); + listMigrationWithStatus(migratorClasses, engine); return; } @@ -180,18 +166,15 @@ public class MigrationControllerInternal { //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names if (!cArgs.excludeClasses.isEmpty()) { migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses); - listMigrationWithStatus(cArgs, migratorClasses, engine); + listMigrationWithStatus(migratorClasses, engine); } List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses); sortList(migratorClassesToRun); if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + "."); logAndPrint("---------- Done ----------"); - LoggingContext.successStatusFields(); } logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts."); @@ -199,7 +182,7 @@ public class MigrationControllerInternal { if (!cArgs.skipPreMigrationSnapShot) { - takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun); + takePreSnapshotIfRequired(engine); } for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { @@ -222,10 +205,7 @@ public class MigrationControllerInternal { SchemaVersions.class ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions); } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e)); - LoggingContext.successStatusFields(); engine.rollback(); continue; } @@ -245,11 +225,8 @@ public class MigrationControllerInternal { try { notificationHelper.triggerEvents(); } catch (AAIException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); logAndPrint("\tcould not event"); logger.error("could not event", e); - LoggingContext.successStatusFields(); } } logAndPrint("---------- Done ----------"); @@ -276,16 +253,13 @@ public class MigrationControllerInternal { List<String> excludeClasses, List<Class<? extends Migrator>> migratorClasses) { - List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses + return migratorClasses .stream() .filter(migratorClass -> !excludeClasses.contains(migratorClass .getSimpleName())).collect(Collectors.toList()); - - return filteredMigratorClasses; } - private void listMigrationWithStatus(CommandLineArgs cArgs, - List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) { + private void listMigrationWithStatus(List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) { sortList(migratorClasses); engine.startTransaction(); System.out.println("---------- List of all migrations ----------"); @@ -317,9 +291,9 @@ public class MigrationControllerInternal { return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext(); } private Set<Class<? extends Migrator>> findClasses(Reflections reflections) { - Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class).stream() - .filter(clazz -> clazz.isAnnotationPresent(MigrationPriority.class)) - .collect(Collectors.toSet()); + Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class).stream() + .filter(clazz -> clazz.isAnnotationPresent(MigrationPriority.class)) + .collect(Collectors.toSet()); /* * TODO- Change this to make sure only classes in the specific $release are added in the runList * Or add a annotation like exclude which folks again need to remember to add ?? @@ -331,22 +305,7 @@ public class MigrationControllerInternal { } - private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) { - - /*int sum = 0; - for (Class<? extends Migrator> migratorClass : migratorClassesToRun) { - if (migratorClass.isAnnotationPresent(Enabled.class)) { - sum += migratorClass.getAnnotation(MigrationPriority.class).value(); - } - } - - if (sum >= DANGER_ZONE) { - - logAndPrint("Entered Danger Zone. Taking snapshot."); - }*/ - - //always take snapshot for now - + private void takePreSnapshotIfRequired(TransactionalGraphEngine engine) { generateSnapshot(engine, "pre"); } @@ -357,15 +316,13 @@ public class MigrationControllerInternal { List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>(); if (cArgs.scripts.isEmpty()) { return migratorClasses; + } - for (Class<? extends Migrator> migratorClass : migratorClasses) { - if (migratorExplicitlySpecified(cArgs, migratorClass.getSimpleName()) - || migratorToRunWhenDisabled(cArgs, migratorClass.getSimpleName())) { + if (migratorExplicitlySpecified(cArgs, migratorClass.getSimpleName()) || migratorToRunWhenDisabled(cArgs, migratorClass.getSimpleName())) { migratorClassesToRun.add(migratorClass); } } - return migratorClassesToRun; } @@ -377,7 +334,7 @@ public class MigrationControllerInternal { } private void sortList(List<Class<? extends Migrator>> migratorClasses) { - Collections.sort(migratorClasses, (m1, m2) -> { + migratorClasses.sort((m1, m2) -> { try { if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) { return 1; @@ -399,7 +356,6 @@ public class MigrationControllerInternal { String dateStr= fd.getDateTime(); String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson"; logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName); - Graph transaction = null; try { Path pathToFile = Paths.get(fileName); @@ -409,14 +365,8 @@ public class MigrationControllerInternal { String [] dataSnapshotArgs = {"-c","THREADED_SNAPSHOT", "-fileName",fileName, "-caller","migration"}; DataSnapshot dataSnapshot = new DataSnapshot(); dataSnapshot.executeCommand(dataSnapshotArgs, true, false, null, "THREADED_SNAPSHOT", null); -// transaction = engine.startTransaction(); -// transaction.io(IoCore.graphson()).writeGraph(fileName); -// engine.rollback(); } catch (IOException e) { - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e)); - LoggingContext.successStatusFields(); engine.rollback(); } @@ -430,7 +380,7 @@ public class MigrationControllerInternal { */ protected void logAndPrint(String msg) { System.out.println(msg); - logger.info(msg); + logger.debug(msg); } /** @@ -448,24 +398,18 @@ public class MigrationControllerInternal { String message; if (migrator.getStatus().equals(Status.FAILURE)) { message = "Migration " + simpleName + " Failed. Rolling back."; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logAndPrint("\t" + message); - LoggingContext.successStatusFields(); migrator.rollback(); } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) { message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back."; - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); logAndPrint("\t" + message); - LoggingContext.successStatusFields(); migrator.rollback(); } else { MDC.put("logFilenameAppender", simpleName + "/" + simpleName); if (cArgs.commit) { if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) { - engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate(); + engine.asAdmin().getTraversalSource().addV(VERTEX_TYPE).property(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate(); } engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE) .property(simpleName, true).iterate(); diff --git a/src/main/java/org/onap/aai/migration/Migrator.java b/src/main/java/org/onap/aai/migration/Migrator.java index 791fec0..7d6a7c1 100644 --- a/src/main/java/org/onap/aai/migration/Migrator.java +++ b/src/main/java/org/onap/aai/migration/Migrator.java @@ -55,8 +55,8 @@ import org.onap.aai.serialization.engines.TransactionalGraphEngine; import org.onap.aai.setup.SchemaVersion; import org.onap.aai.setup.SchemaVersions; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class defines an A&AI Migration @@ -65,7 +65,7 @@ import com.att.eelf.configuration.EELFManager; @MigrationDangerRating(0) public abstract class Migrator implements Runnable { - protected EELFLogger logger = null; + protected Logger logger = null; protected DBSerializer serializer = null; protected Loader loader = null; @@ -96,8 +96,8 @@ public abstract class Migrator implements Runnable { this.schemaVersions = schemaVersions; initDBSerializer(); this.notificationHelper = new NotificationHelper(loader, serializer, loaderFactory, schemaVersions, engine, "AAI-MIGRATION", this.getMigrationName()); - logger = EELFManager.getInstance().getLogger(this.getClass().getSimpleName()); - logger.info("\tInitilization of " + this.getClass().getSimpleName() + " migration script complete."); + logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + logger.debug("\tInitilization of " + this.getClass().getSimpleName() + " migration script complete."); } /** @@ -139,7 +139,7 @@ public abstract class Migrator implements Runnable { logger.error("Unable to generate file with dmaap msgs for " + getMigrationName(), e); } } else { - logger.info("No dmaap msgs detected for " + getMigrationName()); + logger.debug("No dmaap msgs detected for " + getMigrationName()); } } diff --git a/src/main/java/org/onap/aai/migration/NotificationHelper.java b/src/main/java/org/onap/aai/migration/NotificationHelper.java index ff5c030..f10a824 100644 --- a/src/main/java/org/onap/aai/migration/NotificationHelper.java +++ b/src/main/java/org/onap/aai/migration/NotificationHelper.java @@ -37,8 +37,8 @@ import org.onap.aai.serialization.db.DBSerializer; import org.onap.aai.serialization.engines.TransactionalGraphEngine; import org.onap.aai.serialization.engines.query.QueryEngine; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.onap.aai.setup.SchemaVersions; /** @@ -46,7 +46,7 @@ import org.onap.aai.setup.SchemaVersions; */ public class NotificationHelper { - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(NotificationHelper.class); + private static final Logger LOGGER = LoggerFactory.getLogger(NotificationHelper.class); protected final DBSerializer serializer; protected final Loader loader; protected final TransactionalGraphEngine engine; diff --git a/src/main/java/org/onap/aai/migration/PropertyMigrator.java b/src/main/java/org/onap/aai/migration/PropertyMigrator.java index 4599243..15e704d 100644 --- a/src/main/java/org/onap/aai/migration/PropertyMigrator.java +++ b/src/main/java/org/onap/aai/migration/PropertyMigrator.java @@ -68,11 +68,11 @@ public abstract class PropertyMigrator extends Migrator { */ @Override public void run() { - logger.info("-------- Starting PropertyMigrator for node type " + P.within(this.getAffectedNodeTypes().get()) + logger.debug("-------- Starting PropertyMigrator for node type " + P.within(this.getAffectedNodeTypes().get()) + " from property " + OLD_FIELD + " to " + NEW_FIELD + " --------"); modifySchema(); executeModifyOperation(); - logger.info(Migrator.MIGRATION_SUMMARY_COUNT + changedVertexCount + " vertices modified."); + logger.debug(Migrator.MIGRATION_SUMMARY_COUNT + changedVertexCount + " vertices modified."); } protected void modifySchema() { @@ -94,13 +94,13 @@ public abstract class PropertyMigrator extends Migrator { } g.has(OLD_FIELD).sideEffect(t -> { final Vertex v = t.get(); - logger.info("Migrating property for vertex " + v.toString()); + logger.debug("Migrating property for vertex " + v.toString()); final String value = v.value(OLD_FIELD); v.property(OLD_FIELD).remove(); v.property(NEW_FIELD, value); this.touchVertexProperties(v, false); this.changedVertexCount += 1; - logger.info(v.toString() + " : Migrated property " + OLD_FIELD + " to " + NEW_FIELD + " with value = " + value); + logger.debug(v.toString() + " : Migrated property " + OLD_FIELD + " to " + NEW_FIELD + " with value = " + value); }).iterate(); } @@ -121,11 +121,11 @@ public abstract class PropertyMigrator extends Migrator { protected Optional<PropertyKey> addProperty() { if (!graphMgmt.containsPropertyKey(this.NEW_FIELD)) { - logger.info(" PropertyKey [" + this.NEW_FIELD + "] created in the DB. "); + logger.debug(" PropertyKey [" + this.NEW_FIELD + "] created in the DB. "); return Optional.of(graphMgmt.makePropertyKey(this.NEW_FIELD).dataType(this.fieldType).cardinality(this.cardinality) .make()); } else { - logger.info(" PropertyKey [" + this.NEW_FIELD + "] already existed in the DB. "); + logger.debug(" PropertyKey [" + this.NEW_FIELD + "] already existed in the DB. "); return Optional.empty(); } @@ -136,7 +136,7 @@ public abstract class PropertyMigrator extends Migrator { if (graphMgmt.containsGraphIndex(key.get().name())) { logger.debug(" Index [" + key.get().name() + "] already existed in the DB. "); } else { - logger.info("Add index for PropertyKey: [" + key.get().name() + "]"); + logger.debug("Add index for PropertyKey: [" + key.get().name() + "]"); graphMgmt.buildIndex(key.get().name(), Vertex.class).addKey(key.get()).buildCompositeIndex(); } } diff --git a/src/main/java/org/onap/aai/migration/RebuildAllEdges.java b/src/main/java/org/onap/aai/migration/RebuildAllEdges.java index 73f5678..8bc2b8d 100644 --- a/src/main/java/org/onap/aai/migration/RebuildAllEdges.java +++ b/src/main/java/org/onap/aai/migration/RebuildAllEdges.java @@ -61,20 +61,20 @@ public class RebuildAllEdges extends EdgeMigrator { @Override protected void executeModifyOperation() { Instant started = Instant.now(); - logger.info("Started at: " + started); + logger.debug("Started at: " + started); GraphTraversalSource graphTraversalSource = engine.asAdmin().getTraversalSource(); Set<Edge> edges = graphTraversalSource.E().toSet(); rebuildEdges(edges); Instant completed = Instant.now(); - logger.info("Completed at: " + completed + ". Total time taken in ms : " + logger.debug("Completed at: " + completed + ". Total time taken in ms : " + (completed.toEpochMilli() - started.toEpochMilli())); - logger.info(MIGRATION_SUMMARY_COUNT + " Total Edges : " + edges.size() + " . Processed count " + processed + logger.debug(MIGRATION_SUMMARY_COUNT + " Total Edges : " + edges.size() + " . Processed count " + processed + " . Skipped count: " + skipped + "."); - logger.info(MIGRATION_SUMMARY_COUNT + "Edge Missing Parent Property Count: " + logger.debug(MIGRATION_SUMMARY_COUNT + "Edge Missing Parent Property Count: " + edgeMissingParentProperty.size()); - logger.info(MIGRATION_ERROR + "Edge Multiplicity Exception Count : " + logger.debug(MIGRATION_ERROR + "Edge Multiplicity Exception Count : " + edgeMultiplicityExceptionCtr.values().stream().mapToInt(Number::intValue).sum()); - logger.info(MIGRATION_ERROR + "Edge Multiplicity Exception Breakdown : " + edgeMultiplicityExceptionCtr); + logger.debug(MIGRATION_ERROR + "Edge Multiplicity Exception Breakdown : " + edgeMultiplicityExceptionCtr); } @Override diff --git a/src/main/java/org/onap/aai/migration/ValueMigrator.java b/src/main/java/org/onap/aai/migration/ValueMigrator.java index 458796a..f45b20b 100644 --- a/src/main/java/org/onap/aai/migration/ValueMigrator.java +++ b/src/main/java/org/onap/aai/migration/ValueMigrator.java @@ -121,12 +121,12 @@ public abstract class ValueMigrator extends Migrator { this.nodeTotalSuccess.put(nodeType, Integer.toString(this.subTotal)); } - logger.info ("\n \n ******* Final Summary for " + " " + getMigrationName() +" ********* \n"); + logger.debug ("\n \n ******* Final Summary for " + " " + getMigrationName() +" ********* \n"); for (Map.Entry<String, String> migratedNode: nodeTotalSuccess.entrySet()) { - logger.info("Total Migrated Records for " + migratedNode.getKey() +": " + migratedNode.getValue()); + logger.debug("Total Migrated Records for " + migratedNode.getKey() +": " + migratedNode.getValue()); } - logger.info(this.MIGRATION_SUMMARY_COUNT + "Total Migrated Records: "+ migrationSuccess); + logger.debug(this.MIGRATION_SUMMARY_COUNT + "Total Migrated Records: "+ migrationSuccess); } @@ -136,18 +136,18 @@ public abstract class ValueMigrator extends Migrator { String propertyValue = v.property(property).value().toString(); if (propertyValue.isEmpty()) { v.property(property, newValue); - logger.info(String.format("Node Type %s: Property %s is empty, adding value %s", + logger.debug(String.format("Node Type %s: Property %s is empty, adding value %s", nodeType, property, newValue.toString())); this.touchVertexProperties(v, false); updateDmaapList(v); this.migrationSuccess++; this.subTotal++; } else { - logger.info(String.format("Node Type %s: Property %s value already exists - skipping", + logger.debug(String.format("Node Type %s: Property %s value already exists - skipping", nodeType, property)); } } else { - logger.info(String.format("Node Type %s: Property %s does not exist or " + + logger.debug(String.format("Node Type %s: Property %s does not exist or " + "updateExistingValues flag is set to True - adding the property with value %s", nodeType, property, newValue.toString())); v.property(property, newValue); @@ -178,7 +178,7 @@ public abstract class ValueMigrator extends Migrator { private void updateDmaapList(Vertex v){ String dmaapMsg = System.nanoTime() + "_" + v.id().toString() + "_" + v.value("resource-version").toString(); dmaapMsgList.add(dmaapMsg); - logger.info("\tAdding Updated Vertex " + v.id().toString() + " to dmaapMsgList...."); + logger.debug("\tAdding Updated Vertex " + v.id().toString() + " to dmaapMsgList...."); } public boolean isUpdateDmaap(){ diff --git a/src/main/java/org/onap/aai/migration/VertexMerge.java b/src/main/java/org/onap/aai/migration/VertexMerge.java index abf19be..a2c814e 100644 --- a/src/main/java/org/onap/aai/migration/VertexMerge.java +++ b/src/main/java/org/onap/aai/migration/VertexMerge.java @@ -41,8 +41,8 @@ import org.onap.aai.serialization.db.DBSerializer; import org.onap.aai.edges.enums.EdgeType; import org.onap.aai.serialization.db.EdgeSerializer; import org.onap.aai.serialization.engines.TransactionalGraphEngine; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class recursively merges two vertices passed in. @@ -52,7 +52,7 @@ import com.att.eelf.configuration.EELFManager; */ public class VertexMerge { - private final EELFLogger logger = EELFManager.getInstance().getLogger(this.getClass().getSimpleName()); + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); private final GraphTraversalSource g; private final TransactionalGraphEngine engine; @@ -98,7 +98,7 @@ public class VertexMerge { Collection<Vertex> primaryCousins = this.engine.getQueryEngine().findCousinVertices(primary); secondaryCousins.removeAll(primaryCousins); - logger.info("removing vertex after merge: " + secondary ); + logger.debug("removing vertex after merge: " + secondary ); if (this.hasNotifications && secondarySnapshot.isPresent()) { this.notificationHelper.addEvent(secondary, secondarySnapshot.get(), EventAction.DELETE, this.serializer.getURIForVertex(secondary, false), basePath); } diff --git a/src/main/java/org/onap/aai/migration/v16/MigrateBooleanDefaultsToFalse.java b/src/main/java/org/onap/aai/migration/v16/MigrateBooleanDefaultsToFalse.java new file mode 100644 index 0000000..71c1999 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v16/MigrateBooleanDefaultsToFalse.java @@ -0,0 +1,78 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v16; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Status; +import org.onap.aai.migration.ValueMigrator; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(20) +@MigrationDangerRating(2) +//@Enabled +public class MigrateBooleanDefaultsToFalse extends ValueMigrator { + protected static final String CLOUD_REGION_NODE_TYPE = "cloud-region"; + + private static Map<String, Map> map; + private static Map<String, Boolean> pair1; + + + public MigrateBooleanDefaultsToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setBooleanDefaultsToFalse(), false); + + } + + private static Map<String, Map> setBooleanDefaultsToFalse(){ + map = new HashMap<>(); + pair1 = new HashMap<>(); + + pair1.put("orchestration-disabled", false); + map.put("cloud-region", pair1); + + return map; + } + + @Override + public Status getStatus() { + return Status.SUCCESS; + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{CLOUD_REGION_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateBooleanDefaultsToFalse"; + } + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/migration/v16/MigrateInMaintDefaultToFalse.java b/src/main/java/org/onap/aai/migration/v16/MigrateInMaintDefaultToFalse.java new file mode 100644 index 0000000..1a485b9 --- /dev/null +++ b/src/main/java/org/onap/aai/migration/v16/MigrateInMaintDefaultToFalse.java @@ -0,0 +1,82 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.migration.v16; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.onap.aai.edges.EdgeIngestor; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.serialization.db.EdgeSerializer; +import org.onap.aai.serialization.engines.TransactionalGraphEngine; +import org.onap.aai.migration.Enabled; +import org.onap.aai.migration.MigrationDangerRating; +import org.onap.aai.migration.MigrationPriority; +import org.onap.aai.migration.Status; +import org.onap.aai.migration.ValueMigrator; +import org.onap.aai.setup.SchemaVersions; + + +@MigrationPriority(20) +@MigrationDangerRating(2) +//@Enabled +public class MigrateInMaintDefaultToFalse extends ValueMigrator { + + protected static final String ZONE_NODE_TYPE = "zone"; + protected static final String CLOUD_REGION_NODE_TYPE = "cloud-region"; + + private static Map<String, Map> map; + private static Map<String, Boolean> pair; + + public MigrateInMaintDefaultToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) { + super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setInMaintToFalse(), false); + } + + private static Map<String, Map> setInMaintToFalse(){ + map = new HashMap<>(); + pair = new HashMap<>(); + + pair.put("in-maint", false); + + map.put("zone", pair); + map.put("cloud-region", pair); + + + return map; + } + + @Override + public Status getStatus() { + return Status.SUCCESS; + } + + @Override + public Optional<String[]> getAffectedNodeTypes() { + return Optional.of(new String[]{ZONE_NODE_TYPE,CLOUD_REGION_NODE_TYPE}); + } + + @Override + public String getMigrationName() { + return "MigrateInMaintDefaultToFalse"; + } + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/rest/QueryConsumer.java b/src/main/java/org/onap/aai/rest/QueryConsumer.java index 85665da..b709cfa 100644 --- a/src/main/java/org/onap/aai/rest/QueryConsumer.java +++ b/src/main/java/org/onap/aai/rest/QueryConsumer.java @@ -19,19 +19,15 @@ */ package org.onap.aai.rest; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.onap.aai.concurrent.AaiCallable; -import org.onap.aai.dbmap.DBConnectionType; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.ModelType; -import org.onap.aai.rest.dsl.DslQueryProcessor; -import org.onap.aai.logging.LoggingContext; import org.onap.aai.logging.StopWatch; import org.onap.aai.rest.db.HttpEntry; +import org.onap.aai.rest.dsl.DslQueryProcessor; import org.onap.aai.rest.search.GenericQueryProcessor; import org.onap.aai.rest.search.QueryProcessorType; import org.onap.aai.restcore.HttpMethod; @@ -45,6 +41,8 @@ import org.onap.aai.serialization.queryformats.SubGraphStyle; import org.onap.aai.setup.SchemaVersion; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -54,7 +52,6 @@ import javax.ws.rs.*; import javax.ws.rs.core.*; import javax.ws.rs.core.Response.Status; import java.util.List; -import java.util.concurrent.TimeUnit; @Component @Path("{version: v[1-9][0-9]*|latest}/dbquery") @@ -66,7 +63,7 @@ public class QueryConsumer extends RESTAPI { private QueryProcessorType processorType = QueryProcessorType.LOCAL_GROOVY; private static final String TARGET_ENTITY = "DB"; - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(QueryConsumer.class); + private static final Logger LOGGER = LoggerFactory.getLogger(QueryConsumer.class); private HttpEntry traversalUriHttpEntry; @@ -111,15 +108,12 @@ public class QueryConsumer extends RESTAPI { public Response processExecuteQuery(String content, @PathParam("version")String versionParam, @PathParam("uri") @Encoded String uri, @DefaultValue("graphson") @QueryParam("format") String queryFormat,@DefaultValue("no_op") @QueryParam("subgraph") String subgraph, @Context HttpHeaders headers, @Context UriInfo info, @Context HttpServletRequest req) { - String methodName = "executeQuery"; String sourceOfTruth = headers.getRequestHeaders().getFirst("X-FromAppId"); - String realTime = headers.getRequestHeaders().getFirst("Real-Time"); String queryProcessor = headers.getRequestHeaders().getFirst("QueryProcessor"); QueryProcessorType processorType = this.processorType; Response response = null; TransactionalGraphEngine dbEngine = null; try { - LoggingContext.save(); this.checkQueryParams(info.getQueryParameters()); Format format = Format.getFormat(queryFormat); if (queryProcessor != null) { @@ -132,13 +126,11 @@ public class QueryConsumer extends RESTAPI { JsonElement gremlinElement = input.get("gremlin"); JsonElement dslElement = input.get("dsl"); - String queryURI = ""; String gremlin = ""; String dsl = ""; SchemaVersion version = new SchemaVersion(versionParam); - DBConnectionType type = this.determineConnectionType(sourceOfTruth, realTime); - traversalUriHttpEntry.setHttpEntryProperties(version, type); + traversalUriHttpEntry.setHttpEntryProperties(version); dbEngine = traversalUriHttpEntry.getDbEngine(); if (gremlinElement != null) { @@ -147,11 +139,8 @@ public class QueryConsumer extends RESTAPI { if (dslElement != null) { dsl = dslElement.getAsString(); } - GenericQueryProcessor processor = null; + GenericQueryProcessor processor; - LoggingContext.targetEntity(TARGET_ENTITY); - LoggingContext.targetServiceName(methodName); - LoggingContext.startTime(); StopWatch.conditionalStart(); if(!dsl.equals("")){ @@ -175,9 +164,6 @@ public class QueryConsumer extends RESTAPI { result = formater.output(vertices).toString(); - double msecs = StopWatch.stopIfStarted(); - LoggingContext.elapsedTime((long)msecs,TimeUnit.MILLISECONDS); - LoggingContext.successStatusFields(); LOGGER.info ("Completed"); response = Response.status(Status.OK) @@ -190,8 +176,6 @@ public class QueryConsumer extends RESTAPI { AAIException ex = new AAIException("AAI_4000", e); response = consumerExceptionResponseGenerator(headers, info, HttpMethod.GET, ex); } finally { - LoggingContext.restoreIfPossible(); - LoggingContext.successStatusFields(); if (dbEngine != null) { dbEngine.rollback(); } @@ -205,7 +189,7 @@ public class QueryConsumer extends RESTAPI { if (params.containsKey("depth") && params.getFirst("depth").matches("\\d+")) { String depth = params.getFirst("depth"); - Integer i = Integer.parseInt(depth); + int i = Integer.parseInt(depth); if (i > 1) { throw new AAIException("AAI_3303"); } diff --git a/src/main/java/org/onap/aai/rest/dsl/DslListener.java b/src/main/java/org/onap/aai/rest/dsl/DslListener.java index e41a946..70d8bf8 100644 --- a/src/main/java/org/onap/aai/rest/dsl/DslListener.java +++ b/src/main/java/org/onap/aai/rest/dsl/DslListener.java @@ -19,8 +19,8 @@ */ package org.onap.aai.rest.dsl; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.antlr.v4.runtime.tree.TerminalNode; import org.onap.aai.AAIDslBaseListener; import org.onap.aai.AAIDslParser; @@ -40,7 +40,7 @@ import java.util.Map; */ public class DslListener extends AAIDslBaseListener { - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DslQueryProcessor.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DslQueryProcessor.class); private final EdgeIngestor edgeRules; //TODO Use StringBuilder to build the query than concat diff --git a/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java b/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java index 582f0ea..3e77e6c 100644 --- a/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java +++ b/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java @@ -19,8 +19,8 @@ */ package org.onap.aai.rest.dsl; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.tree.ParseTree; @@ -38,7 +38,7 @@ import java.nio.charset.StandardCharsets; */ public class DslQueryProcessor { - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DslQueryProcessor.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DslQueryProcessor.class); private DslListener dslListener; diff --git a/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java b/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java index 8f83751..0964bc0 100644 --- a/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java +++ b/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java @@ -20,7 +20,10 @@ package org.onap.aai.rest.search; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.onap.aai.aailog.logs.AaiDBGraphadminMetricLog; import org.onap.aai.restcore.search.GremlinGroovyShell; +import org.onap.aai.util.AAIConstants; +import org.onap.aai.util.GraphAdminConstants; import java.util.Map; @@ -32,12 +35,18 @@ public class GroovyShellImpl extends GenericQueryProcessor { @Override protected GraphTraversal<?,?> runQuery(String query, Map<String, Object> params) { - + + AaiDBGraphadminMetricLog metricLog = new AaiDBGraphadminMetricLog (GraphAdminConstants.AAI_GRAPHADMIN_MS); + metricLog.pre(uri); + params.put("g", this.dbEngine.asAdmin().getTraversalSource()); GremlinGroovyShell shell = new GremlinGroovyShell(); - return shell.executeTraversal(query, params); + GraphTraversal<?,?> graphTraversal = shell.executeTraversal(query, params); + + metricLog.post(); + return graphTraversal; } } diff --git a/src/main/java/org/onap/aai/schema/GenTester.java b/src/main/java/org/onap/aai/schema/GenTester.java index 04ade0a..84ecf19 100644 --- a/src/main/java/org/onap/aai/schema/GenTester.java +++ b/src/main/java/org/onap/aai/schema/GenTester.java @@ -20,8 +20,8 @@ package org.onap.aai.schema; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.schema.JanusGraphManagement; import org.onap.aai.config.PropertyPasswordConfiguration; @@ -29,8 +29,6 @@ import org.onap.aai.dbgen.SchemaGenerator; import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.ErrorLogHelper; -import org.onap.aai.logging.LoggingContext; -import org.onap.aai.logging.LoggingContext.StatusCode; import org.onap.aai.util.*; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @@ -40,7 +38,8 @@ import java.util.UUID; public class GenTester { - private static EELFLogger LOGGER; + private static Logger LOGGER; + private static boolean historyEnabled; /** * The main method. @@ -55,19 +54,9 @@ public class GenTester { Properties props = System.getProperties(); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); - LOGGER = EELFManager.getInstance().getLogger(GenTester.class); + LOGGER = LoggerFactory.getLogger(GenTester.class); boolean addDefaultCR = true; - LoggingContext.init(); - LoggingContext.component("DBGenTester"); - LoggingContext.partnerName("AAI-TOOLS"); - LoggingContext.targetEntity("AAI"); - LoggingContext.requestId(UUID.randomUUID().toString()); - LoggingContext.serviceName("AAI"); - LoggingContext.targetServiceName("main"); - LoggingContext.statusCode(StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); initializer.initialize(ctx); @@ -80,13 +69,18 @@ public class GenTester { } catch (Exception e) { AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); LOGGER.error("Problems running the tool "+aai.getMessage()); - LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); + if( historyEnabled ) { + String amsg = "GenTester may only be used when history.enabled=false. "; + System.out.println(amsg); + LOGGER.debug(amsg); + return; + } try { - LOGGER.info("GenTester uses either cql jar or Cassandra jar"); + LOGGER.debug("GenTester uses either cql jar or Cassandra jar"); AAIConfig.init(); if (args != null && args.length > 0 ){ @@ -100,7 +94,7 @@ public class GenTester { // an HBase copyTable can be used to set up a copy of the db. String imsg = " ---- NOTE --- about to load a graph without doing any schema processing (takes a little while) -------- "; System.out.println(imsg); - LOGGER.info(imsg); + LOGGER.debug(imsg); graph = AAIGraph.getInstance().getGraph(); if( graph == null ){ @@ -110,7 +104,7 @@ public class GenTester { else { String amsg = "Successfully loaded a JanusGraph graph without doing any schema work. "; System.out.println(amsg); - LOGGER.auditEvent(amsg); + LOGGER.debug(amsg); return; } } else if ("GEN_DB_WITH_NO_DEFAULT_CR".equals(args[0])) { @@ -121,8 +115,6 @@ public class GenTester { String emsg = "Unrecognized argument passed to GenTester.java: [" + args[0] + "]. "; System.out.println(emsg); - LoggingContext.statusCode(StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); LOGGER.error(emsg); emsg = "Either pass no argument for normal processing, or use 'GEN_DB_WITH_NO_SCHEMA'."; @@ -137,7 +129,7 @@ public class GenTester { ErrorLogHelper.loadProperties(); String imsg = " ---- NOTE --- about to open graph (takes a little while)--------;"; System.out.println(imsg); - LOGGER.info(imsg); + LOGGER.debug(imsg); graph = AAIGraph.getInstance().getGraph(); if( graph == null ){ @@ -152,18 +144,18 @@ public class GenTester { imsg = "-- Loading new schema elements into JanusGraph --"; System.out.println(imsg); - LOGGER.info(imsg); + LOGGER.debug(imsg); SchemaGenerator.loadSchemaIntoJanusGraph(graph, graphMgt, null); if( graph != null ){ imsg = "-- graph commit"; System.out.println(imsg); - LOGGER.info(imsg); + LOGGER.debug(imsg); graph.tx().commit(); imsg = "-- graph shutdown "; System.out.println(imsg); - LOGGER.info(imsg); + LOGGER.debug(imsg); graph.close(); } @@ -174,11 +166,10 @@ public class GenTester { - LOGGER.auditEvent("-- all done, if program does not exit, please kill."); + LOGGER.debug("-- all done, if program does not exit, please kill."); System.exit(0); } -} - +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/schema/GenTester4Hist.java b/src/main/java/org/onap/aai/schema/GenTester4Hist.java new file mode 100644 index 0000000..eefb7b0 --- /dev/null +++ b/src/main/java/org/onap/aai/schema/GenTester4Hist.java @@ -0,0 +1,175 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.schema; + +import com.att.eelf.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.schema.JanusGraphManagement; +import org.onap.aai.config.PropertyPasswordConfiguration; +import org.onap.aai.dbgen.SchemaGenerator4Hist; +import org.onap.aai.dbmap.AAIGraph; +import org.onap.aai.exceptions.AAIException; +import org.onap.aai.logging.ErrorLogHelper; +import org.onap.aai.util.*; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import java.util.Properties; +import java.util.UUID; + + +public class GenTester4Hist { + + private static Logger LOGGER; + private static boolean historyEnabled; + + /** + * The main method. + * + * @param args the arguments + */ + public static void main(String[] args) throws AAIException{ + + JanusGraph graph = null; + System.setProperty("aai.service.name", GenTester4Hist.class.getSimpleName()); + // Set the logging file properties to be used by EELFManager + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); + LOGGER = LoggerFactory.getLogger(GenTester4Hist.class); + boolean addDefaultCR = false; // For History, we do not add the default CloudRegion + + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); + PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); + initializer.initialize(ctx); + try { + ctx.scan( + "org.onap.aai.config", + "org.onap.aai.setup" + ); + ctx.refresh(); + } catch (Exception e) { + AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); + LOGGER.error("Problems running the tool "+aai.getMessage()); + ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); + throw aai; + } + + historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false")); + if( !historyEnabled ) { + String amsg = "GenTester4Hist may only be used when history.enabled=true. "; + System.out.println(amsg); + LOGGER.debug(amsg); + return; + } + + try { + LOGGER.debug("GenTester4Hist uses either cql jar or Cassandra jar"); + + AAIConfig.init(); + if (args != null && args.length > 0 ){ + if( "genDbRulesOnly".equals(args[0]) ){ + ErrorLogHelper.logError("AAI_3100", + " This option is no longer supported. What was in DbRules is now derived from the OXM files. "); + return; + } + else if ( "GEN_DB_WITH_NO_SCHEMA".equals(args[0]) ){ + // Note this is done to create an empty DB with no Schema so that + // an HBase copyTable can be used to set up a copy of the db. + String imsg = " ---- NOTE --- about to load a graph without doing any schema processing (takes a little while) -------- "; + System.out.println(imsg); + LOGGER.debug(imsg); + graph = AAIGraph.getInstance().getGraph(); + + if( graph == null ){ + ErrorLogHelper.logError("AAI_5102", "Error creating JanusGraph graph."); + return; + } + else { + String amsg = "Successfully loaded a JanusGraph graph without doing any schema work. "; + System.out.println(amsg); + LOGGER.debug(amsg); + return; + } + } else if ("GEN_DB_WITH_NO_DEFAULT_CR".equals(args[0])) { + addDefaultCR = false; + } + else { + ErrorLogHelper.logError("AAI_3000", "Unrecognized argument passed to GenTester4Hist.java: [" + args[0] + "]. "); + + String emsg = "Unrecognized argument passed to GenTester4Hist.java: [" + args[0] + "]. "; + System.out.println(emsg); + LOGGER.error(emsg); + + emsg = "Either pass no argument for normal processing, or use 'GEN_DB_WITH_NO_SCHEMA'."; + System.out.println(emsg); + LOGGER.error(emsg); + + return; + } + } + + //AAIConfig.init(); + ErrorLogHelper.loadProperties(); + String imsg = " ---- NOTE --- about to open graph (takes a little while)--------;"; + System.out.println(imsg); + LOGGER.debug(imsg); + graph = AAIGraph.getInstance().getGraph(); + + if( graph == null ){ + ErrorLogHelper.logError("AAI_5102", "Error creating JanusGraph graph. "); + return; + } + + GraphAdminDBUtils.logConfigs(graph.configuration()); + + // Load the propertyKeys, indexes and edge-Labels into the DB + JanusGraphManagement graphMgt = graph.openManagement(); + + imsg = "-- Loading new schema elements into JanusGraph --"; + System.out.println(imsg); + LOGGER.debug(imsg); + SchemaGenerator4Hist.loadSchemaIntoJanusGraph(graph, graphMgt, null); + + if( graph != null ){ + imsg = "-- graph commit"; + System.out.println(imsg); + LOGGER.debug(imsg); + graph.tx().commit(); + + imsg = "-- graph shutdown "; + System.out.println(imsg); + LOGGER.debug(imsg); + graph.close(); + } + + } catch(Exception ex) { + ErrorLogHelper.logError("AAI_4000", ex.getMessage()); + System.exit(1); + } + + LOGGER.debug("-- all done, if program does not exit, please kill."); + System.exit(0); + } + + + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/service/AuthorizationService.java b/src/main/java/org/onap/aai/service/AuthorizationService.java index d2597d0..9b5a614 100644 --- a/src/main/java/org/onap/aai/service/AuthorizationService.java +++ b/src/main/java/org/onap/aai/service/AuthorizationService.java @@ -19,8 +19,8 @@ */ package org.onap.aai.service; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.eclipse.jetty.util.security.Password; import org.onap.aai.Profiles; import org.onap.aai.util.AAIConstants; @@ -41,7 +41,7 @@ import java.util.stream.Stream; @Service public class AuthorizationService { - private static final EELFLogger logger = EELFManager.getInstance().getLogger(AuthorizationService.class); + private static final Logger logger = LoggerFactory.getLogger(AuthorizationService.class); private final Map<String, String> authorizedUsers = new HashMap<>(); @@ -72,7 +72,7 @@ public class AuthorizationService { String[] usernamePasswordArray = usernamePassword.split(":"); if(usernamePasswordArray == null || usernamePasswordArray.length != 3){ - throw new RuntimeException("Not a valid entry for the realm.properties entry: " + usernamePassword); + throw new RuntimeException("This username / pwd is not a valid entry in realm.properties"); } String username = usernamePasswordArray[0]; diff --git a/src/main/java/org/onap/aai/util/ExceptionTranslator.java b/src/main/java/org/onap/aai/util/ExceptionTranslator.java index a395c23..17c5667 100644 --- a/src/main/java/org/onap/aai/util/ExceptionTranslator.java +++ b/src/main/java/org/onap/aai/util/ExceptionTranslator.java @@ -19,27 +19,33 @@ */ package org.onap.aai.util; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.commons.lang3.exception.ExceptionUtils; import org.onap.aai.GraphAdminApp; import org.onap.aai.exceptions.AAIException; +import org.onap.aai.logging.LogFormatTools; public class ExceptionTranslator { - private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(ExceptionTranslator.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ExceptionTranslator.class); public static AAIException schemaServiceExceptionTranslator(Exception ex) { AAIException aai = null; - LOGGER.info("Exception is " + ExceptionUtils.getRootCause(ex).getMessage() + "Root cause is"+ ExceptionUtils.getRootCause(ex).toString()); - if(ExceptionUtils.getRootCause(ex).getMessage().contains("NodeIngestor")){ - aai = new AAIException("AAI_3026","Error reading OXM from SchemaService - Investigate"); - } - else if(ExceptionUtils.getRootCause(ex).getMessage().contains("EdgeIngestor")){ - aai = new AAIException("AAI_3027","Error reading EdgeRules from SchemaService - Investigate"); - } - else if(ExceptionUtils.getRootCause(ex).getMessage().contains("Connection refused")){ - aai = new AAIException("AAI_3025","Error connecting to SchemaService - Investigate"); - }else { - aai = new AAIException("AAI_3025","Error connecting to SchemaService - Please Investigate"); + if ( ExceptionUtils.getRootCause(ex) == null || ExceptionUtils.getRootCause(ex).getMessage() == null ) { + aai = new AAIException("AAI_3025","Error parsing exception - Please Investigate" + + LogFormatTools.getStackTop(ex)); + } else { + LOGGER.info("Exception is " + ExceptionUtils.getRootCause(ex).getMessage() + "Root cause is"+ ExceptionUtils.getRootCause(ex).toString()); + if(ExceptionUtils.getRootCause(ex).getMessage().contains("NodeIngestor")){ + aai = new AAIException("AAI_3026","Error reading OXM from SchemaService - Investigate"); + } + else if(ExceptionUtils.getRootCause(ex).getMessage().contains("EdgeIngestor")){ + aai = new AAIException("AAI_3027","Error reading EdgeRules from SchemaService - Investigate"); + } + else if(ExceptionUtils.getRootCause(ex).getMessage().contains("Connection refused")){ + aai = new AAIException("AAI_3025","Error connecting to SchemaService - Investigate"); + }else { + aai = new AAIException("AAI_3025","Error connecting to SchemaService - Please Investigate"); + } } return aai; diff --git a/src/main/java/org/onap/aai/util/GraphAdminConstants.java b/src/main/java/org/onap/aai/util/GraphAdminConstants.java index 017d92e..62ba593 100644 --- a/src/main/java/org/onap/aai/util/GraphAdminConstants.java +++ b/src/main/java/org/onap/aai/util/GraphAdminConstants.java @@ -22,6 +22,7 @@ package org.onap.aai.util; public final class GraphAdminConstants { public static final int AAI_SNAPSHOT_DEFAULT_THREADS_FOR_CREATE = 15; + public static final Long AAI_SNAPSHOT_DEFAULT_MAX_NODES_PER_FILE_FOR_CREATE = 120000L; public static final int AAI_SNAPSHOT_DEFAULT_MAX_ERRORS_PER_THREAD = 25; public static final Long AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS = 1L; public static final Long AAI_SNAPSHOT_DEFAULT_EDGE_ADD_DELAY_MS = 1L; @@ -36,7 +37,8 @@ public final class GraphAdminConstants { public static final int AAI_DUPETOOL_DEFAULT_MAX_FIX = 25; public static final int AAI_DUPETOOL_DEFAULT_SLEEP_MINUTES = 7; - + /** Micro-service Names */ + public static final String AAI_GRAPHADMIN_MS = "aai-graphadmin"; /** diff --git a/src/main/java/org/onap/aai/util/GraphAdminDBUtils.java b/src/main/java/org/onap/aai/util/GraphAdminDBUtils.java index 992223e..202bc0a 100644 --- a/src/main/java/org/onap/aai/util/GraphAdminDBUtils.java +++ b/src/main/java/org/onap/aai/util/GraphAdminDBUtils.java @@ -21,12 +21,12 @@ package org.onap.aai.util; import java.util.Iterator; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GraphAdminDBUtils { - private static EELFLogger LOGGER = EELFManager.getInstance().getLogger(GraphAdminDBUtils.class); + private static Logger LOGGER = LoggerFactory.getLogger(GraphAdminDBUtils.class); public static void logConfigs(org.apache.commons.configuration.Configuration configuration) { diff --git a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java index d9615b0..36d01e1 100644 --- a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java +++ b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java @@ -20,24 +20,22 @@ package org.onap.aai.util; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.onap.aai.db.props.AAIProperties; import org.onap.aai.dbmap.AAIGraph; -import org.onap.aai.dbmap.DBConnectionType; import org.onap.aai.exceptions.AAIException; -import org.onap.aai.introspection.*; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; import org.onap.aai.migration.EventAction; import org.onap.aai.migration.NotificationHelper; -import org.onap.aai.rest.ueb.UEBNotification; import org.onap.aai.serialization.db.DBSerializer; -import org.onap.aai.serialization.engines.QueryStyle; import org.onap.aai.serialization.engines.JanusGraphDBEngine; +import org.onap.aai.serialization.engines.QueryStyle; import org.onap.aai.serialization.engines.TransactionalGraphEngine; -import org.onap.aai.setup.SchemaVersions; import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.setup.SchemaVersions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.slf4j.MDC; import java.io.IOException; @@ -47,11 +45,9 @@ import java.nio.file.Paths; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; -import javax.ws.rs.core.Response.Status; - public class SendDeleteMigrationNotifications { - protected EELFLogger logger = EELFManager.getInstance().getLogger(SendDeleteMigrationNotifications.class.getSimpleName()); + protected Logger logger = LoggerFactory.getLogger(SendDeleteMigrationNotifications.class.getSimpleName()); private String config; private String path; @@ -104,7 +100,7 @@ public class SendDeleteMigrationNotifications { Map<Integer, String> deleteDataMap = processFile(); int count = 0; for (Map.Entry<Integer, String> entry : deleteDataMap.entrySet()) { - logger.info("Processing " + entry.getKey() + " :: Data :: " + entry.getValue()); + logger.debug("Processing " + entry.getKey() + " :: Data :: " + entry.getValue()); String data = entry.getValue(); Introspector obj = null; if (data.contains("#@#")) { @@ -118,7 +114,7 @@ public class SendDeleteMigrationNotifications { count++; if (count >= this.numToBatch) { trigger(); - logger.info("Triggered " + entry.getKey()); + logger.debug("Triggered " + entry.getKey()); count = 0; Thread.sleep(this.sleepInMilliSecs); } @@ -158,7 +154,7 @@ public class SendDeleteMigrationNotifications { private void initFields() { this.loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); - this.engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader); + this.engine = new JanusGraphDBEngine(queryStyle, loader); try { this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.eventSource); } catch (AAIException e) { @@ -176,7 +172,7 @@ public class SendDeleteMigrationNotifications { protected void logAndPrint(String msg) { System.out.println(msg); - logger.info(msg); + logger.debug(msg); } diff --git a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java index a94b435..0fbe520 100644 --- a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java +++ b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java @@ -26,7 +26,6 @@ import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.logging.ErrorLogHelper; -import org.onap.aai.logging.LoggingContext; import org.onap.aai.migration.EventAction; import org.onap.aai.setup.SchemaVersions; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @@ -40,15 +39,6 @@ public class SendDeleteMigrationNotificationsMain { Arrays.asList(args).stream().forEach(System.out::println); String requestId = UUID.randomUUID().toString(); - LoggingContext.init(); - LoggingContext.partnerName("Migration"); - LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.component("SendMigrationNotifications"); - LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.targetServiceName("main"); - LoggingContext.requestId(requestId); - LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); @@ -61,8 +51,6 @@ public class SendDeleteMigrationNotificationsMain { ctx.refresh(); } catch (Exception e) { AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); - LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } diff --git a/src/main/java/org/onap/aai/util/SendMigrationNotifications.java b/src/main/java/org/onap/aai/util/SendMigrationNotifications.java index 577f577..c51de1e 100644 --- a/src/main/java/org/onap/aai/util/SendMigrationNotifications.java +++ b/src/main/java/org/onap/aai/util/SendMigrationNotifications.java @@ -20,23 +20,25 @@ package org.onap.aai.util; import com.att.eelf.configuration.Configuration; -import com.att.eelf.configuration.EELFLogger; -import com.att.eelf.configuration.EELFManager; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.onap.aai.db.props.AAIProperties; import org.onap.aai.dbmap.AAIGraph; -import org.onap.aai.dbmap.DBConnectionType; import org.onap.aai.exceptions.AAIException; -import org.onap.aai.introspection.*; +import org.onap.aai.introspection.Introspector; +import org.onap.aai.introspection.Loader; +import org.onap.aai.introspection.LoaderFactory; +import org.onap.aai.introspection.ModelType; import org.onap.aai.migration.EventAction; import org.onap.aai.migration.NotificationHelper; import org.onap.aai.serialization.db.DBSerializer; -import org.onap.aai.serialization.engines.QueryStyle; import org.onap.aai.serialization.engines.JanusGraphDBEngine; +import org.onap.aai.serialization.engines.QueryStyle; import org.onap.aai.serialization.engines.TransactionalGraphEngine; -import org.onap.aai.setup.SchemaVersions; import org.onap.aai.setup.SchemaVersion; +import org.onap.aai.setup.SchemaVersions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.slf4j.MDC; import java.io.IOException; @@ -47,7 +49,7 @@ import java.util.*; public class SendMigrationNotifications { - protected EELFLogger logger = EELFManager.getInstance().getLogger(SendMigrationNotifications.class.getSimpleName()); + protected Logger logger = LoggerFactory.getLogger(SendMigrationNotifications.class.getSimpleName()); private String config; private String path; @@ -111,7 +113,7 @@ public class SendMigrationNotifications { logAndPrint("Vertex " + entry.getKey() + " query returned " + vertexes.size() + " vertexes." ); continue; } else { - logger.info("Processing " + entry.getKey() + "resource-version " + entry.getValue()); + logger.debug("Processing " + entry.getKey() + "resource-version " + entry.getValue()); v = vertexes.get(0); if (notifyOn.isEmpty() || notifyOn.contains(v.value(AAIProperties.NODE_TYPE).toString())) { if (entry.getValue().equals(v.value(AAIProperties.RESOURCE_VERSION).toString())) { @@ -121,7 +123,7 @@ public class SendMigrationNotifications { count++; if (count >= this.numToBatch) { trigger(); - logger.info("Triggered " + entry.getKey()); + logger.debug("Triggered " + entry.getKey()); count = 0; Thread.sleep(this.sleepInMilliSecs); } @@ -164,7 +166,7 @@ public class SendMigrationNotifications { private void initFields() { this.loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version); - this.engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader); + this.engine = new JanusGraphDBEngine(queryStyle, loader); try { this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.eventSource); } catch (AAIException e) { @@ -182,7 +184,7 @@ public class SendMigrationNotifications { protected void logAndPrint(String msg) { System.out.println(msg); - logger.info(msg); + logger.debug(msg); } diff --git a/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java b/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java index 17a127a..d3670f2 100644 --- a/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java +++ b/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java @@ -26,7 +26,6 @@ import org.onap.aai.dbmap.AAIGraph; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.LoaderFactory; import org.onap.aai.logging.ErrorLogHelper; -import org.onap.aai.logging.LoggingContext; import org.onap.aai.migration.EventAction; import org.onap.aai.setup.SchemaVersions; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @@ -40,16 +39,7 @@ public class SendMigrationNotificationsMain { Arrays.asList(args).stream().forEach(System.out::println); String requestId = UUID.randomUUID().toString(); - LoggingContext.init(); - LoggingContext.partnerName("Migration"); - LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.component("SendMigrationNotifications"); - LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS); - LoggingContext.targetServiceName("main"); - LoggingContext.requestId(requestId); - LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE); - LoggingContext.responseCode(LoggingContext.SUCCESS); - + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration(); initializer.initialize(ctx); @@ -62,8 +52,6 @@ public class SendMigrationNotificationsMain { } catch (Exception e) { AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e); System.out.println("Problems running tool "+aai.getMessage()); - LoggingContext.statusCode(LoggingContext.StatusCode.ERROR); - LoggingContext.responseCode(LoggingContext.DATA_ERROR); ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry"); throw aai; } diff --git a/src/main/java/org/onap/aai/util/UniquePropertyCheck.java b/src/main/java/org/onap/aai/util/UniquePropertyCheck.java index e96c252..2db3dd5 100644 --- a/src/main/java/org/onap/aai/util/UniquePropertyCheck.java +++ b/src/main/java/org/onap/aai/util/UniquePropertyCheck.java @@ -1,288 +1,277 @@ -/**
- * ============LICENSE_START=======================================================
- * org.onap.aai
- * ================================================================================
- * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END=========================================================
- */
-package org.onap.aai.util;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.onap.aai.GraphAdminApp;
-import org.onap.aai.exceptions.AAIException;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.logging.LoggingContext.StatusCode;
-import org.slf4j.MDC;
-
-import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-
-import org.janusgraph.core.JanusGraphFactory;
-import org.janusgraph.core.JanusGraph;
-import org.onap.aai.dbmap.AAIGraphConfig;
-
-public class UniquePropertyCheck {
-
-
- private static final String FROMAPPID = "AAI-UTILS";
- private static final String TRANSID = UUID.randomUUID().toString();
- private static final String COMPONENT = "UniquePropertyCheck";
-
- /**
- * The main method.
- *
- * @param args the arguments
- */
- public static void main(String[] args) {
-
-
- Properties props = System.getProperties();
- props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
- props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
- EELFLogger logger = EELFManager.getInstance().getLogger(UniquePropertyCheck.class.getSimpleName());
-
- LoggingContext.init();
- LoggingContext.partnerName(FROMAPPID);
- LoggingContext.serviceName(GraphAdminApp.APP_NAME);
- LoggingContext.component(COMPONENT);
- LoggingContext.targetEntity(GraphAdminApp.APP_NAME);
- LoggingContext.targetServiceName("main");
- LoggingContext.requestId(TRANSID);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
-
- MDC.put("logFilenameAppender", UniquePropertyCheck.class.getSimpleName());
-
- if( args == null || args.length != 1 ){
- String msg = "usage: UniquePropertyCheck propertyName \n";
- System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
- logAndPrint(logger, msg );
- System.exit(1);
- }
- String propertyName = args[0];
- Graph graph = null;
-
- try {
- AAIConfig.init();
- System.out.println(" ---- NOTE --- about to open graph (takes a little while)--------\n");
- JanusGraph tGraph = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(UniquePropertyCheck.class.getSimpleName()).withGraphType("realtime").buildConfiguration());
-
- if( tGraph == null ) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
- logAndPrint(logger, " Error: Could not get JanusGraph ");
- System.exit(1);
- }
-
- graph = tGraph.newTransaction();
- if( graph == null ){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
- logAndPrint(logger, "could not get graph object in UniquePropertyCheck() \n");
- System.exit(0);
- }
- }
- catch (AAIException e1) {
- String msg = "Threw Exception: [" + e1.toString() + "]";
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
- logAndPrint(logger, msg);
- System.exit(0);
- }
- catch (Exception e2) {
- String msg = "Threw Exception: [" + e2.toString() + "]";
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
- logAndPrint(logger, msg);
- System.exit(0);
- }
-
- runTheCheckForUniqueness( TRANSID, FROMAPPID, graph, propertyName, logger );
- System.exit(0);
-
- }// End main()
-
-
- /**
- * Run the check for uniqueness.
- *
- * @param transId the trans id
- * @param fromAppId the from app id
- * @param graph the graph
- * @param propertyName the property name
- * @param logger the logger
- * @return the boolean
- */
- public static Boolean runTheCheckForUniqueness( String transId, String fromAppId, Graph graph,
- String propertyName, EELFLogger logger ){
-
- // Note - property can be found in more than one nodetype
- // our uniqueness constraints are always across the entire db - so this
- // tool looks across all nodeTypes that the property is found in.
- Boolean foundDupesFlag = false;
-
- HashMap <String,String> valuesAndVidHash = new HashMap <String, String> ();
- HashMap <String,String> dupeHash = new HashMap <String, String> ();
-
- int propCount = 0;
- int dupeCount = 0;
- Iterator<Vertex> vertItor = graph.traversal().V().has(propertyName);
- while( vertItor.hasNext() ){
- propCount++;
- Vertex v = vertItor.next();
- String thisVid = v.id().toString();
- Object val = (v.<Object>property(propertyName)).orElse(null);
- if( valuesAndVidHash.containsKey(val) ){
- // We've seen this one before- track it in our dupe hash
- dupeCount++;
- if( dupeHash.containsKey(val) ){
- // This is not the first one being added to the dupe hash for this value
- String updatedDupeList = dupeHash.get(val) + "|" + thisVid;
- dupeHash.put(val.toString(), updatedDupeList);
- }
- else {
- // This is the first time we see this value repeating
- String firstTwoVids = valuesAndVidHash.get(val) + "|" + thisVid;
- dupeHash.put(val.toString(), firstTwoVids);
- }
- }
- else {
- valuesAndVidHash.put(val.toString(), thisVid);
- }
- }
-
-
- String info = "\n Found this property [" + propertyName + "] " + propCount + " times in our db.";
- logAndPrint(logger, info);
- info = " Found " + dupeCount + " cases of duplicate values for this property.\n\n";
- logAndPrint(logger, info);
-
- try {
- if( ! dupeHash.isEmpty() ){
- Iterator <?> dupeItr = dupeHash.entrySet().iterator();
- while( dupeItr.hasNext() ){
- foundDupesFlag = true;
- Map.Entry pair = (Map.Entry) dupeItr.next();
- String dupeValue = pair.getKey().toString();;
- String vidsStr = pair.getValue().toString();
- String[] vidArr = vidsStr.split("\\|");
- logAndPrint(logger, "\n\n -------------- Found " + vidArr.length
- + " nodes with " + propertyName + " of this value: [" + dupeValue + "]. Node details: ");
-
- for( int i = 0; i < vidArr.length; i++ ){
- String vidString = vidArr[i];
- Long idLong = Long.valueOf(vidString);
- Vertex tvx = graph.traversal().V(idLong).next();
- showPropertiesAndEdges( TRANSID, FROMAPPID, tvx, logger );
- }
- }
- }
- }
- catch( Exception e2 ){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- logAndPrint(logger, "Threw Exception: [" + e2.toString() + "]");
- }
-
-
- return foundDupesFlag;
-
- }// end of runTheCheckForUniqueness()
-
-
- /**
- * Show properties and edges.
- *
- * @param transId the trans id
- * @param fromAppId the from app id
- * @param tVert the t vert
- * @param logger the logger
- */
- private static void showPropertiesAndEdges( String transId, String fromAppId, Vertex tVert,
- EELFLogger logger ){
-
- if( tVert == null ){
- logAndPrint(logger, "Null node passed to showPropertiesAndEdges.");
- }
- else {
- String nodeType = "";
- Object ob = tVert.<String>property("aai-node-type").orElse(null);
- if( ob == null ){
- nodeType = "null";
- }
- else{
- nodeType = ob.toString();
- }
-
- logAndPrint(logger, " AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
- logAndPrint(logger, " Property Detail: ");
- Iterator<VertexProperty<Object>> pI = tVert.properties();
- while( pI.hasNext() ){
- VertexProperty<Object> tp = pI.next();
- Object val = tp.value();
- logAndPrint(logger, "Prop: [" + tp.key() + "], val = [" + val + "] ");
- }
-
- Iterator <Edge> eI = tVert.edges(Direction.BOTH);
- if( ! eI.hasNext() ){
- logAndPrint(logger, "No edges were found for this vertex. ");
- }
- while( eI.hasNext() ){
- Edge ed = eI.next();
- String lab = ed.label();
- Vertex vtx;
- if (tVert.equals(ed.inVertex())) {
- vtx = ed.outVertex();
- } else {
- vtx = ed.inVertex();
- }
- if( vtx == null ){
- logAndPrint(logger, " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
- }
- else {
- String nType = vtx.<String>property("aai-node-type").orElse(null);
- String vid = vtx.id().toString();
- logAndPrint(logger, "Found an edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid);
- }
- }
- }
- } // End of showPropertiesAndEdges()
-
-
- /**
- * Log and print.
- *
- * @param logger the logger
- * @param msg the msg
- */
- protected static void logAndPrint(EELFLogger logger, String msg) {
- System.out.println(msg);
- logger.info(msg);
- }
-
-}
-
+/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.util; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; + +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.JanusGraphFactory; +import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog; +import org.onap.aai.dbmap.AAIGraphConfig; +import org.onap.aai.exceptions.AAIException; +import org.onap.logging.filter.base.ONAPComponents; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import com.att.eelf.configuration.Configuration; + +public class UniquePropertyCheck { + + + private static final String FROMAPPID = "AAI-UTILS"; + private static final String TRANSID = UUID.randomUUID().toString(); + private static final String COMPONENT = "UniquePropertyCheck"; + + /** + * The main method. + * + * @param args the arguments + */ + public static void main(String[] args) { + + + Properties props = System.getProperties(); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS); + props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG); + Logger logger = LoggerFactory.getLogger(UniquePropertyCheck.class); + MDC.put("logFilenameAppender", UniquePropertyCheck.class.getSimpleName()); + AaiScheduledTaskAuditLog auditLog = new AaiScheduledTaskAuditLog(); + auditLog.logBefore("UniquePropertyCheck", ONAPComponents.AAI.toString()); + + if( args == null || args.length != 1 ){ + String msg = "usage: UniquePropertyCheck propertyName \n"; + System.out.println(msg); + //LoggingContext.statusCode(StatusCode.ERROR); + //LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR); + logAndPrint(logger, msg ); + System.exit(1); + } + String propertyName = args[0]; + Graph graph = null; + + try { + AAIConfig.init(); + System.out.println(" ---- NOTE --- about to open graph (takes a little while)--------\n"); + JanusGraph tGraph = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(UniquePropertyCheck.class.getSimpleName()).withGraphType("realtime").buildConfiguration()); + + if( tGraph == null ) { + //LoggingContext.statusCode(StatusCode.ERROR); + //LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); + logAndPrint(logger, " Error: Could not get JanusGraph "); + System.exit(1); + } + + graph = tGraph.newTransaction(); + if( graph == null ){ + //LoggingContext.statusCode(StatusCode.ERROR); + //LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR); + logAndPrint(logger, "could not get graph object in UniquePropertyCheck() \n"); + System.exit(0); + } + } + catch (AAIException e1) { + String msg = "Threw Exception: [" + e1.toString() + "]"; + //LoggingContext.statusCode(StatusCode.ERROR); + //LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); + logAndPrint(logger, msg); + System.exit(0); + } + catch (Exception e2) { + String msg = "Threw Exception: [" + e2.toString() + "]"; + //LoggingContext.statusCode(StatusCode.ERROR); + //LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR); + logAndPrint(logger, msg); + System.exit(0); + } + + runTheCheckForUniqueness( TRANSID, FROMAPPID, graph, propertyName, logger ); + auditLog.logAfter(); + System.exit(0); + + }// End main() + + + /** + * Run the check for uniqueness. + * + * @param transId the trans id + * @param fromAppId the from app id + * @param graph the graph + * @param propertyName the property name + * @param logger the logger + * @return the boolean + */ + public static Boolean runTheCheckForUniqueness( String transId, String fromAppId, Graph graph, + String propertyName, Logger logger ){ + + // Note - property can be found in more than one nodetype + // our uniqueness constraints are always across the entire db - so this + // tool looks across all nodeTypes that the property is found in. + Boolean foundDupesFlag = false; + + HashMap <String,String> valuesAndVidHash = new HashMap <String, String> (); + HashMap <String,String> dupeHash = new HashMap <String, String> (); + + int propCount = 0; + int dupeCount = 0; + Iterator<Vertex> vertItor = graph.traversal().V().has(propertyName); + while( vertItor.hasNext() ){ + propCount++; + Vertex v = vertItor.next(); + String thisVid = v.id().toString(); + Object val = (v.<Object>property(propertyName)).orElse(null); + if( valuesAndVidHash.containsKey(val) ){ + // We've seen this one before- track it in our dupe hash + dupeCount++; + if( dupeHash.containsKey(val) ){ + // This is not the first one being added to the dupe hash for this value + String updatedDupeList = dupeHash.get(val) + "|" + thisVid; + dupeHash.put(val.toString(), updatedDupeList); + } + else { + // This is the first time we see this value repeating + String firstTwoVids = valuesAndVidHash.get(val) + "|" + thisVid; + dupeHash.put(val.toString(), firstTwoVids); + } + } + else { + valuesAndVidHash.put(val.toString(), thisVid); + } + } + + + String info = "\n Found this property [" + propertyName + "] " + propCount + " times in our db."; + logAndPrint(logger, info); + info = " Found " + dupeCount + " cases of duplicate values for this property.\n\n"; + logAndPrint(logger, info); + + try { + if( ! dupeHash.isEmpty() ){ + Iterator <?> dupeItr = dupeHash.entrySet().iterator(); + while( dupeItr.hasNext() ){ + foundDupesFlag = true; + Map.Entry pair = (Map.Entry) dupeItr.next(); + String dupeValue = pair.getKey().toString();; + String vidsStr = pair.getValue().toString(); + String[] vidArr = vidsStr.split("\\|"); + logAndPrint(logger, "\n\n -------------- Found " + vidArr.length + + " nodes with " + propertyName + " of this value: [" + dupeValue + "]. Node details: "); + + for( int i = 0; i < vidArr.length; i++ ){ + String vidString = vidArr[i]; + Long idLong = Long.valueOf(vidString); + Vertex tvx = graph.traversal().V(idLong).next(); + showPropertiesAndEdges( TRANSID, FROMAPPID, tvx, logger ); + } + } + } + } + catch( Exception e2 ){ + //LoggingContext.statusCode(StatusCode.ERROR); + //LoggingContext.responseCode(LoggingContext.DATA_ERROR); + logAndPrint(logger, "Threw Exception: [" + e2.toString() + "]"); + } + + + return foundDupesFlag; + + }// end of runTheCheckForUniqueness() + + + /** + * Show properties and edges. + * + * @param transId the trans id + * @param fromAppId the from app id + * @param tVert the t vert + * @param logger the logger + */ + private static void showPropertiesAndEdges( String transId, String fromAppId, Vertex tVert, + Logger logger ){ + + if( tVert == null ){ + logAndPrint(logger, "Null node passed to showPropertiesAndEdges."); + } + else { + String nodeType = ""; + Object ob = tVert.<String>property("aai-node-type").orElse(null); + if( ob == null ){ + nodeType = "null"; + } + else{ + nodeType = ob.toString(); + } + + logAndPrint(logger, " AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]"); + logAndPrint(logger, " Property Detail: "); + Iterator<VertexProperty<Object>> pI = tVert.properties(); + while( pI.hasNext() ){ + VertexProperty<Object> tp = pI.next(); + Object val = tp.value(); + logAndPrint(logger, "Prop: [" + tp.key() + "], val = [" + val + "] "); + } + + Iterator <Edge> eI = tVert.edges(Direction.BOTH); + if( ! eI.hasNext() ){ + logAndPrint(logger, "No edges were found for this vertex. "); + } + while( eI.hasNext() ){ + Edge ed = eI.next(); + String lab = ed.label(); + Vertex vtx; + if (tVert.equals(ed.inVertex())) { + vtx = ed.outVertex(); + } else { + vtx = ed.inVertex(); + } + if( vtx == null ){ + logAndPrint(logger, " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< "); + } + else { + String nType = vtx.<String>property("aai-node-type").orElse(null); + String vid = vtx.id().toString(); + logAndPrint(logger, "Found an edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid); + } + } + } + } // End of showPropertiesAndEdges() + + + /** + * Log and print. + * + * @param logger the logger + * @param msg the msg + */ + protected static void logAndPrint(Logger logger, String msg) { + System.out.println(msg); + logger.info(msg); + } + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/web/JerseyConfiguration.java b/src/main/java/org/onap/aai/web/JerseyConfiguration.java index 436946c..629d11e 100644 --- a/src/main/java/org/onap/aai/web/JerseyConfiguration.java +++ b/src/main/java/org/onap/aai/web/JerseyConfiguration.java @@ -19,25 +19,28 @@ */ package org.onap.aai.web; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import javax.annotation.Priority; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.container.ContainerResponseFilter; + +import org.apache.tinkerpop.gremlin.structure.T; import org.glassfish.jersey.filter.LoggingFilter; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.servlet.ServletProperties; import org.onap.aai.rest.QueryConsumer; import org.onap.aai.rest.util.EchoResponse; +import org.onap.logging.filter.base.AuditLogContainerFilter; import org.reflections.Reflections; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Profile; import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; -import javax.annotation.Priority; -import javax.ws.rs.container.ContainerRequestFilter; -import javax.ws.rs.container.ContainerResponseFilter; -import java.util.List; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - @Component public class JerseyConfiguration extends ResourceConfig { @@ -54,11 +57,11 @@ public class JerseyConfiguration extends ResourceConfig { register(EchoResponse.class); - //Request Filters - registerFiltersForRequests(); - // Response Filters - registerFiltersForResponses(); - + //Filters + registerFilters(ContainerRequestFilter.class); + registerFilters(ContainerResponseFilter.class); + registerFilters(AuditLogContainerFilter.class); + property(ServletProperties.FILTER_FORWARD_ON_404, true); // Following registers the request headers and response headers @@ -68,13 +71,14 @@ public class JerseyConfiguration extends ResourceConfig { } } - public void registerFiltersForRequests() { + public <T> void registerFilters(Class<T> type) { // Find all the classes within the interceptors package - Reflections reflections = new Reflections("org.onap.aai.interceptors"); - // Filter them based on the clazz that was passed in - Set<Class<? extends ContainerRequestFilter>> filters = reflections.getSubTypesOf(ContainerRequestFilter.class); - + Reflections loggingReflections = new Reflections("org.onap.aai.aailog.filter"); + Reflections reflections = new Reflections("org.onap.aai.interceptors"); + // Filter them based on the clazz that was passed in + Set<Class<? extends T>> filters = loggingReflections.getSubTypesOf(type); + filters.addAll(reflections.getSubTypesOf(type)); // Check to ensure that each of the filter has the @Priority annotation and if not throw exception for (Class filterClass : filters) { @@ -84,7 +88,7 @@ public class JerseyConfiguration extends ResourceConfig { } // Turn the set back into a list - List<Class<? extends ContainerRequestFilter>> filtersList = filters + List<Class<? extends T>> filtersList = filters .stream() .filter(f -> { if (f.isAnnotationPresent(Profile.class) @@ -101,37 +105,4 @@ public class JerseyConfiguration extends ResourceConfig { // Then register this to the jersey application filtersList.forEach(this::register); } - - public void registerFiltersForResponses() { - - // Find all the classes within the interceptors package - Reflections reflections = new Reflections("org.onap.aai.interceptors"); - // Filter them based on the clazz that was passed in - Set<Class<? extends ContainerResponseFilter>> filters = reflections.getSubTypesOf(ContainerResponseFilter.class); - - - // Check to ensure that each of the filter has the @Priority annotation and if not throw exception - for (Class filterClass : filters) { - if (filterClass.getAnnotation(Priority.class) == null) { - throw new RuntimeException("Container filter " + filterClass.getName() + " does not have @Priority annotation"); - } - } - - // Turn the set back into a list - List<Class<? extends ContainerResponseFilter>> filtersList = filters.stream() - .filter(f -> { - if (f.isAnnotationPresent(Profile.class) - && !env.acceptsProfiles(f.getAnnotation(Profile.class).value())) { - return false; - } - return true; - }) - .collect(Collectors.toList()); - - // Sort them by their priority levels value - filtersList.sort((c1, c2) -> Integer.valueOf(c1.getAnnotation(Priority.class).value()).compareTo(c2.getAnnotation(Priority.class).value())); - - // Then register this to the jersey application - filtersList.forEach(this::register); - } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index ce036d9..698a6da 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -48,7 +48,7 @@ schema.ingest.file=${server.local.startpath}/application.properties schema.uri.base.path=/aai # Lists all of the versions in the schema -schema.version.list=v10,v11,v12,v13,v14,v15,v16 +schema.version.list=v10,v11,v12,v13,v14,v15,v16,v17,v18,v19 # Specifies from which version should the depth parameter to default to zero schema.version.depth.start=v10 # Specifies from which version should the related link be displayed in response payload @@ -73,3 +73,5 @@ schema.service.ssl.key-store=${server.local.startpath}etc/auth/aai_keystore schema.service.ssl.key-store-password=password(OBF:1vn21ugu1saj1v9i1v941sar1ugw1vo0) schema.service.ssl.trust-store=${server.local.startpath}etc/auth/aai_keystore schema.service.ssl.trust-store-password=password(OBF:1vn21ugu1saj1v9i1v941sar1ugw1vo0) + +aaf.cadi.file=${server.local.startpath}/cadi.properties
\ No newline at end of file diff --git a/src/main/resources/etc/appprops/aaiconfig.properties b/src/main/resources/etc/appprops/aaiconfig.properties index 41ed4fb..ae5084f 100644 --- a/src/main/resources/etc/appprops/aaiconfig.properties +++ b/src/main/resources/etc/appprops/aaiconfig.properties @@ -72,6 +72,7 @@ aai.grooming.default.sleep.minutes=7 # Used by Data Snapshot aai.datasnapshot.default.threads.for.create=16 +aai.datasnapshot.max.nodes.per.file.for.create=120000 # Used by DupeTool aai.dupeTool.default.max.fix=25 @@ -146,5 +147,14 @@ aai.disable.check.grooming.running=true # THREADED_SNAPSHOT 2 aai.datasnapshot.params=JUST_TAKE_SNAPSHOT +#Data export task properties +aai.dataexport.enable=false +aai.dataexport.enable.schema.validation=false +aai.dataexport.output.location=/etc/scriptdata/addmanualdata/tenant_isolation/payload +aai.dataexport.enable.multiple.snapshots=false +aai.dataexport.node.config.location=/etc/scriptdata/tenant_isolation/nodes.json +aai.dataexport.input.filter.config.location=/etc/scriptdata/tenant_isolation/inputFilters.json +aai.dataexport.enable.partial.graph=false + # Threshold for margin of error (in ms) for resources_with_sot format to derive the most recent http method performed aai.resource.formatter.threshold=10 diff --git a/src/main/resources/etc/appprops/error.properties b/src/main/resources/etc/appprops/error.properties index 1550d6c..54f00b7 100644 --- a/src/main/resources/etc/appprops/error.properties +++ b/src/main/resources/etc/appprops/error.properties @@ -3,180 +3,186 @@ #Key=Disposition:Category:Severity:Error Code:HTTP ResponseCode:RESTError Code:Error Message #------------------------------------------------------------------------------- ---------- # testing code, please don't change unless error utility source code changes -AAI_TESTING=5:2:WARN:0000:400:0001:Error code for testing +AAI_TESTING=5:2:WARN:0000:400:0001:Error code for testing:0 # General success -AAI_0000=0:0:INFO:0000:200:0000:Success +AAI_0000=0:0:INFO:0000:200:0000:Success:0 # health check success -AAI_0001=0:0:INFO:0001:200:0001:Success X-FromAppId=%1 X-TransactionId=%2 -AAI_0002=0:0:INFO:0002:200:0001:Successful health check +AAI_0001=0:0:INFO:0001:200:0001:Success X-FromAppId=%1 X-TransactionId=%2:0 +AAI_0002=0:0:INFO:0002:200:0001:Successful health check:0 # Success with additional info -AAI_0003=0:3:INFO:0003:202:0003:Success with additional info performing %1 on %2. Added %3 with key %4 -AAI_0004=0:3:INFO:0004:202:0003:Added prerequisite object to db +AAI_0003=0:3:INFO:0003:202:0003:Success with additional info performing %1 on %2. Added %3 with key %4:0 +AAI_0004=0:3:INFO:0004:202:0003:Added prerequisite object to db:0 #--- aairest: 3000-3299 # svc errors -AAI_3000=5:2:INFO:3000:400:3000:Invalid input performing %1 on %2 -AAI_3001=5:6:INFO:3001:404:3001:Resource not found for %1 using id %2 -AAI_3002=5:1:WARN:3002:400:3002:Error writing output performing %1 on %2 -AAI_3003=5:1:WARN:3003:400:3003:Failed to make edge to missing target node of type %3 with keys %4 performing %1 on %2 -AAI_3005=5:6:WARN:3005:404:3001:Node cannot be directly accessed for read, must be accessed via ancestor(s) -AAI_3006=5:6:WARN:3006:404:3001:Node cannot be directly accessed for write, must be accessed via ancestor(s) -AAI_3007=5:6:INFO:3007:410:3007:This version (%1) of the API is retired, please migrate to %2 -AAI_3008=5:6:WARN:3008:400:3008:URI is not encoded in UTF-8 -AAI_3009=5:6:WARN:3009:400:3002:Malformed URL -AAI_3010=5:6:WARN:3010:400:3002:Cannot write via this URL -AAI_3011=5:6:WARN:3011:400:3000:Unknown XML namespace used in payload -AAI_3012=5:6:WARN:3012:400:3012:Unrecognized AAI function -AAI_3013=5:6:WARN:3013:400:3013:Query payload missing required parameters %1 -AAI_3014=5:6:WARN:3014:400:3014:Query payload is invalid %1 -AAI_3025=5:4:FATAL:3025:500:3025:Error connecting to Schema Service - Investigate -AAI_3026=5:4:FATAL:3026:500:3026:Error reading OXM from Schema Service - Investigate -AAI_3027=5:4:FATAL:3026:500:3026:Error reading EdgeRules from Schema Service - Investigate +AAI_3000=5:2:INFO:3000:400:3000:Invalid input performing %1 on %2:300 +AAI_3001=5:6:INFO:3001:404:3001:Resource not found for %1 using id %2:300 +AAI_3002=5:1:WARN:3002:400:3002:Error writing output performing %1 on %2:300 +AAI_3003=5:1:WARN:3003:400:3003:Failed to make edge to missing target node of type %3 with keys %4 performing %1 on %2:400 +AAI_3005=5:6:WARN:3005:404:3001:Node cannot be directly accessed for read, must be accessed via ancestor(s):100 +AAI_3006=5:6:WARN:3006:404:3001:Node cannot be directly accessed for write, must be accessed via ancestor(s):100 +AAI_3007=5:6:INFO:3007:410:3007:This version (%1) of the API is retired, please migrate to %2:500 +AAI_3008=5:6:ERROR:3008:400:3008:URI is not encoded in UTF-8:300 +AAI_3009=5:6:WARN:3009:400:3002:Malformed URL:300 +AAI_3010=5:6:WARN:3010:400:3002:Cannot write via this URL:100 +AAI_3011=5:6:WARN:3011:400:3000:Unknown XML namespace used in payload:300 +AAI_3012=5:6:WARN:3012:400:3012:Unrecognized AAI function:300 +AAI_3013=5:6:WARN:3013:400:3013:Query payload missing required parameters %1:300 +AAI_3014=5:6:WARN:3014:400:3014:Query payload is invalid %1:300 +AAI_3015=5:6:INFO:3015:410:3015:The %1 capability is retired, please contact the A&AI SE team to identify a replacement query:500 +AAI_3016=5:6:INFO:3007:400:3016:Request uri is not valid, please check the version %1:500 +AAI_3017=5:6:INFO:3007:400:3016:Request uri is not valid, please check the uri %1:500 +AAI_3025=5:4:FATAL:3025:500:3025:Error connecting to Schema Service:400 +AAI_3026=5:4:FATAL:3026:500:3026:Error reading OXM from Schema Service:400 +AAI_3027=5:4:FATAL:3026:500:3026:Error reading EdgeRules from Schema Service:400 # pol errors -AAI_3100=5:1:WARN:3100:400:3100:Unsupported operation %1 -AAI_3101=5:1:WARN:3101:403:3101:Attempt by client %1 to execute API %2 -AAI_3102=5:1:WARN:3102:400:3102:Error parsing input performing %1 on %2 -AAI_3300=5:1:WARN:3300:403:3300:Unauthorized -AAI_3301=5:1:WARN:3301:401:3301:Stale credentials -AAI_3302=5:1:WARN:3302:401:3301:Not authenticated -AAI_3303=5:1:WARN:3303:403:3300:Too many objects would be returned by this request, please refine your request and retry +AAI_3100=5:1:WARN:3100:400:3100:Unsupported operation %1:300 +AAI_3101=5:1:WARN:3101:403:3101:Attempt by client %1 to execute API %2:100 +AAI_3102=5:1:WARN:3102:400:3102:Error parsing input performing %1 on %2:300 +AAI_3300=5:1:WARN:3300:403:3300:Unauthorized:100 +AAI_3301=5:1:WARN:3301:401:3301:Stale credentials:100 +AAI_3302=5:1:WARN:3302:401:3301:Not authenticated:100 +AAI_3303=5:1:WARN:3303:403:3300:Too many objects would be returned by this request, please refine your request and retry:500 #--- aaigen: 4000-4099 -AAI_4000=5:4:ERROR:4000:500:3002:Internal Error -AAI_4001=5:4:FATAL:4001:500:3002:Configuration file not found -AAI_4002=5:4:FATAL:4002:500:3002:Error reading Configuration file -AAI_4003=5:4:ERROR:4003:500:3002:Error writing to log file -AAI_4004=5:4:FATAL:4004:500:3002:Error reading/parsing the error properties file -AAI_4005=5:4:FATAL:4005:500:3002:Missing or invalid configuration parameter -AAI_4006=5:4:FATAL:4006:500:3002:Unexpected error in service -AAI_4007=5:4:WARN:4007:500:3102:Input parsing error -AAI_4008=5:4:ERROR:4008:500:3002:Output parsing error -AAI_4009=4:0:WARN:4009:400:3000:Invalid X-FromAppId in header -AAI_4010=4:0:WARN:4010:400:3000:Invalid X-TransactionId in header -AAI_4011=5:4:ERROR:4011:500:3002:Missing data for REST error response -AAI_4014=4:0:WARN:4014:400:3000:Invalid Accept header -AAI_4015=4:0:WARN:4015:400:3000:You must provide at least one indexed property -AAI_4016=4:0:WARN:4016:400:3000:The depth parameter must be a number or the string "all" -AAI_4017=5:2:INFO:4017:400:3000:Could not set property -AAI_4018=5:2:WARN:4018:400:3000:Unable to convert the string to integer +AAI_4000=5:4:ERROR:4000:500:3002:Internal Error:900 +AAI_4001=5:4:FATAL:4001:500:3002:Configuration file not found:500 +AAI_4002=5:4:FATAL:4002:500:3002:Error reading Configuration file:500 +AAI_4003=5:4:ERROR:4003:500:3002:Error writing to log file:500 +AAI_4004=5:4:FATAL:4004:500:3002:Error reading/parsing the error properties file:500 +AAI_4005=5:4:FATAL:4005:500:3002:Missing or invalid configuration parameter:500 +AAI_4006=5:4:FATAL:4006:500:3002:Unexpected error in service:500 +AAI_4007=5:4:WARN:4007:500:3102:Input parsing error:500 +AAI_4008=5:4:ERROR:4008:500:3002:Output parsing error:500 +AAI_4009=4:0:WARN:4009:400:3000:Invalid X-FromAppId in header:300 +AAI_4010=4:0:WARN:4010:400:3000:Invalid X-TransactionId in header:300 +AAI_4011=5:4:ERROR:4011:500:3002:Missing data for REST error response:500 +AAI_4014=4:0:WARN:4014:400:3000:Invalid Accept header:300 +AAI_4015=4:0:WARN:4015:400:3000:You must provide at least one indexed property:400 +AAI_4016=4:0:WARN:4016:400:3000:The depth parameter must be a number or the string "all":300 +AAI_4017=5:2:INFO:4017:400:3000:Could not set property:300 +AAI_4018=5:2:WARN:4018:400:3000:Unable to convert the string to integer:500 #--- aaidbmap: 5102-5199 -AAI_5102=5:4:FATAL:5102:500:3002:Graph database is null after open -AAI_5105=5:4:ERROR:5105:500:3002:Unexpected error reading/updating database -AAI_5106=5:4:WARN:5106:404:3001:Node not found -AAI_5107=5:2:WARN:5107:400:3000:Required information missing -AAI_5108=5:2:WARN:5108:200:0:Unexpected information in request being ignored +AAI_5102=5:4:FATAL:5102:500:3002:Graph database is null after open:400 +AAI_5105=5:4:ERROR:5105:500:3002:Unexpected error reading/updating database:300 +AAI_5106=5:4:WARN:5106:404:3001:Node not found:300 +AAI_5107=5:2:WARN:5107:400:3000:Required information missing:300 +AAI_5108=5:2:WARN:5108:200:0:Unexpected information in request being ignored:300 #--- aaidbgen: 6101-6199 -AAI_6101=5:4:ERROR:6101:500:3002:null JanusGraph object passed -AAI_6102=5:4:WARN:6102:400:3000:Passed-in property is not valid for this nodeType -AAI_6103=5:4:WARN:6103:400:3000:Required Node-property not found in input data -AAI_6104=5:4:WARN:6104:400:3000:Required Node-property was passed with no data -AAI_6105=5:4:WARN:6105:400:3000:Node-Key-Property not defined in DbMaps -AAI_6106=5:4:WARN:6106:400:3000:Passed-in property is not valid for this edgeType -AAI_6107=5:4:WARN:6107:400:3000:Required Edge-property not found in input data -AAI_6108=5:4:WARN:6108:400:3000:Required Edge-property was passed with no data -AAI_6109=5:4:WARN:6109:400:3000:Bad dependent Node value -AAI_6110=5:4:ERROR:6110:400:3100:Node cannot be deleted -AAI_6111=5:4:WARN:6111:400:3000:JSON processing error -AAI_6112=5:4:ERROR:6112:400:3000:More than one node found by getUniqueNode() -AAI_6114=5:4:INFO:6114:404:3001:Node Not Found -AAI_6115=5:4:ERROR:6115:400:3000:Unrecognized NodeType -AAI_6116=5:4:ERROR:6116:400:3000:Unrecognized Property -AAI_6117=5:4:ERROR:6117:400:3000:Uniqueness constraint violated -AAI_6118=5:4:WARN:6118:400:3000:Required Field not passed. -AAI_6120=5:4:WARN:6120:400:3000:Bad Parameter Passed -AAI_6121=5:4:ERROR:6121:400:3000:Problem with internal AAI reference data -AAI_6122=5:4:ERROR:6122:400:3000:Data Set not complete in DB for this request -AAI_6123=5:4:ERROR:6123:500:3000:Bad Data found by DataGrooming Tool - Investigate -AAI_6124=5:4:ERROR:6124:500:3000:File read/write error -AAI_6125=5:4:WARN:6125:500:3000:Problem Pulling Data Set -AAI_6126=5:4:ERROR:6126:400:3000:Edge cannot be deleted -AAI_6127=5:4:INFO:6127:404:3001:Edge Not Found -AAI_6128=5:4:INFO:6128:500:3000:Unexpected error -AAI_6129=5:4:INFO:6129:404:3003:Error making edge to target node -AAI_6130=5:4:WARN:6130:412:3000:Precondition Required -AAI_6131=5:4:WARN:6131:412:3000:Precondition Failed -AAI_6132=5:4:WARN:6132:400:3000:Bad Model Definition -AAI_6133=5:4:WARN:6133:400:3000:Bad Named Query Definition -AAI_6134=5:4:ERROR:6134:500:6134:Could not persist transaction to storage back end. Exhausted retry amount -AAI_6135=5:4:WARN:6135:412:3000:Resource version specified on create -AAI_6136=5:4:ERROR:6136:400:3000:Object cannot hold multiple items -AAI_6137=5:4:ERROR:6137:400:3000:Cannot perform writes on multiple vertices -AAI_6138=5:4:ERROR:6138:400:3000:Cannot delete multiple vertices -AAI_6139=5:4:ERROR:6139:404:3000:Attempted to add edge to vertex that does not exist -AAI_6140=5:4:ERROR:6140:400:3000:Edge multiplicity violated -AAI_6141=5:4:WARN:6141:400:3000:Please Refine Query -AAI_6142=5:4:INFO:6142:400:3000:Retrying transaction -AAI_6143=5:4:INFO:6143:400:3000:Ghost vertex found -AAI_6144=5:4:WARN:6144:400:3000:Cycle found in graph -AAI_6145=5:4:ERROR:6145:400:3000:Cannot create a nested/containment edge via relationship -AAI_6146=5:4:ERROR:6146:400:3000:Ambiguous identity map found, use a URI instead -AAI_6147=5:4:ERROR:6147:400:3000:Payload Limit Reached, reduce payload +AAI_6101=5:4:ERROR:6101:500:3002:null JanusGraph object passed:400 +AAI_6102=5:4:WARN:6102:400:3000:Passed-in property is not valid for this nodeType:400 +AAI_6103=5:4:WARN:6103:400:3000:Required Node-property not found in input data:400 +AAI_6104=5:4:WARN:6104:400:3000:Required Node-property was passed with no data:400 +AAI_6105=5:4:WARN:6105:400:3000:Node-Key-Property not defined in DbMaps:400 +AAI_6106=5:4:WARN:6106:400:3000:Passed-in property is not valid for this edgeType:400 +AAI_6107=5:4:WARN:6107:400:3000:Required Edge-property not found in input data:400 +AAI_6108=5:4:WARN:6108:400:3000:Required Edge-property was passed with no data:400 +AAI_6109=5:4:WARN:6109:400:3000:Bad dependent Node value:400 +AAI_6110=5:4:ERROR:6110:400:3100:Node cannot be deleted:500 +AAI_6111=5:4:WARN:6111:400:3000:JSON processing error:300 +AAI_6112=5:4:ERROR:6112:400:3000:More than one node found by getUniqueNode():400 +AAI_6114=5:4:INFO:6114:404:3001:Node Not Found:300 +AAI_6115=5:4:ERROR:6115:400:3000:Unrecognized NodeType:400 +AAI_6116=5:4:ERROR:6116:400:3000:Unrecognized Property:400 +AAI_6117=5:4:ERROR:6117:400:3000:Uniqueness constraint violated:400 +AAI_6118=5:4:WARN:6118:400:3000:Required Field not passed.:400 +AAI_6120=5:4:WARN:6120:400:3000:Bad Parameter Passed:300 +AAI_6121=5:4:ERROR:6121:400:3000:Problem with internal AAI reference data:400 +AAI_6122=5:4:ERROR:6122:400:3000:Data Set not complete in DB for this request:400 +AAI_6123=5:4:ERROR:6123:500:3000:Bad Data found by DataGrooming Tool - Investigate:300 +AAI_6124=5:4:ERROR:6124:500:3000:File read/write error:500 +AAI_6125=5:4:WARN:6125:500:3000:Problem Pulling Data Set:500 +AAI_6126=5:4:ERROR:6126:400:3000:Edge cannot be deleted:400 +AAI_6127=5:4:INFO:6127:404:3001:Edge Not Found:400 +AAI_6128=5:4:INFO:6128:500:3000:Unexpected error:900 +AAI_6129=5:4:INFO:6129:404:3003:Error making edge to target node:400 +AAI_6130=5:4:WARN:6130:412:3000:Precondition Required:300 +AAI_6131=5:4:WARN:6131:412:3000:Precondition Failed:300 +AAI_6132=5:4:WARN:6132:400:3000:Bad Model Definition:500 +AAI_6133=5:4:WARN:6133:400:3000:Bad Named Query Definition:500 +AAI_6134=5:4:ERROR:6134:500:6134:Could not persist transaction to storage back end. Exhausted retry amount:500 +AAI_6135=5:4:WARN:6135:412:3000:Resource version specified on create:300 +AAI_6136=5:4:ERROR:6136:400:3000:Object cannot hold multiple items:400 +AAI_6137=5:4:ERROR:6137:400:3000:Cannot perform writes on multiple vertices:400 +AAI_6138=5:4:ERROR:6138:400:3000:Cannot delete multiple vertices:400 +AAI_6139=5:4:ERROR:6139:404:3000:Attempted to add edge to vertex that does not exist:400 +AAI_6140=5:4:ERROR:6140:400:3000:Edge multiplicity violated:400 +AAI_6141=5:4:WARN:6141:400:3000:Please Refine Query:400 +AAI_6142=5:4:INFO:6142:400:3000:Retrying transaction:900 +AAI_6143=5:4:INFO:6143:400:3000:Ghost vertex found:400 +AAI_6144=5:4:WARN:6144:400:3000:Cycle found in graph:400 +AAI_6145=5:4:ERROR:6145:400:3000:Cannot create a nested/containment edge via relationship:400 +AAI_6146=5:4:ERROR:6146:400:3000:Ambiguous identity map found, use a URI instead:300 +AAI_6147=5:4:ERROR:6147:400:3000:Payload Limit Reached, reduce payload:300 +AAI_6148=5:4:ERROR:6148:400:3000:More than one node found %1:300 +AAI_6149=5:4:ERROR:6149:404:3000:No relationship was found:300 #--- aaicsvp: 7101-7199 -AAI_7101=5:4:ERROR:7101:500:3002:Unexpected error in CSV file processing -AAI_7102=5:4:ERROR:7102:500:3002:Error in cleanup temporary directory -#AAI_7103=4:2:ERROR:7103:500:3002:Unsupported user -AAI_7104=5:4:ERROR:7104:500:3002:Failed to create directory -AAI_7105=5:4:ERROR:7105:500:3002:Temporary directory exists -AAI_7106=5:4:ERROR:7106:500:3002:Cannot delete -AAI_7107=5:4:ERROR:7107:500:3002:Input file does not exist -AAI_7108=5:4:ERROR:7108:500:3002:Output file does not exist -AAI_7109=5:4:ERROR:7109:500:3002:Error closing file -AAI_7110=5:4:ERROR:7110:500:3002:Error loading/reading properties file -AAI_7111=5:4:ERROR:7111:500:3002:Error executing shell script -AAI_7112=5:4:ERROR:7112:500:3002:Error creating output file -AAI_7113=5:4:ERROR:7113:500:3002:Trailer record error -AAI_7114=5:4:ERROR:7114:500:3002:Input file error -AAI_7115=5:4:ERROR:7115:500:3002:Unexpected error -AAI_7116=5:4:ERROR:7116:500:3002:Request error -AAI_7117=5:4:ERROR:7117:500:3002:Error in get http client object -AAI_7118=5:4:ERROR:7118:500:3002:Script Error -AAI_7119=5:4:ERROR:7119:500:3002:Unknown host +AAI_7101=5:4:ERROR:7101:500:3002:Unexpected error in CSV file processing:900 +AAI_7102=5:4:ERROR:7102:500:3002:Error in cleanup temporary directory:500 +#AAI_7103=4:2:ERROR:7103:500:3002:Unsupported user:100 +AAI_7104=5:4:ERROR:7104:500:3002:Failed to create directory:500 +AAI_7105=5:4:ERROR:7105:500:3002:Temporary directory exists:500 +AAI_7106=5:4:ERROR:7106:500:3002:Cannot delete:500 +AAI_7107=5:4:ERROR:7107:500:3002:Input file does not exist:500 +AAI_7108=5:4:ERROR:7108:500:3002:Output file does not exist:500 +AAI_7109=5:4:ERROR:7109:500:3002:Error closing file:500 +AAI_7110=5:4:ERROR:7110:500:3002:Error loading/reading properties file:500 +AAI_7111=5:4:ERROR:7111:500:3002:Error executing shell script:900 +AAI_7112=5:4:ERROR:7112:500:3002:Error creating output file:500 +AAI_7113=5:4:ERROR:7113:500:3002:Trailer record error:300 +AAI_7114=5:4:ERROR:7114:500:3002:Input file error:300 +AAI_7115=5:4:ERROR:7115:500:3002:Unexpected error:900 +AAI_7116=5:4:ERROR:7116:500:3002:Request error:900 +AAI_7117=5:4:ERROR:7117:500:3002:Error in get http client object:500 +AAI_7118=5:4:ERROR:7118:500:3002:Script Error:900 +AAI_7119=5:4:ERROR:7119:500:3002:Unknown host:900 #--- aaisdnc: 7201-7299 -AAI_7202=5:4:ERROR:7202:500:3002:Error getting connection to odl -AAI_7203=5:4:ERROR:7203:500:3002:Unexpected error calling DataChangeNotification API -AAI_7204=5:4:ERROR:7204:500:3002:Error returned by DataChangeNotification API -AAI_7205=5:4:ERROR:7205:500:3002:Unexpected error running notifySDNCOnUpdate -#AAI_7206=5:4:ERROR:7206:500:3002:Invalid data returned from ODL +AAI_7202=5:4:ERROR:7202:500:3002:Error getting connection to odl:200 +AAI_7203=5:4:ERROR:7203:500:3002:Unexpected error calling DataChangeNotification API:200 +AAI_7204=5:4:ERROR:7204:500:3002:Error returned by DataChangeNotification API:200 +AAI_7205=5:4:ERROR:7205:500:3002:Unexpected error running notifySDNCOnUpdate:200 +#AAI_7206=5:4:ERROR:7206:500:3002:Invalid data returned from ODL:200 #--- NotificationEvent, using UEB space -AAI_7350=5:4:ERROR:7305:500:3002:Notification event creation failed +AAI_7350=5:4:ERROR:7305:500:3002:Notification event creation failed:500 #--- aairestctlr: 7401-7499 -AAI_7401=5:4:ERROR:7401:500:3002:Error connecting to AAI REST API -AAI_7402=5:4:ERROR:7402:500:3002:Unexpected error -AAI_7403=5:4:WARN:7403:400:3001:Request error -AAI_7404=5:4:INFO:7404:404:3001:Node not found -AAI_7405=5:4:WARN:7405:200:0:UUID not formatted correctly, generating UUID -AAI_7406=5:4:ERROR:7406:400:7406:Request Timed Out +AAI_7401=5:4:ERROR:7401:500:3002:Error connecting to AAI REST API:200 +AAI_7402=5:4:ERROR:7402:500:3002:Unexpected error:900 +AAI_7403=5:4:WARN:7403:400:3001:Request error:900 +AAI_7404=5:4:INFO:7404:404:3001:Node not found:300 +AAI_7405=5:4:WARN:7405:200:0:UUID not formatted correctly, generating UUID:300 +AAI_7406=5:4:ERROR:7406:400:7406:Request Timed Out:200 #--- aaicsiovals: 7501-7599 -#AAI_7501=5:4:WARN:7501:500:3002:Error getting connection to CSI-OVALS -AAI_7502=5:4:WARN:7502:500:3002:Bad parameter when trying to build request for CSI-OVALS -AAI_7503=5:4:WARN:7503:500:3002:Error returned by CSI-OVALS +#AAI_7501=5:4:WARN:7501:500:3002:Error getting connection to CSI-OVALS:200 +AAI_7502=5:4:WARN:7502:500:3002:Bad parameter when trying to build request for CSI-OVALS:300 +AAI_7503=5:4:WARN:7503:500:3002:Error returned by CSI-OVALS:500 #-- dataexport: 8001-8099 -AAI_8001=5:4:WARN:8001:500:3002:Unable to find data snapshots -AAI_8002=5:4:ERROR:8002:500:3002:Script Error -AAI_8003=5:4:ERROR:8003:500:3002:Dynamic Payload Generator Error +AAI_8001=5:4:WARN:8001:500:3002:Unable to find data snapshots:500 +AAI_8002=5:4:ERROR:8002:500:3002:Script Error:500 +AAI_8003=5:4:ERROR:8003:500:3002:Dynamic Payload Generator Error:500 #--- aaiauth: 9101-9199 -AAI_9101=5:0:WARN:9101:403:3300:User is not authorized to perform function -#AAI_9102=5:0:WARN:9102:401:3301:Refresh credentials from source -#AAI_9103=5:0:WARN:9103:403:3300:User not found -#AAI_9104=5:0:WARN:9104:401:3302:Authentication error -#AAI_9105=5:0:WARN:9105:403:3300:Authorization error -#AAI_9106=5:0:WARN:9106:403:3300:Invalid AppId -#AAI_9107=5:0:WARN:9107:403:3300:No Username in Request -AAI_9107=5:0:WARN:9107:403:3300:SSL is not provided in request, please contact admin -AAI_9108=5:0:WARN:9107:403:3300:Basic auth credentials is not provided in the request +AAI_9101=5:0:WARN:9101:403:3300:User is not authorized to perform function:100 +#AAI_9102=5:0:WARN:9102:401:3301:Refresh credentials from source:100 +#AAI_9103=5:0:WARN:9103:403:3300:User not found:100 +#AAI_9104=5:0:WARN:9104:401:3302:Authentication error:100 +#AAI_9105=5:0:WARN:9105:403:3300:Authorization error:100 +#AAI_9106=5:0:WARN:9106:403:3300:Invalid AppId:300 +#AAI_9107=5:0:WARN:9107:403:3300:No Username in Request:100 +AAI_9107=5:0:WARN:9107:403:3300:SSL is not provided in request, please contact admin:100 +AAI_9108=5:0:WARN:9107:403:3300:Basic auth credentials is not provided in the request:100 +AAI_9109=5:0:WARN:9109:403:3300:User is not allowed to perform implicit delete:500 #--- aaiinstar: 9201-9299 #AAI_9201=5:4:ERROR:9201:500:3002:Unable to send notification -AAI_9202=5:4:ERROR:9202:500:3002:Unable to start a thread +AAI_9202=5:4:ERROR:9202:500:3002:Unable to start a thread:500 diff --git a/src/main/resources/etc/appprops/janusgraph-cached.properties b/src/main/resources/etc/appprops/janusgraph-cached.properties index 23e5c3d..40bf765 100644 --- a/src/main/resources/etc/appprops/janusgraph-cached.properties +++ b/src/main/resources/etc/appprops/janusgraph-cached.properties @@ -2,7 +2,7 @@ # ============LICENSE_START======================================================= # org.onap.aai # ================================================================================ -# Copyright © 2017-18 AT&T Intellectual Property. All rights reserved. +# Copyright © 2017-18 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ # ============LICENSE_END========================================================= query.fast-property=true +query.smart-limit=false # the following parameters are not reloaded automatically and require a manual bounce storage.backend=inmemory storage.hostname=localhost diff --git a/src/main/resources/etc/appprops/janusgraph-realtime.properties b/src/main/resources/etc/appprops/janusgraph-realtime.properties index b45d7ee..3a7df75 100644 --- a/src/main/resources/etc/appprops/janusgraph-realtime.properties +++ b/src/main/resources/etc/appprops/janusgraph-realtime.properties @@ -2,7 +2,7 @@ # ============LICENSE_START======================================================= # org.onap.aai # ================================================================================ -# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ # ============LICENSE_END========================================================= query.fast-property=true +query.smart-limit=false # the following parameters are not reloaded automatically and require a manual bounce storage.backend=inmemory storage.hostname=localhost diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 71262e9..eff961f 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -31,13 +31,30 @@ <property name="AJSC_HOME" value="${AJSC_HOME:-.}" /> <property name="logDirectory" value="${AJSC_HOME}/logs" /> + <!-- Old patterns <property name="eelfLogPattern" value="%ecompStartTime|%date{yyyy-MM-dd'T'HH:mm:ss.SSSZ, UTC}|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{partnerName}|%ecompStatusCode|%X{responseCode}|%replace(%replace(%X{responseDescription}){'\\|', '!'}){'\r|\n', '^'}|%X{instanceUUID}|%level|%X{severity}|%X{serverIpAddress}|%ecompElapsedTime|%X{server}|%X{clientIpAddress}|%eelfClassOfCaller|%X{unused}|%X{processKey}|%X{customField1}|%X{customField2}|%X{customField3}|%X{customField4}|co=%X{component}:%replace(%replace(%m){'\\|', '!'}){'\r|\n', '^'}%n"/> <property name="eelfAuditLogPattern" value="%ecompStartTime|%date{yyyy-MM-dd'T'HH:mm:ss.SSSZ, UTC}|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{partnerName}|%ecompStatusCode|%X{responseCode}|%replace(%replace(%X{responseDescription}){'\\|', '!'}){'\r|\n|\r\n', '^'}|%X{instanceUUID}|%level|%X{severity}|%X{serverIpAddress}|%ecompElapsedTime|%X{server}|%X{clientIpAddress}|%eelfClassOfCaller|%X{unused}|%X{processKey}|%X{customField1}|%X{customField2}|%X{customField3}|%X{customField4}|co=%X{component}:%replace(%replace(%m){'\\|', '!'}){'\r|\n', '^'}%n"/> <property name="eelfMetricLogPattern" value="%ecompStartTime|%date{yyyy-MM-dd'T'HH:mm:ss.SSSZ, UTC}|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{partnerName}|%X{targetEntity}|%X{targetServiceName}|%ecompStatusCode|%X{responseCode}|%replace(%replace(%X{responseDescription}){'\\|', '!'}){'\r|\n', '^'}|%X{instanceUUID}|%level|%X{severity}|%X{serverIpAddress}|%ecompElapsedTime|%X{server}|%X{clientIpAddress}|%eelfClassOfCaller|%X{unused}|%X{processKey}|%X{targetVirtualEntity}|%X{customField1}|%X{customField2}|%X{customField3}|%X{customField4}|co=%X{component}:%replace(%replace(%m){'\\|', '!'}){'\r|\n', '^'}%n"/> - <!-- <property name="eelfErrorLogPattern" value="%ecompStartTime|%X{requestId}|%-10t|%X{serviceName}|%X{partnerName}|%X{targetEntity}|%X{targetServiceName}|%ecompErrorCategory|%X{responseCode}|%replace(%replace(%X{responseDescription}){'\\|', '!'}){'\r|\n|\r\n', '^'}|co=%X{component}:%replace(%replace(%m){'\\|', '!'}){'\r|\n', '^'}%n"/> --> + <property name="eelfErrorLogPattern" value="%ecompStartTime|%X{requestId}|%-10t|%X{serviceName}|%X{partnerName}|%X{targetEntity}|%X{targetServiceName}|%ecompErrorCategory|%X{responseCode}|%replace(%replace(%X{responseDescription}){'\\|', '!'}){'\r|\n|\r\n', '^'}|co=%X{component}:%replace(%replace(%m){'\\|', '!'}){'\r|\n', '^'}%n"/> <property name="eelfErrorLogPattern" value="%ecompStartTime|%X{requestId}|%-10t|%X{serviceName}|%X{partnerName}|%X{targetEntity}|%X{targetServiceName}|%ecompErrorCategory|%ecompResponseCode|%ecompResponseDescription|co=%X{component}:%replace(%replace(%m){'\\|', '!'}){'\r|\n', '^'}%n"/> <property name="eelfTransLogPattern" value="%ecompStartTime|%date{yyyy-MM-dd'T'HH:mm:ss.SSSZ, UTC}|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{partnerName}|%ecompStatusCode|%X{responseCode}|%replace(%replace(%X{responseDescription}){'\\|', '!'}){'\r|\n', '^'}|%X{instanceUUID}|%level|%X{severity}|%X{serverIpAddress}|%ecompElapsedTime|%X{server}|%X{clientIpAddress}|%eelfClassOfCaller|%X{unused}|%X{processKey}|%X{customField1}|%X{customField2}|%X{customField3}|%X{customField4}|co=%X{partnerName}:%m%n"/> - + --> + <property name="p_tim" value="%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}"/> + <property name="p_lvl" value="%level"/> + <property name="p_log" value="%logger"/> + <property name="p_mdc" value="%replace(%replace(%mdc){'\t','\\\\t'}){'\n', '\\\\n'}"/> + <property name="p_msg" value="%replace(%replace(%msg){'\t', '\\\\t'}){'\n','\\\\n'}"/> + <property name="p_exc" value="%replace(%replace(%rootException){'\t', '\\\\t'}){'\n','\\\\n'}"/> + <property name="p_mak" value="%replace(%replace(%marker){'\t', '\\\\t'}){'\n','\\\\n'}"/> + <property name="p_thr" value="%thread"/> + <property name="pattern" value="%nopexception${p_tim}\t${p_thr}\t${p_lvl}\t${p_log}\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t%n"/> + <!-- Patterns from onap demo --> + <property name="errorPattern" value="%X{LogTimestamp}|%X{RequestID}|%thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%.-5level|%X{ErrorCode}|%X{ErrorDesc}|%msg%n" /> + <property name="debugPattern" value="%X{LogTimestamp}|%X{RequestID}|%msg\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t|^%n" /> + + <property name="auditPattern" value="%X{EntryTimestamp}|%X{LogTimestamp}|%X{RequestID}|%X{ServiceInstanceID}|%thread||%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}||||${p_mak}|${p_mdc}|||%msg%n" /> + <property name="metricPattern" value="%X{InvokeTimestamp}|%X{LogTimestamp}|%X{RequestID}|%X{ServiceInstanceID}|%thread||%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}||||%X{TargetVirtualEntity}|${p_mak}|${p_mdc}|||%msg%n" /> + <property name="transLogPattern" value="%X{LogTimestamp}|%date{yyyy-MM-dd'T'HH:mm:ss.SSSZ, UTC}|%X{RequestID}|%X{ServiceInstanceID}|%-10t|%X{ServerFQDN}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%replace(%replace(%X{ResponseDesc}){'\\|', '!'}){'\r|\n', '^'}|%X{InstanceUUID}|%level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{clientIpAddress}||%X{unused}|%X{processKey}|%X{customField1}|%X{customField2}|%X{customField3}|%X{customField4}|co=%X{PartnerName}:%m%n"/> <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" /> <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" /> <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" /> @@ -66,70 +83,64 @@ <appender-ref ref="SANE" /> </appender> - <appender name="METRIC" - class="ch.qos.logback.core.rolling.RollingFileAppender"> - <filter class="ch.qos.logback.classic.filter.LevelFilter"> - <level>INFO</level> - <onMatch>ACCEPT</onMatch> - <onMismatch>DENY</onMismatch> - </filter> + <appender name="METRIC" class="ch.qos.logback.core.rolling.RollingFileAppender"> <file>${logDirectory}/rest/metrics.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/rest/metrics.log.%d{yyyy-MM-dd} </fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfMetricLogPattern}</pattern> + <encoder> + <pattern>${metricPattern}</pattern> </encoder> </appender> <appender name="asyncMETRIC" class="ch.qos.logback.classic.AsyncAppender"> <queueSize>1000</queueSize> <includeCallerData>true</includeCallerData> - <appender-ref ref="METRIC" /> + <appender-ref ref="METRIC"/> </appender> <appender name="DEBUG" - class="ch.qos.logback.core.rolling.RollingFileAppender"> + class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>DEBUG</level> <onMatch>ACCEPT</onMatch> <onMismatch>DENY</onMismatch> </filter> <file>${logDirectory}/rest/debug.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/rest/debug.log.%d{yyyy-MM-dd} - </fileNamePattern> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/rest/debug.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfLogPattern}</pattern> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> <appender name="asyncDEBUG" class="ch.qos.logback.classic.AsyncAppender"> <queueSize>1000</queueSize> - <includeCallerData>true</includeCallerData> <appender-ref ref="DEBUG" /> + <includeCallerData>true</includeCallerData> </appender> <appender name="ERROR" - class="ch.qos.logback.core.rolling.RollingFileAppender"> + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/rest/error.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/rest/error.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> <level>WARN</level> </filter> - <file>${logDirectory}/rest/error.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/rest/error.log.%d{yyyy-MM-dd} - </fileNamePattern> - </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${"eelfErrorLogPattern"}</pattern> + <encoder> + <pattern>${errorPattern}</pattern> </encoder> </appender> <appender name="asyncERROR" class="ch.qos.logback.classic.AsyncAppender"> <queueSize>1000</queueSize> - <includeCallerData>true</includeCallerData> - <appender-ref ref="ERROR" /> + <appender-ref ref="ERROR"/> </appender> <appender name="AUDIT" @@ -139,8 +150,8 @@ <fileNamePattern>${logDirectory}/rest/audit.log.%d{yyyy-MM-dd} </fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfAuditLogPattern}</pattern> + <encoder> + <pattern>${auditPattern}</pattern> </encoder> </appender> @@ -162,8 +173,8 @@ <fileNamePattern>${logDirectory}/rest/translog.log.%d{yyyy-MM-dd} </fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfTransLogPattern}</pattern> + <encoder> + <pattern>${transLogPattern}</pattern> </encoder> </appender> @@ -183,11 +194,26 @@ <fileNamePattern>${logDirectory}/dmaapAAIEventConsumer/error.log.%d{yyyy-MM-dd} </fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${"eelfErrorLogPattern"}</pattern> + <encoder> + <pattern>${"errorPattern"}</pattern> </encoder> </appender> - + <appender name="dmaapAAIEventConsumerInfo" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/dmaapAAIEventConsumer/dmaap-transaction.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/dmaapAAIEventConsumer/dmaap-transaction.log.%d{yyyy-MM-dd} + </fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> + </encoder> + </appender> <appender name="dmaapAAIEventConsumerDebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> @@ -200,8 +226,8 @@ <fileNamePattern>${logDirectory}/dmaapAAIEventConsumer/debug.log.%d{yyyy-MM-dd} </fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfLogPattern}</pattern> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> <appender name="dmaapAAIEventConsumerMetric" @@ -216,8 +242,8 @@ <fileNamePattern>${logDirectory}/dmaapAAIEventConsumer/metrics.log.%d{yyyy-MM-dd} </fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfMetricLogPattern}</pattern> + <encoder> + <pattern>${metricPattern}</pattern> </encoder> </appender> <appender name="external" @@ -230,8 +256,8 @@ <fileNamePattern>${logDirectory}/external/external.log.%d{yyyy-MM-dd} </fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfLogPattern}</pattern> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> @@ -244,8 +270,8 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/dataGrooming/error.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfErrorLogPattern}</pattern> + <encoder> + <pattern>${errorPattern}</pattern> </encoder> </appender> @@ -259,28 +285,29 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/dataGrooming/debug.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfLogPattern}</pattern> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> - <appender name="dataGroomingmetric" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <appender name="dataGroomingaudit" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>INFO</level> <onMatch>ACCEPT</onMatch> <onMismatch>DENY</onMismatch> </filter> - <File>${logDirectory}/dataGrooming/metrics.log</File> + <File>${logDirectory}/dataGrooming/audit.log</File> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/dataGrooming/metrics.log.%d{yyyy-MM-dd}</fileNamePattern> + <fileNamePattern>${logDirectory}/dataGrooming/audit.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfMetricLogPattern}</pattern> + <encoder> + <pattern>${auditPattern}</pattern> </encoder> </appender> <!-- DataGrooming logs ended --> - + + <!-- DataSnapshot logs started --> <appender name="dataSnapshot" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> @@ -290,8 +317,8 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/dataSnapshot/error.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfErrorLogPattern}</pattern> + <encoder> + <pattern>${errorPattern}</pattern> </encoder> </appender> @@ -305,28 +332,74 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/dataSnapshot/debug.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfLogPattern}</pattern> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> - <appender name="dataSnapshotmetric" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <appender name="dataSnapshotaudit" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>INFO</level> <onMatch>ACCEPT</onMatch> <onMismatch>DENY</onMismatch> </filter> - <File>${logDirectory}/dataSnapshot/metrics.log</File> + <File>${logDirectory}/dataSnapshot/audit.log</File> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/dataSnapshot/metrics.log.%d{yyyy-MM-dd}</fileNamePattern> + <fileNamePattern>${logDirectory}/dataSnapshot/audit.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfMetricLogPattern}</pattern> + <encoder> + <pattern>${auditPattern}</pattern> </encoder> </appender> <!-- DataSnapshot logs ended --> + <!-- HistoryTruncate logs started --> + <appender name="historyTruncate" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>WARN</level> + </filter> + <File>${logDirectory}/historyTruncate/error.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/historyTruncate/error.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${errorPattern}</pattern> + </encoder> + </appender> + + <appender name="historyTruncatedebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>DEBUG</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/historyTruncate/debug.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/historyTruncate/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> + </encoder> + </appender> + + <appender name="historyTruncateaudit" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/historyTruncate/audit.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/historyTruncate/audit.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${auditPattern}</pattern> + </encoder> + </appender> + <!-- historyTruncate logs ended --> + + <!-- CreateDBSchema logs started --> <appender name="createDBSchema" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> @@ -336,8 +409,8 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/createDBSchema/error.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${"eelfErrorLogPattern"}</pattern> + <encoder> + <pattern>${"errorPattern"}</pattern> </encoder> </appender> @@ -351,8 +424,8 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/createDBSchema/debug.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfLogPattern}</pattern> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> @@ -366,8 +439,8 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/createDBSchema/metrics.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfMetricLogPattern}</pattern> + <encoder> + <pattern>${metricPattern}</pattern> </encoder> </appender> <!-- CreateDBSchema logs ended --> @@ -381,8 +454,8 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/misc/error.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${"eelfErrorLogPattern"}</pattern> + <encoder> + <pattern>${"errorPattern"}</pattern> </encoder> </appender> @@ -396,72 +469,170 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/misc/debug.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfLogPattern}</pattern> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> - <appender name="dataCleanupmetric" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <appender name="dataCleanupaudit" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>INFO</level> <onMatch>ACCEPT</onMatch> <onMismatch>DENY</onMismatch> </filter> - <File>${logDirectory}/misc/metrics.log</File> + <File>${logDirectory}/misc/audit.log</File> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/misc/metrics.log.%d{yyyy-MM-dd}</fileNamePattern> + <fileNamePattern>${logDirectory}/misc/audit.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfMetricLogPattern}</pattern> + <encoder> + <pattern>${auditPattern}</pattern> </encoder> </appender> <!-- DataCleanupTasks logs ended --> - - <!-- pullInvData logs started --> - <appender name="pullInvData" class="ch.qos.logback.core.rolling.RollingFileAppender"> + + <!-- dupeTool logs started --> + <appender name="dupeTooldebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>DEBUG</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/dupetool/debug.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/dupetool/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> + </encoder> + </appender> + + <appender name="dupeToolerror" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>WARN</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/dupeTool/error.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/dupeTool/error.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${errorPattern}</pattern> + </encoder> + </appender> + <!-- dupeTool logs ended --> + + <!-- dynamicPayloadGenerator log starts here --> + <appender name="dynamicPayloadGeneratorError" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> <level>WARN</level> </filter> - <File>${logDirectory}/pullInvData/error.log</File> + <File>${logDirectory}/dynamicPayloadGenerator/error.log</File> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/pullInvData/error.log.%d{yyyy-MM-dd}</fileNamePattern> + <fileNamePattern>${logDirectory}/dynamicPayloadGenerator/error.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${"eelfErrorLogPattern"}</pattern> + <encoder> + <pattern>${errorPattern}</pattern> </encoder> </appender> - <appender name="pullInvDatadebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <appender name="dynamicPayloadGeneratorDebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>DEBUG</level> <onMatch>ACCEPT</onMatch> <onMismatch>DENY</onMismatch> </filter> - <File>${logDirectory}/pullInvData/debug.log</File> + <File>${logDirectory}/dynamicPayloadGenerator/debug.log</File> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/pullInvData/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + <fileNamePattern>${logDirectory}/dynamicPayloadGenerator/debug.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfLogPattern}</pattern> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> - <appender name="pullInvDatametric" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <appender name="dynamicPayloadGeneratorAudit" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>INFO</level> <onMatch>ACCEPT</onMatch> <onMismatch>DENY</onMismatch> </filter> - <File>${logDirectory}/pullInvData/metrics.log</File> + <File>${logDirectory}/dataExport/audit.log</File> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/pullInvData/metrics.log.%d{yyyy-MM-dd}</fileNamePattern> + <fileNamePattern>${logDirectory}/dynamicPayloadGenerator/audit.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfMetricLogPattern}</pattern> + <encoder> + <pattern>${auditPattern}</pattern> + </encoder> + </appender> + <!-- dynamicPayloadGenerator log ends here --> + + + <!-- forceDelete logs started --> + <appender name="forceDeletedebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>DEBUG</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/forceDelete/debug.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/forceDelete/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> + </encoder> + </appender> + + <appender name="forceDeleteerror" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>WARN</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/forceDelete/error.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/forceDelete/error.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${errorPattern}</pattern> + </encoder> + </appender> + <!-- forceDelete logs ended --> + + <!-- migration logs started --> + <appender name="migrationdebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>DEBUG</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/migration/debug.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/migration/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> - <!-- pullInvData logs ended --> - <!-- DataGrooming logs started --> + + <appender name="migrationerror" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>WARN</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/migration/error.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/migration/error.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${errorPattern}</pattern> + </encoder> + </appender> + <!-- migration logs ended --> + + <!-- DataGrooming logs started --> <appender name="dataExportError" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> <level>WARN</level> @@ -470,8 +641,8 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/dataExport/error.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfErrorLogPattern}</pattern> + <encoder> + <pattern>${errorPattern}</pattern> </encoder> </appender> @@ -485,30 +656,151 @@ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>${logDirectory}/dataExport/debug.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfLogPattern}</pattern> + <encoder> + <pattern>${debugPattern}</pattern> + </encoder> + </appender> + + <appender name="dataExportAudit" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/dataExport/audit.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/dataExport/audit.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${auditPattern}</pattern> + </encoder> + </appender> + + <!-- schemaMod log starts --> + <appender name="schemaModdebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>DEBUG</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/schemaMod/debug.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/schemaMod/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> + </encoder> + </appender> + + <appender name="schemaModerror" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>WARN</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/schemaMod/error.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/schemaMod/error.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${errorPattern}</pattern> + </encoder> + </appender> + <!-- schemaMod log ends --> + + <!-- uniquePropertyCheck log starts here --> + <appender name="uniquePropertyCheckdebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>DEBUG</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/uniquePropertyCheck/debug.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/uniquePropertyCheck/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> + </encoder> + </appender> + <appender name="uniquePropertyCheckmetric" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/uniquePropertyCheck/metrics.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/uniquePropertyCheck/metrics.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${metricPattern}</pattern> + </encoder> + </appender> + + <appender name="uniquePropertyCheckerror" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>WARN</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/uniquePropertyCheck/error.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/uniquePropertyCheck/error.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${errorPattern}</pattern> + </encoder> + </appender> + <!-- uniquePropertyCheck log ends here --> + + <!-- dynamicPayloadGenerator log starts here --> + <appender name="dynamicPayloadGeneratorError" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>WARN</level> + </filter> + <File>${logDirectory}/dynamicPayloadGenerator/error.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/dynamicPayloadGenerator/error.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${errorPattern}</pattern> + </encoder> + </appender> + + <appender name="dynamicPayloadGeneratorDebug" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>DEBUG</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/dynamicPayloadGenerator/debug.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/dynamicPayloadGenerator/debug.log.%d{yyyy-MM-dd}</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> </encoder> </appender> - <appender name="dataExportMetric" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <appender name="dynamicPayloadGeneratorAudit" class="ch.qos.logback.core.rolling.RollingFileAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>INFO</level> <onMatch>ACCEPT</onMatch> <onMismatch>DENY</onMismatch> </filter> - <File>${logDirectory}/dataExport/metrics.log</File> + <File>${logDirectory}/dataExport/audit.log</File> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <fileNamePattern>${logDirectory}/dataExport/metrics.log.%d{yyyy-MM-dd}</fileNamePattern> + <fileNamePattern>${logDirectory}/dynamicPayloadGenerator/audit.log.%d{yyyy-MM-dd}</fileNamePattern> </rollingPolicy> - <encoder class="org.onap.aai.logging.EcompEncoder"> - <pattern>${eelfMetricLogPattern}</pattern> + <encoder> + <pattern>${auditPattern}</pattern> </encoder> </appender> + <!-- dynamicPayloadGenerator log ends here --> <logger name="org.onap.aai" level="DEBUG" additivity="false"> <appender-ref ref="asyncDEBUG" /> - <appender-ref ref="asyncERROR" /> - <appender-ref ref="asyncMETRIC" /> <appender-ref ref="asyncSANE" /> </logger> @@ -590,7 +882,7 @@ <maxFileSize>5MB</maxFileSize> </triggeringPolicy> <encoder> - <pattern>eelfAuditLogPattern</pattern> + <pattern>auditPattern</pattern> </encoder> </appender> <appender name="perfLogs" @@ -612,6 +904,25 @@ <pattern>"%d [%thread] %-5level %logger{1024} - %msg%n"</pattern> </encoder> </appender> + <appender name="auth" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>DEBUG</level> + </filter> + <file>${logDirectory}/auth/auth.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/auth/auth.log.%d{yyyy-MM-dd} + </fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ, UTC}[%thread] %-5level %logger{1024} - %msg%n</pattern> + </encoder> + </appender> + <appender name="asyncAUTH" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>1000</queueSize> + <includeCallerData>true</includeCallerData> + <appender-ref ref="auth" /> + </appender> <logger name="AuditRecord" level="INFO" additivity="false"> <appender-ref ref="auditLogs" /> </logger> @@ -622,19 +933,30 @@ <appender-ref ref="perfLogs" /> </logger> <!-- logback jms appenders & loggers definition ends here --> - - <logger name="org.onap.aai.interceptors.post" level="DEBUG" - additivity="false"> - <appender-ref ref="asynctranslog" /> + <logger name="org.onap.aai.aaf" level="DEBUG" additivity="false"> + <appender-ref ref="asyncAUTH" /> </logger> - - <logger name="org.onap.aai.interceptors.pre.SetLoggingContext" level="DEBUG"> + <logger name="org.onap.aai.aailog.filter.RestClientLoggingInterceptor" level="INFO"> + <appender-ref ref="asyncMETRIC"/> + </logger> + <logger name="org.onap.logging.filter.base.AbstractMetricLogFilter" level="INFO"> + <appender-ref ref="asyncMETRIC"/> + </logger> + <logger name="org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog" level="INFO"> <appender-ref ref="asyncAUDIT"/> </logger> - - <logger name="org.onap.aai.interceptors.post.ResetLoggingContext" level="DEBUG"> + <logger name="org.onap.logging.filter.base.AbstractAuditLogFilter" level="INFO"> <appender-ref ref="asyncAUDIT"/> </logger> + <logger name="org.onap.aai.aailog.logs.AaiDBMetricLog" level="INFO"> + <appender-ref ref="asyncMETRIC"/> + </logger> + <logger name="org.onap.aai.logging.ErrorLogHelper" level="WARN"> + <appender-ref ref="asyncERROR"/> + </logger> + <logger name="org.onap.aai.interceptors.post" level="DEBUG" additivity="false"> + <appender-ref ref="asynctranslog" /> + </logger> <logger name="org.onap.aai.dmaap" level="DEBUG" additivity="false"> <appender-ref ref="dmaapAAIEventConsumer" /> @@ -645,14 +967,20 @@ <logger name="org.onap.aai.datasnapshot" level="DEBUG" additivity="false"> <appender-ref ref="dataSnapshot"/> <appender-ref ref="dataSnapshotdebug"/> - <appender-ref ref="dataSnapshotmetric"/> + <appender-ref ref="dataSnapshotaudit"/> <appender-ref ref="STDOUT"/> </logger> + <logger name="org.onap.aai.historytruncate" level="DEBUG" additivity="false"> + <appender-ref ref="historyTruncate"/> + <appender-ref ref="historyTruncatedebug"/> + <appender-ref ref="historyTruncateaudit"/> + </logger> + <logger name="org.onap.aai.datagrooming" level="DEBUG" additivity="false"> <appender-ref ref="dataGrooming"/> <appender-ref ref="dataGroomingdebug"/> - <appender-ref ref="dataGroomingmetric"/> + <appender-ref ref="dataGroomingaudit"/> <appender-ref ref="STDOUT"/> </logger> @@ -661,29 +989,50 @@ <appender-ref ref="createDBSchemadebug"/> <appender-ref ref="createDBSchemametric"/> </logger> - + + <logger name="org.onap.aai.dbgen.DupeTool" level="DEBUG" additivity="false"> + <appender-ref ref="dupeTooldebug" /> + <appender-ref ref="dupeToolerror" /> + </logger> + + <logger name="org.onap.aai.dbgen.DynamicPayloadGenerator" level="DEBUG" additivity="false"> + <appender-ref ref="dynamicPayloadGeneratorAudit" /> + <appender-ref ref="dynamicPayloadGeneratorError" /> + <appender-ref ref="dynamicPayloadGeneratorDebug" /> + </logger> + <logger name="org.onap.aai.dbgen" level="DEBUG" additivity="false"> <appender-ref ref="createDBSchema"/> <appender-ref ref="createDBSchemadebug"/> <appender-ref ref="createDBSchemametric"/> </logger> - <logger name="org.onap.aai.dbgen.PullInvData" level="DEBUG" additivity="false"> - <appender-ref ref="pullInvData"/> - <appender-ref ref="pullInvDatadebug"/> - <appender-ref ref="pullInvDatametric"/> - </logger> - - <logger name="org.onap.aai.datacleanup" level="INFO" additivity="false"> + <logger name="org.onap.aai.datacleanup" level="DEBUG" additivity="false"> <appender-ref ref="dataCleanuperror" /> <appender-ref ref="dataCleanupdebug" /> - <appender-ref ref="dataCleanupmetric" /> + <appender-ref ref="dataCleanupaudit" /> <appender-ref ref="STDOUT"/> </logger> + + <logger name="org.onap.aai.migration" level="DEBUG" additivity="false"> + <appender-ref ref="migrationdebug" /> + <appender-ref ref="migrationerror" /> + </logger> + + <logger name="org.onap.aai.util.SendMigrationNotifications" level="DEBUG" additivity="false"> + <appender-ref ref="migrationdebug" /> + <appender-ref ref="migrationerror" /> + </logger> + + <logger name="org.onap.aai.util.SendDeleteMigrationNotifications" level="DEBUG" additivity="false"> + <appender-ref ref="migrationdebug" /> + <appender-ref ref="migrationerror" /> + </logger> + <logger name="org.onap.aai.dataexport" level="DEBUG" additivity="false"> <appender-ref ref="dataExportError"/> <appender-ref ref="dataExportDebug"/> - <appender-ref ref="dataExportMetric"/> + <appender-ref ref="dataExportAudit"/> <appender-ref ref="STDOUT"/> </logger> <logger name="org.apache" level="WARN" /> diff --git a/src/main/resources/org/janusgraph/graphdb/configuration/janusgraph.internal.properties b/src/main/resources/org/janusgraph/graphdb/configuration/janusgraph.internal.properties new file mode 100644 index 0000000..dcbbf83 --- /dev/null +++ b/src/main/resources/org/janusgraph/graphdb/configuration/janusgraph.internal.properties @@ -0,0 +1,6 @@ +# TODO: Please remove this file once all we move away from 0.2.3 +# The reason we need this is because we built our own 0.2.3-SNAPSHOT +# Janusgraph checks this file to see which it is compatible with + +janusgraph.compatible-versions=0.1.0,0.1.1,0.2.0,0.2.1,0.2.2,0.2.3-SNAPSHOT +janusgraph.version=0.2.3 diff --git a/src/main/resources/uniquePropertyCheck-logback.xml b/src/main/resources/uniquePropertyCheck-logback.xml index ca0c2c7..b4522d5 100644 --- a/src/main/resources/uniquePropertyCheck-logback.xml +++ b/src/main/resources/uniquePropertyCheck-logback.xml @@ -1,6 +1,21 @@ <configuration> <property name="logDirectory" value="${AJSC_HOME}/logs" /> + <property name="p_tim" value="%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}"/> + <property name="p_lvl" value="%level"/> + <property name="p_log" value="%logger"/> + <property name="p_mdc" value="%replace(%replace(%mdc){'\t','\\\\t'}){'\n', '\\\\n'}"/> + <property name="p_msg" value="%replace(%replace(%msg){'\t', '\\\\t'}){'\n','\\\\n'}"/> + <property name="p_exc" value="%replace(%replace(%rootException){'\t', '\\\\t'}){'\n','\\\\n'}"/> + <property name="p_mak" value="%replace(%replace(%marker){'\t', '\\\\t'}){'\n','\\\\n'}"/> + <property name="p_thr" value="%thread"/> + <property name="pattern" value="%nopexception${p_tim}\t${p_thr}\t${p_lvl}\t${p_log}\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t%n"/> + <property name="errorPattern" value="%X{LogTimestamp}|%X{RequestID}|%thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%.-5level|%X{ErrorCode}|%X{ErrorDesc}|%msg%n" /> + <property name="debugPattern" value="%X{LogTimestamp}|%X{RequestID}|%msg\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t|^%n" /> + <property name="auditPattern" value="%X{EntryTimestamp}|%X{LogTimestamp}|%X{RequestID}|%X{ServiceInstanceID}|%thread||%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}||||${p_mak}|${p_mdc}|||%msg%n" /> + <property name="metricPattern" value="%X{InvokeTimestamp}|%X{LogTimestamp}|%X{RequestID}|%X{ServiceInstanceID}|%thread||%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}||||%X{TargetVirtualEntity}|${p_mak}|${p_mdc}|||%msg%n" /> + <property name="transLogPattern" value="%X{LogTimestamp}|%date{yyyy-MM-dd'T'HH:mm:ss.SSSZ, UTC}|%X{RequestID}|%X{ServiceInstanceID}|%-10t|%X{ServerFQDN}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%replace(%replace(%X{ResponseDesc}){'\\|', '!'}){'\r|\n', '^'}|%X{InstanceUUID}|%level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{clientIpAddress}||%X{unused}|%X{processKey}|%X{customField1}|%X{customField2}|%X{customField3}|%X{customField4}|co=%X{PartnerName}:%m%n"/> + <appender name="uniquePropertyChecklog" class="ch.qos.logback.classic.sift.SiftingAppender"> <filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>INFO</level> @@ -24,7 +39,7 @@ </fileNamePattern> </rollingPolicy> <encoder> - <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%m%n</pattern> + <pattern>${auditPattern}</pattern> </encoder> </appender> </sift> @@ -48,14 +63,13 @@ <logger name="ch.qos.logback.core" level="WARN" additivity="false"> <appender-ref ref="uniquePropertyChecklog" /> </logger> - <logger name="com.att.eelf" level="WARN" additivity="false"> + <logger name="org.onap.aai" level="INFO" additivity="false"> <appender-ref ref="uniquePropertyChecklog" /> </logger> - <logger name="org.onap.aai" level="INFO" additivity="false"> + <logger name="org.onap.aai.util.UniquePropertyCheck" level="INFO" additivity="false"> <appender-ref ref="uniquePropertyChecklog" /> </logger> - <root level="INFO"> <appender-ref ref="uniquePropertyChecklog" /> </root> diff --git a/src/main/resources/updatePropertyTool-logback.xml b/src/main/resources/updatePropertyTool-logback.xml new file mode 100644 index 0000000..ab4edaf --- /dev/null +++ b/src/main/resources/updatePropertyTool-logback.xml @@ -0,0 +1,148 @@ +<!-- + + ============LICENSE_START======================================================= + org.onap.aai + ================================================================================ + Copyright © 2017 AT&T Intellectual Property. All rights reserved. + ================================================================================ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============LICENSE_END========================================================= + + ECOMP is a trademark and service mark of AT&T Intellectual Property. + +--> +<configuration> + <property name="logDirectory" value="${AJSC_HOME}/logs" /> + + <appender name="updatePropertyToollog" class="ch.qos.logback.classic.sift.SiftingAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <!-- This is MDC value --> + <!-- We will assign a value to 'logFilenameAppender' via Java code --> + <discriminator> + <key>logFilenameAppender</key> + <defaultValue>console</defaultValue> + </discriminator> + <sift> + <!-- A standard RollingFileAppender, the log file is based on 'logFileName' + at runtime --> + <appender name="FILE-${logFilenameAppender}" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/updatePropertyTool/${logFilenameAppender}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/updatePropertyTool/${logFilenameAppender}.log.%d{yyyy-MM-dd} + </fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%m%n</pattern> + </encoder> + </appender> + </sift> + </appender> + + <appender name="updatePropertyToolError" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>WARN</level> + </filter> + <File>${logDirectory}/updatePropertyTool/error.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/updatePropertyTool/error.log.%d{yyyy-MM-dd} + </fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%m%n</pattern> + </encoder> + </appender> + + <appender name="updatePropertyToolDebug" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>DEBUG</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/updatePropertyTool/debug.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/updatePropertyTool/debug.log.%d{yyyy-MM-dd} + </fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%m%n</pattern> + </encoder> + </appender> + + <appender name="updatePropertyToolMetric" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + <File>${logDirectory}/updatePropertyTool/metrics.log</File> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/updatePropertyTool/metrics.log.%d{yyyy-MM-dd} + </fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%m%n</pattern> + </encoder> + </appender> + + <logger name="org.reflections" level="ERROR" additivity="false"> + <appender-ref ref="updatePropertyToollog" /> + </logger> + <logger name="org.apache.zookeeper" level="ERROR" additivity="false"> + <appender-ref ref="updatePropertyToollog" /> + </logger> + <logger name="org.apache.hadoop" level="ERROR" additivity="false"> + <appender-ref ref="updatePropertyToollog" /> + </logger> + <logger name="org.janusgraph" level="ERROR" additivity="false"> + <appender-ref ref="updatePropertyToollog" /> + </logger> + <logger name="ch.qos.logback.classic" level="ERROR" additivity="false"> + <appender-ref ref="updatePropertyToollog" /> + </logger> + <logger name="ch.qos.logback.core" level="ERROR" additivity="false"> + <appender-ref ref="updatePropertyToollog" /> + </logger> + <logger name="com.att.eelf" level="ERROR" additivity="false"> + <appender-ref ref="updatePropertyToollog" /> + <appender-ref ref="updatePropertyToolError" /> + <appender-ref ref="updatePropertyToolMetric" /> + </logger> + <logger name="org.onap.aai" level="ERROR" additivity="false"> + <appender-ref ref="updatePropertyToollog" /> + <appender-ref ref="updatePropertyToolError" /> + <appender-ref ref="updatePropertyToolMetric" /> + <appender-ref ref="updatePropertyToolDebug" /> + </logger> + + <root level="INFO"> + <appender-ref ref="updatePropertyToollog" /> + <appender-ref ref="updatePropertyToolMetric" /> + </root> + <root level="ERROR"> + <appender-ref ref="updatePropertyToolError" /> + </root> + <root level="DEBUG"> + <appender-ref ref="updatePropertyToolDebug" /> + </root> + <root level="WARN"> + <appender-ref ref="updatePropertyToolMetric" /> + </root> +</configuration>
\ No newline at end of file diff --git a/src/main/scripts/add_vertex_label.sh b/src/main/scripts/add_vertex_label.sh new file mode 100644 index 0000000..f026bd0 --- /dev/null +++ b/src/main/scripts/add_vertex_label.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +filename=$1; + +if [ -z "${filename}" ]; then + echo "Please provide a graphson file"; + exit 1; +fi; + +if [ ! -f "${filename}" ]; then + echo "Unable to find the graphson file ${filename}"; + exit 1; +fi; + +sed 's/"label":"vertex"\(.*\)"aai-node-type":\[{"id":"\([^"]*\)","value":"\([^"]*\)"/"label":"\3"\1"aai-node-type":[{"id":"\2","value":"\3"/g' ${filename} > "with_label_${filename}";
\ No newline at end of file diff --git a/src/main/scripts/common_functions.sh b/src/main/scripts/common_functions.sh index 55fb516..f279334 100644 --- a/src/main/scripts/common_functions.sh +++ b/src/main/scripts/common_functions.sh @@ -41,7 +41,7 @@ execute_spring_jar(){ export SOURCE_NAME=$(grep '^schema.source.name=' ${PROJECT_HOME}/resources/application.properties | cut -d"=" -f2-); # Needed for the schema ingest library beans - eval $(grep '^schema\.' ${PROJECT_HOME}/resources/application.properties | \ + eval $(egrep '^(schema|server|history)\.' ${PROJECT_HOME}/resources/application.properties | \ sed 's/^\(.*\)$/JAVA_OPTS="$JAVA_OPTS -D\1"/g' | \ sed 's/${server.local.startpath}/${PROJECT_HOME}\/resources/g'| \ sed 's/${schema.source.name}/'${SOURCE_NAME}'/g'\ @@ -49,7 +49,7 @@ execute_spring_jar(){ JAVA_OPTS="${JAVA_OPTS} ${JAVA_POST_OPTS}"; - ${JAVA_HOME}/bin/java ${JVM_OPTS} ${JAVA_OPTS} -jar ${EXECUTABLE_JAR} "$@" || { + "${JAVA_HOME}/bin/java" ${JVM_OPTS} ${JAVA_OPTS} -jar ${EXECUTABLE_JAR} "$@" || { echo "Failed to run the tool $0 successfully"; exit 1; } diff --git a/src/main/scripts/dynamicPayloadGenerator.sh b/src/main/scripts/dynamicPayloadGenerator.sh index 2140354..323b161 100644 --- a/src/main/scripts/dynamicPayloadGenerator.sh +++ b/src/main/scripts/dynamicPayloadGenerator.sh @@ -31,7 +31,7 @@ # -s (optional) true or false to enable or disable schema, By default it is true for production, # you can change to false if the snapshot has duplicates # -c (optional) config file to use for loading snapshot into memory. -# -o (required) output file to store the data files +# -o (optional) output file to store the data files # -f (optional) PAYLOAD or DMAAP-MR # -n (optional) input file for the script # @@ -54,22 +54,22 @@ display_usage() { Usage: $0 [options] 1. Usage: dynamicPayloadGenerator -d <graphsonPath> -o <output-path> - 2. This script has 2 arguments that are required. + 2. This script has 1 argument that is required. a. -d (required) Name of the fully qualified Datasnapshot file that you need to load - b. -o (required) output file to store the data files + 3. Optional Parameters: a. -s (optional) true or false to enable or disable schema, By default it is true for production, b. -c (optional) config file to use for loading snapshot into memory. By default it is set to /opt/app/aai-graphadmin/resources/etc/appprops/dynamic.properties c. -f (optional) PAYLOAD or DMAAP-MR d. -n (optional) input file specifying the nodes and relationships to export. Default: /opt/app/aai-graphadmin/scriptdata/tenant_isolation/nodes.json - e. -m (optional) true or false to read multiple snapshots or not, by default is false - f. -i (optional) the file containing the input filters based on node property and regex/value. By default, it is: /opt/app/aai-graphadmin/scriptdata/tenant_isolation/inputFilters.json + e. -i (optional) the file containing the input filters based on node property and regex/value. By default, it is: /opt/app/aai-graphadmin/scriptdata/tenant_isolation/inputFilters.json + f. -o (optional) output directory to store the data files 4. For example (there are many valid ways to use it): dynamicPayloadGenerator.sh -d '/opt/app/snapshots/snaphot.graphSON' -o '/opt/app/aai-graphadmin/resources/etc/scriptdata/addmanualdata/tenant_isolation/' dynamicPayloadGenerator.sh -d '/opt/app/snapshots/snaphot.graphSON' -s false -c '/opt/app/aai-graphadmin/resources/etc/appprops/dynamic.properties' -o '/opt/app/aai-graphadmin/resources/etc/scriptdata/addmanualdata/tenant_isolation/' -f PAYLOAD -n '/opt/app/aai-graphadmin/resources/etc/scriptdata/tenant_isolation/nodes.json' - -m false -i '/opt/app/aai-graphadmin/resources/etc/scriptdata/tenant_isolation/inputFilters.json' + -i '/opt/app/aai-graphadmin/resources/etc/scriptdata/tenant_isolation/inputFilters.json' EOF } @@ -86,7 +86,7 @@ check_user; source_profile; export JVM_OPTS="-Xmx9000m -Xms9000m" -while getopts ":f:s:d:n:c:i:m:o:p:" opt; do +while getopts ":f:s:d:n:c:i:o:" opt; do case ${opt} in f ) PAYLOAD=$OPTARG @@ -112,14 +112,6 @@ while getopts ":f:s:d:n:c:i:m:o:p:" opt; do INPUT_FILTER_FILE=$OPTARG echo ${opt} ;; - m ) - MULTIPLE_SNAPSHOTS=$OPTARG - echo ${opt} - ;; - p ) - PARTIAL=$OPTARG - echo ${opt} - ;; o ) OUTPUT_DIR=$OPTARG echo ${opt} @@ -145,11 +137,42 @@ for nodeType in ${nodes[@]} grep "aai-node-type.*\"value\":\"$nodeType\"" $INPUT_DATASNAPSHOT_FILE'.P'* >>$INPUT_DATASNAPSHOT_FILE'.out' cat $INPUT_DATASNAPSHOT_FILE'.out' | cut -d':' -f2- > $INPUT_DATASNAPSHOT_FILE'.partial' done +if [ -z ${OUTPUT_DIR} ] +then + OUTPUT_DIR=${PROJECT_HOME}/data/scriptdata/addmanualdata/tenant_isolation/payload +fi - -execute_spring_jar org.onap.aai.dbgen.DynamicPayloadGenerator ${PROJECT_HOME}/resources/dynamicPayloadGenerator-logback.xml -s ${VALIDATE_SCHEMA} \ - -f ${PAYLOAD} -o ${OUTPUT_DIR} -c ${DYNAMIC_CONFIG_FILE} -i ${INPUT_FILTER_FILE} -m ${MULTIPLE_SNAPSHOTS} \ - -d ${INPUT_DATASNAPSHOT_FILE} -n ${NODE_CONFIG_FILE} ; - +# Build the command +COMMAND="execute_spring_jar org.onap.aai.dbgen.DynamicPayloadGenerator ${PROJECT_HOME}/resources/dynamicPayloadGenerator-logback.xml" +if [ ! -z ${VALIDATE_SCHEMA} ] +then + COMMAND="${COMMAND} -s ${VALIDATE_SCHEMA}" +fi +if [ ! -z ${PAYLOAD} ] +then + COMMAND="${COMMAND} -f ${PAYLOAD}" +fi +if [ ! -z ${INPUT_FILTER_FILE} ] +then + COMMAND="${COMMAND} -i ${INPUT_FILTER_FILE}" +fi +if [ ! -z ${NODE_CONFIG_FILE} ] +then + COMMAND="${COMMAND} -n ${NODE_CONFIG_FILE}" +fi +if [ ! -z ${INPUT_DATASNAPSHOT_FILE} ] +then + COMMAND="${COMMAND} -d ${INPUT_DATASNAPSHOT_FILE}" +else + display_usage + exit 1 +fi +# Removing the multiple snapshot option because there is just one .partial file +# (-m ${MULTIPLE_SNAPSHOTS}) +# The class only needs to read the ".partial" file and the default value for multiple snapshots is false if you don't pass it +#execute_spring_jar org.onap.aai.dbgen.DynamicPayloadGenerator ${PROJECT_HOME}/resources/dynamicPayloadGenerator-logback.xml -s ${VALIDATE_SCHEMA} \ +# -f ${PAYLOAD} -o ${OUTPUT_DIR} -c ${DYNAMIC_CONFIG_FILE} -i ${INPUT_FILTER_FILE} -m ${MULTIPLE_SNAPSHOTS} \ +# -d ${INPUT_DATASNAPSHOT_FILE} -n ${NODE_CONFIG_FILE} ; +${COMMAND}; end_date; exit 0 diff --git a/src/main/scripts/historyCreateDBSchema.sh b/src/main/scripts/historyCreateDBSchema.sh new file mode 100644 index 0000000..7a08a68 --- /dev/null +++ b/src/main/scripts/historyCreateDBSchema.sh @@ -0,0 +1,37 @@ +#!/bin/ksh +# +# ============LICENSE_START======================================================= +# org.onap.aai +# ================================================================================ +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# The script invokes GenTester java class to create the DB schema +# +# + +COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) +. ${COMMON_ENV_PATH}/common_functions.sh +start_date; +check_user; +source_profile; +if [ -z "$1" ]; then + execute_spring_jar org.onap.aai.schema.GenTester4Hist ${PROJECT_HOME}/resources/logback.xml +else + execute_spring_jar org.onap.aai.schema.GenTester4Hist ${PROJECT_HOME}/resources/logback.xml "$1" +fi; +end_date; +exit 0
\ No newline at end of file diff --git a/src/main/scripts/historyDbInitialLoad.sh b/src/main/scripts/historyDbInitialLoad.sh new file mode 100644 index 0000000..1341b86 --- /dev/null +++ b/src/main/scripts/historyDbInitialLoad.sh @@ -0,0 +1,54 @@ +#!/bin/ksh +# +# This script uses "history" versions of dataSnapshot and SchemaGenerator (via genTester) +# java classes to do the INITIAL load of a history database based on regular dataSnapShot +# files (assumed to be 'clean') from an existing non-history database. +# Steps: +# 1) Make sure the db is empty: clear out any existing data and schema. +# 2) rebuild the schema (using the SchemaGenerator4Hist) +# 3) reload data from the passed-in datafiles (which must found in the dataSnapShots directory and +# contain a json view of the db data). +# + +COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) +. ${COMMON_ENV_PATH}/common_functions.sh + +start_date; +check_user; + +if [ "$#" -lt 1 ]; then + echo "Illegal number of parameters" + echo "usage: $0 base_snapshot_filename" + exit 1 +fi + +source_profile; +export JAVA_PRE_OPTS=${JAVA_PRE_OPTS:--Xms6g -Xmx8g}; + +#### Step 1) Make sure the target database is cleared +echo "---- First Step: clear the db ----" +execute_spring_jar org.onap.aai.datasnapshot.DataSnapshot4HistInit ${PROJECT_HOME}/resources/logback.xml "-c" "CLEAR_ENTIRE_DATABASE" "-f" "$1" +if [ "$?" -ne "0" ]; then + echo "Problem clearing out database." + exit 1 +fi + +#### Step 2) rebuild the db-schema +echo "---- Second Step: rebuild the db schema ----" +execute_spring_jar org.onap.aai.schema.GenTester4Hist ${PROJECT_HOME}/resources/logback.xml "GEN_DB_WITH_NO_DEFAULT_CR" +if [ "$?" -ne "0" ]; then + echo "Problem rebuilding the schema (SchemaGenerator4Hist)." + exit 1 +fi + +#### Step 3) load the data from snapshot files +echo "---- Third Step: Load data from snapshot files ----" +execute_spring_jar org.onap.aai.datasnapshot.DataSnapshot4HistInit ${PROJECT_HOME}/resources/logback.xml "-c" "MULTITHREAD_RELOAD" "-f" "$1" +if [ "$?" -ne "0" ]; then + echo "Problem reloading data into the database." + end_date; + exit 1 +fi + +end_date; +exit 0 diff --git a/src/main/scripts/historySchemaMod.sh b/src/main/scripts/historySchemaMod.sh new file mode 100644 index 0000000..c098f0e --- /dev/null +++ b/src/main/scripts/historySchemaMod.sh @@ -0,0 +1,44 @@ +#!/bin/ksh +# +# This script is used to correct mistakes made in the database schema. +# It currently just allows you to change either the dataType and/or indexType on properties used by nodes. +# +# NOTE - This script is for the History db. That is different than the +# regular schemaMod in these two ways: 1) it will never create a unique index. +# Indexes can be created, but they will never be defined as unique. +# 2) the last parameter (preserveDataFlag) is ignored since for history, we do +# not want to 'migrate' old data. Old data should not disappear or change. +# +# +# To use this script, you need to pass four parameters: +# propertyName -- the name of the property that you need to change either the index or dataType on +# targetDataType -- whether it's changing or not, you need to give it: String, Integer, Boolean or Long +# targetIndexInfo -- whether it's changing or not, you need to give it: index, noIndex or uniqueIndex +# preserveDataFlag -- true or false. The only reason I can think of why you'd ever want to +# set this to false would be maybe if you were changing to an incompatible dataType so didn't +# want it to try to use the old data (and fail). But 99% of the time this will just be 'true'. +# +# Ie. historySchemaMod flavor-id String index true +# + +COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) +. ${COMMON_ENV_PATH}/common_functions.sh +start_date; +check_user; + +if [ "$#" -ne 4 ]; then + echo "Illegal number of parameters" + echo "usage: $0 propertyName targetDataType targetIndexInfo preserveDataFlag" + exit 1 +fi + +source_profile; +execute_spring_jar org.onap.aai.dbgen.schemamod.SchemaMod4Hist ${PROJECT_HOME}/resources/schemaMod-logback.xml "$1" "$2" "$3" "$4" +if [ "$?" -ne "0" ]; then + echo "Problem executing schemaMod " + end_date; + exit 1 +fi + +end_date; +exit 0 diff --git a/src/main/scripts/historyTruncateDb.sh b/src/main/scripts/historyTruncateDb.sh new file mode 100644 index 0000000..b0ad39e --- /dev/null +++ b/src/main/scripts/historyTruncateDb.sh @@ -0,0 +1,53 @@ +#!/bin/ksh + +### +# ============LICENSE_START======================================================= +# org.onap.aai +# ================================================================================ +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +### +# +# +# historyTruncateDb.sh -- This tool is usually run from a cron. +# It uses the application.property "history.truncate.window.days" to look for +# and delete nodes and edges that have an end-ts earlier than the truncate window. +# Or, that can be over-ridden using the command line param, "-truncateWindowDays". +# That is - they were deleted from the 'real' database before the window. +# So, if the window is set to 90 days, we will delete all nodes and edges +# from the history db that were deleted from the real db more than 90 days ago. +# +# It also uses the property, "history.truncate.mode". Can be over-ridden using +# the command line property "-truncateMode" +# "LOG_ONLY" - look for candidate nodes/edges, but just log them (no deleting) +# "DELETE_AND_LOG" - like it says... does the deletes and logs what +# it deleted (node and edge properties) +# "SILENT_DELETE" - not entirely silent, but will pare the logs way back to +# just recording vertex and edge ids that are deleted. +# +# Ie. historyTruncateDb.sh -truncateWindowDays 60 -truncateMode LOG_ONLY +# +# + +COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) +. ${COMMON_ENV_PATH}/common_functions.sh + +start_date; +check_user; +source_profile; +execute_spring_jar org.onap.aai.historytruncate.HistoryTruncate ${PROJECT_HOME}/resources/logback.xml "$@" +end_date; + +exit 0
\ No newline at end of file diff --git a/src/main/scripts/resend-dmaap-events.sh b/src/main/scripts/resend-dmaap-events.sh new file mode 100644 index 0000000..2afa3a7 --- /dev/null +++ b/src/main/scripts/resend-dmaap-events.sh @@ -0,0 +1,362 @@ +#!/bin/bash + +### +# ============LICENSE_START======================================================= +# org.onap.aai +# ================================================================================ +# Copyright (C) 2017-18 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +### + +# +# resend-dmaap-events.sh -- This tool is used to resend dmaap events. +# On certain scenarios due to dns or other networking issue, if A&AI fails to publish events +# We need a mechanism to resend the dmaap events for objects that haven't modified since +# So if a pserver object event was supposed to be sent but got lost and a later dmaap event +# was sent out then we shouldn't be sending dmaap messages +# It identifies if a dmaap message was already sent by looking at the resource version +# of the dmaap object that was failed to sendand checks the snapshot and see if they are the same +# +# Parameters: +# +# -b, (required) <string> the base url for the dmaap server +# -e, (required) <file> filename containing the missed events +# -l, (optional) indicating that the script should be run it debug mode +# it will not send the dmaap messages to dmaap server +# but it will write to a file named resend_dmaap_server.out +# -p, (required) <string> the password for the dmaap server +# -s, (required) <file> containing the data snapshot graphson file to compare the resource versions against +# -u, (required) <string> the username for the dmaap server +# +# An example of how to use the tool: +# Please use right credentials and right dmaap server in the cluster +# +# ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 +# +# For each dmaap message in the example_events.txt, it will check +# against graphson and try to send it to the dmaap server +# If the example_events.txt contains two events one that wasn't sent to dmaap +# and the other that was already updated by another PUT/DELETE +# and the output of the run will look something like this: +# +# Output: +# Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f was sent +# Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f not sent +# +# If lets say, there was a username password issue, you will see something like this: +# Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f was not sent due to dmaap error, please check logs +# Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f not sent +# +# From the directory in which you are executing the script (not where the script is located) +# You can have it be located and executed in the same place +# Check for a file called resend_dmaap_error.log as it will give you more details on the error +# +# For testing purposes, if you are trying to run this script and don't want to actually +# send it to a dmaap server, then you can run either of the following: +# +# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 +# or +# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 +# +# Following will output what would have been sent out based on checking the datasnapshot with example_events.txt +# +# Output: +# Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f was sent +# Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f not sent +# +# Also it will write the dmaap events to a file called dmaap_messages.out that +# would have been sent out in the current directory where you are executing this script +# + +current_directory=$( cd "$(dirname "$0")" ; pwd -P ); +resend_error_log=${current_directory}/resend_dmaap_error.log +resend_output=${current_directory}/dmaap_messages.out + +# Prints the usage of the shell script +usage(){ + echo "Usage $0 [options...]"; + echo; + echo " -b, <string> the base url for the dmaap server"; + echo " -e, <file> filename containing the missed events"; + echo " -l, (optional) indicating that the script should be run it debug mode" + echo " it will not send the dmaap messages to dmaap server " + echo " but it will write to a file named resend_dmaap_server.out" + echo " -p, <string> the password for the dmaap server"; + echo " -s, <file> containing the data snapshot graphson file to compare the resource versions against"; + echo " -u, <string> the username for the dmaap server"; + echo; + echo; + echo " An example of how to use the tool:"; + echo " Please use right credentials and right dmaap server in the cluster"; + echo; + echo " ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905"; + echo; + echo " For each dmaap message in the example_events.txt, it will check "; + echo " against graphson and try to send it to the dmaap server"; + echo " If the example_events.txt contains two events one that wasn't sent to dmaap"; + echo " and the other that was already updated by another PUT/DELETE"; + echo " and the output of the run will look something like this:"; + echo; + echo " Output:"; + echo " Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f was sent"; + echo " Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f not sent"; + echo " "; + echo " If lets say, there was a username password issue, you will see something like this:"; + echo " Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f was not sent due to dmaap error, please check logs"; + echo " Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f not sent"; + echo; + echo " From the directory in which you are executing the script (not where the script is located)"; + echo " You can have it be located and executed in the same place "; + echo " Check for a file called resend_dmaap_error.log as it will give you more details on the error"; + echo; + echo " For testing purposes, if you are trying to run this script and don't want to actually"; + echo " send it to a dmaap server, then you can run either of the following:"; + echo; + echo " ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545"; + echo " or"; + echo " ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905"; + echo; + echo " Following will output what would have been sent out based on checking the datasnapshot with example_events.txt"; + echo; + echo " Output:"; + echo " Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f was sent"; + echo " Dmaap Event with Id 7f7d8a7b-4034-46f3-a969-d7e5cbcbf75f not sent"; + echo; + echo " Also it will write the dmaap events to a file called dmaap_messages.out that "; + echo " would have been sent out in the current directory where you are executing this script"; + exit; +} + +# Validate the arguments being passed by user +# Checks if the argument of the string is greater than zero +# Also check if the file actually exists +validate(){ + local type_of_file=$1; + + if [ $# -eq 0 ]; then + echo "Error expecting the validate method to have at least one argument indicating what type"; + exit -1; + fi; + + shift; + + local arg=$1; + + if [ -z "$arg" ]; then + echo "Error missing the expected argument for ${type_of_file}"; + exit -1; + fi; + + if [ ! -f "$arg" ]; then + echo "Error: file $arg cannot be found, please check the file again"; + exit -1; + fi; +} + +# Checks if the resource version in dmaap message passed for an aai-uri +# is the same as the value in the snapshot file for that version +# If the resource version is the same it will return 0 for success +# Otherwise it will return non zero to indicate that this method failed +resource_version_matches_snapshot_file(){ + + local snapshot_file=$1; + local entity_link=$2; + local resource_version=$3; + local action=$4; + + if [ -z ${resource_version} ]; then + echo "Missing the parameter resource version to be passed"; + return -1; + fi + + # Modify the entity link passed to remove the /aai/v[0-9]+ + aai_uri=$(echo $entity_link | sed 's/\/aai\/v[0-9][0-9]*//g'); + + local line=$(grep '"value":"'${aai_uri}'"' ${snapshot_file} 2> /dev/null); + + if [ -z "${line}" ] ; then + if [ "${action}" = "DELETE" ]; then + return 0; + else + return -1; + fi; + fi; + + cnt=$(echo $line | grep -o '"resource-version":\[{"id":"[^"]*","value":"'$resource_version'"}\]' | wc -l); + + if [ $cnt -eq 1 ]; then + return 0; + else + return -1; + fi; +} + +# From a array being passed, it will determine the smallest element +# and return the index of the smallest element +# If the array length is zero, then it will return -1 +retrieve_smallest_index(){ + + local elements=("${@}"); + + if [ ${#elements} -eq 0 ]; then + return -1; + fi; + + local smallest_element=${elements[0]}; + + local index=0; + local smallest_index=0; + + for element in ${elements[@]}; do + if [ $element -lt $smallest_element ]; then + smallest_index=${index}; + fi; + index=$((index+1)); + done; + + return ${smallest_index}; +} + +# Send the dmaap event to the host based on +# the line that was send to the function +send_dmaap(){ + + local local_mode=$1; + local line=$2; + local username=$3; + local password=$4; + local baseurl=$5; + local resp_code=0; + + generated_file=$(uuidgen); + + local json_file=/tmp/${generated_file}.json; + local curl_output=/tmp/${generated_file}.txt; + + echo ${line} > ${json_file}; + > ${curl_output}; + id=$(echo $line | grep -o '"id":"[^"]*"' | cut -d":" -f2- | sed 's/"//g'); + + if [ "$local_mode" = true ]; then + echo $line >> ${resend_output}; + else + + response_code=$(curl \ + -k -o ${curl_output} -s -w "%{http_code}\n" \ + -u "${username}:${password}" \ + -X POST \ + -H "Content-Type: application/json" \ + -d "@${json_file}" \ + "${baseurl}/events/AAI-EVENT"\ + ); + + if [ "$response_code" -ne "200" ]; then + echo -n "Response failure for dmaap message with id ${id}," >> ${resend_error_log}; + echo " code: ${response_code} body: $(cat ${curl_output})" >> ${resend_error_log}; + resp_code=-1; + fi; + fi; + + if [ -f "${json_file}" ]; then + rm $json_file; + fi; + + if [ -f "${curl_output}" ]; then + rm $curl_output; + fi; + + return ${resp_code}; +} + +# Validates the events file and the snapshot file +# Goes through each line in the missed events file +# Gets all the resource versions there are +# Finds the smallest resource version there +# checks if the smallest resource version for the aai uri +# is what is currently in the last snapshot file provided by user +# If it is, it will send an dmaap event out + +main(){ + + if [ "${#}" -eq 0 ]; then + usage; + fi; + + # Get the first character of the first command line argument + # If the character doesn't start with dash (-) + # Then fail the script and display usage + + if [ "${1:0:1}" != "-" ]; then + echo "Invalid option: $1" >&2 + usage; + fi; + + while getopts ":e:s:u:lp:b:h" opt; do + case ${opt} in + l ) # Specify that the application will not send messages to dmaap but save it a file + local local_mode=true + ;; + e ) # Specify the file for missed events + local missed_events_file=$OPTARG + ;; + s ) # Specify the file for snapshot + local snapshot_file=$OPTARG + ;; + u ) # Specify the username to dmaap + local username=$OPTARG + ;; + p ) # Specify the password to dmaap + local password=$OPTARG + ;; + b ) # Specify the baseurl to dmaap + local hostname=$OPTARG + ;; + h ) + usage; + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage; + ;; + esac + done; + + validate "events_file" $missed_events_file; + validate "snapshot_file" $snapshot_file; + + if [ "$local_mode" = true ]; then + > ${resend_output}; + fi; + + while read dmaap_event; do + entity_link=$(echo $dmaap_event | grep -o '"entity-link":"[^"]*"' | cut -d":" -f2- | sed 's/"//g'); + id=$(echo $dmaap_event | grep -o '"id":"[^"]*"' | cut -d":" -f2- | sed 's/"//g'); + action=$(echo $dmaap_event | grep -o '"action":"[^"]*"' | cut -d":" -f2- | sed 's/"//g'); + smallest_resource_version=$(echo $dmaap_event | jq -M '.' | grep 'resource-version' | sort | tail -1 | sed 's/[^0-9]//g'); + resource_version_matches_snapshot_file "${snapshot_file}" "${entity_link}" "${smallest_resource_version}" "${action}" && { + send_dmaap "${local_mode}" "$dmaap_event" "$username" "$password" "$hostname" && { + echo "Dmaap Event with Id $id was sent"; + } || { + echo "Dmaap Event with Id $id was not sent due to dmaap error, please check logs"; + } + } || { + echo "Dmaap Event with Id $id not sent"; + } + + done < ${missed_events_file}; + +} + +main $@ diff --git a/src/main/scripts/schemaMod.sh b/src/main/scripts/schemaMod.sh index d1fb009..c7b8ce9 100644 --- a/src/main/scripts/schemaMod.sh +++ b/src/main/scripts/schemaMod.sh @@ -3,20 +3,17 @@ # This script is used to correct mistakes made in the database schema. # It currently just allows you to change either the dataType and/or indexType on properties used by nodes. # -# NOTE - Titan is not elegant in 0.5.3 about making changes to the schema. Bad properties never -# actually leave the database, they just get renamed and stop getting used. So it is -# really worthwhile to get indexes and dataTypes correct the first time around. +# NOTE - JanusGraph is not particularly elegant in about making changes to the schema. +# So it is really worthwhile to get indexes and dataTypes correct the first time around. # Note also - This script just makes changes to the schema that is currently live. # If you were to create a new schema in a brandy-new environment, it would look like -# whatever ex5.json (as of June 2015) told it to look like. So, part of making a -# change to the db schema should Always first be to make the change in ex5.json so that -# future environments will have the change. This script is just to change existing -# instances of the schema since schemaGenerator (as of June 2015) does not update things - it -# just does the initial creation. +# whatever our OXM files told it to look like. So, part of making a +# change to the live db schema should Always first be to make the change in the appropriate +# OXM schema file so that future environments will have the change. This script is +# just to change existing instances of the schema since schemaGenerator does not +# update things - it just does the initial creation. # -# Boy, this is getting to be a big comment section... -# -# To use this script, you need to pass four parameters: +# To use this script, there are 4 required parameters, and one optional: # propertyName -- the name of the property that you need to change either the index or dataType on # targetDataType -- whether it's changing or not, you need to give it: String, Integer, Boolean or Long # targetIndexInfo -- whether it's changing or not, you need to give it: index, noIndex or uniqueIndex @@ -24,7 +21,11 @@ # set this to false would be maybe if you were changing to an incompatible dataType so didn't # want it to try to use the old data (and fail). But 99% of the time this will just be 'true'. # +# commitBlockSize -- OPTIONAL -- how many updates to commit at once. +# Default will be used if no value is passed. +# # Ie. schemaMod flavor-id String index true +# or, schemaMod flavor-id String noIndex true 50000 # COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) @@ -32,14 +33,14 @@ COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) start_date; check_user; -if [ "$#" -ne 4 ]; then +if [ "$#" -ne 4 ] && [ "$#" -ne 5 ]; then echo "Illegal number of parameters" - echo "usage: $0 propertyName targetDataType targetIndexInfo preserveDataFlag" + echo "usage: $0 propertyName targetDataType targetIndexInfo preserveDataFlag [blockSize]" exit 1 fi source_profile; -execute_spring_jar org.onap.aai.dbgen.schemamod.SchemaMod ${PROJECT_HOME}/resources/schemaMod-logback.xml "$1" "$2" "$3" "$4" +execute_spring_jar org.onap.aai.dbgen.schemamod.SchemaMod ${PROJECT_HOME}/resources/schemaMod-logback.xml "$@" if [ "$?" -ne "0" ]; then echo "Problem executing schemaMod " end_date; diff --git a/src/main/scripts/updatePropertyTool.sh b/src/main/scripts/updatePropertyTool.sh new file mode 100644 index 0000000..7e53a3f --- /dev/null +++ b/src/main/scripts/updatePropertyTool.sh @@ -0,0 +1,58 @@ +#!/bin/ksh +# +# ============LICENSE_START======================================================= +# org.onap.aai +# ================================================================================ +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +# updatePropertyTool.sh -- This tool is used to update properties in corrupt vertices +# in the event that an update or delete occurs to a node simultaneously, resulting +# in inconsistent data. Updating the aai-uri can reset the index and restore +# the GET information on the node. +# +# Parameters: +# +# At least one of following two parameters are required +# The following values are needed to identify the node(s) to be updated +# --filename, -f filename of a .txt extension required with a list of vertexIds. Vertex Ids must be separated line after line in text file. +# --vertexId, -v option that may occur multiple times as entries of a list +# +# --property, -p (required) value to be updated in the corrupted node +# --help, -h (optional) used to display help on usage of the function +# +# +# For example: +# +# updatePropertyTool.sh --filename myFile.txt --vertexId 123 --property myProperty +# updatePropertyTool.sh --filename myFile.txt --vertexId 123 --vertexId 456 --property myProperty +# updatePropertyTool.sh -f myFile.txt --vertexId 123 -v 456 -p myProperty +# updatePropertyTool.sh -f myFile.txt -p -myProperty +# updatePropertyTool.sh -v 123 -v 456 -p -myProperty +# updatePropertyTool.sh -v 123 -p -myProperty +# +COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) +. ${COMMON_ENV_PATH}/common_functions.sh + +start_date; + +check_user; +source_profile; +execute_spring_jar org.onap.aai.dbgen.UpdatePropertyTool ${PROJECT_HOME}/resources/updatePropertyTool-logback.xml "$@" +end_date; + +exit 0 |