aboutsummaryrefslogtreecommitdiffstats
path: root/src/main/java
diff options
context:
space:
mode:
authorLaMont, William(wl2432) <wl2432@att.com>2020-02-28 16:48:41 -0500
committerLaMont, William(wl2432) <wl2432@att.com>2020-03-03 18:10:37 -0500
commit9b65b7126bc77c18c196ed2b6e625a5a864c98dc (patch)
tree7e0dd55538ad2af20c414369f628134e9f3191ae /src/main/java
parent30282f9e3f6ab0ed9599f5bb2e2c743cbd099089 (diff)
support for v17 through v19
Issue-ID: AAI-2751 Change-Id: I75c9b2057065e1374f25e2b6b12de0a8003ac781 Signed-off-by: LaMont, William(wl2432) <wl2432@att.com>
Diffstat (limited to 'src/main/java')
-rw-r--r--src/main/java/org/onap/aai/GraphAdminApp.java55
-rw-r--r--src/main/java/org/onap/aai/aailog/logs/AaiDBGraphadminMetricLog.java115
-rw-r--r--src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java92
-rw-r--r--src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java102
-rw-r--r--src/main/java/org/onap/aai/dataexport/DataExportTasks.java72
-rw-r--r--src/main/java/org/onap/aai/datagrooming/DataGrooming.java289
-rw-r--r--src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java51
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java270
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java1079
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java49
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java7
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader4HistInit.java454
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java24
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java181
-rw-r--r--src/main/java/org/onap/aai/db/schema/AuditOXM.java6
-rw-r--r--src/main/java/org/onap/aai/db/schema/ScriptDriver.java15
-rw-r--r--src/main/java/org/onap/aai/dbgen/DupeTool.java193
-rw-r--r--src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java1805
-rw-r--r--src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java162
-rw-r--r--src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java6
-rw-r--r--src/main/java/org/onap/aai/dbgen/UpdatePropertyTool.java79
-rw-r--r--src/main/java/org/onap/aai/dbgen/UpdatePropertyToolInternal.java383
-rw-r--r--src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java91
-rw-r--r--src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java200
-rw-r--r--src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java317
-rw-r--r--src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java181
-rw-r--r--src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternalBatch.java451
-rw-r--r--src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java518
-rw-r--r--src/main/java/org/onap/aai/historytruncate/HistoryTruncateTasks.java108
-rw-r--r--src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java98
-rw-r--r--src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java6
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java64
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java6
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java75
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java2
-rw-r--r--src/main/java/org/onap/aai/migration/EdgeMigrator.java2
-rw-r--r--src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java16
-rw-r--r--src/main/java/org/onap/aai/migration/MigrationController.java14
-rw-r--r--src/main/java/org/onap/aai/migration/MigrationControllerInternal.java130
-rw-r--r--src/main/java/org/onap/aai/migration/Migrator.java12
-rw-r--r--src/main/java/org/onap/aai/migration/NotificationHelper.java6
-rw-r--r--src/main/java/org/onap/aai/migration/PropertyMigrator.java14
-rw-r--r--src/main/java/org/onap/aai/migration/RebuildAllEdges.java12
-rw-r--r--src/main/java/org/onap/aai/migration/ValueMigrator.java14
-rw-r--r--src/main/java/org/onap/aai/migration/VertexMerge.java8
-rw-r--r--src/main/java/org/onap/aai/migration/v16/MigrateBooleanDefaultsToFalse.java78
-rw-r--r--src/main/java/org/onap/aai/migration/v16/MigrateInMaintDefaultToFalse.java82
-rw-r--r--src/main/java/org/onap/aai/rest/QueryConsumer.java30
-rw-r--r--src/main/java/org/onap/aai/rest/dsl/DslListener.java6
-rw-r--r--src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java6
-rw-r--r--src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java13
-rw-r--r--src/main/java/org/onap/aai/schema/GenTester.java51
-rw-r--r--src/main/java/org/onap/aai/schema/GenTester4Hist.java175
-rw-r--r--src/main/java/org/onap/aai/service/AuthorizationService.java8
-rw-r--r--src/main/java/org/onap/aai/util/ExceptionTranslator.java34
-rw-r--r--src/main/java/org/onap/aai/util/GraphAdminConstants.java4
-rw-r--r--src/main/java/org/onap/aai/util/GraphAdminDBUtils.java6
-rw-r--r--src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java30
-rw-r--r--src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java12
-rw-r--r--src/main/java/org/onap/aai/util/SendMigrationNotifications.java24
-rw-r--r--src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java14
-rw-r--r--src/main/java/org/onap/aai/util/UniquePropertyCheck.java565
-rw-r--r--src/main/java/org/onap/aai/web/JerseyConfiguration.java75
63 files changed, 6166 insertions, 2881 deletions
diff --git a/src/main/java/org/onap/aai/GraphAdminApp.java b/src/main/java/org/onap/aai/GraphAdminApp.java
index 3b4a377..1030f3a 100644
--- a/src/main/java/org/onap/aai/GraphAdminApp.java
+++ b/src/main/java/org/onap/aai/GraphAdminApp.java
@@ -20,16 +20,16 @@
package org.onap.aai;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.onap.aai.aailog.logs.AaiDebugLog;
import org.onap.aai.config.PropertyPasswordConfiguration;
import org.onap.aai.dbmap.AAIGraph;
import java.util.Properties;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.logging.ErrorLogHelper;
import org.onap.aai.logging.LogFormatTools;
-import org.onap.aai.logging.LoggingContext;
import org.onap.aai.nodes.NodeIngestor;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
@@ -54,17 +54,19 @@ import java.util.UUID;
// This will add the ScheduledTask that was created in aai-common
// Add more packages where you would need to scan for files
@ComponentScan(basePackages = {
- "org.onap.aai.tasks",
- "org.onap.aai.config",
- "org.onap.aai.service",
- "org.onap.aai.setup",
- "org.onap.aai.rest",
- "org.onap.aai.web",
- "org.onap.aai.interceptors",
- "org.onap.aai.datasnapshot",
- "org.onap.aai.datagrooming",
- "org.onap.aai.dataexport",
- "org.onap.aai.datacleanup"
+ "org.onap.aai.tasks",
+ "org.onap.aai.config",
+ "org.onap.aai.service",
+ "org.onap.aai.setup",
+ "org.onap.aai.aaf",
+ "org.onap.aai.rest",
+ "org.onap.aai.web",
+ "org.onap.aai.interceptors",
+ "org.onap.aai.datasnapshot",
+ "org.onap.aai.datagrooming",
+ "org.onap.aai.dataexport",
+ "org.onap.aai.datacleanup",
+ "org.onap.aai.aailog"
})
@EnableAsync
@EnableScheduling
@@ -72,10 +74,16 @@ import java.util.UUID;
public class GraphAdminApp {
public static final String APP_NAME = "GraphAdmin";
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(GraphAdminApp.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(GraphAdminApp.class);
private static final String FROMAPPID = "AAI-GA";
private static final String TRANSID = UUID.randomUUID().toString();
+
+ private static AaiDebugLog debugLog = new AaiDebugLog();
+ static {
+ debugLog.setupMDC();
+ }
+
@Autowired
private Environment env;
@@ -86,17 +94,6 @@ public class GraphAdminApp {
@PostConstruct
private void initialize(){
loadDefaultProps();
- initializeLoggingContext();
- }
-
- private static void initializeLoggingContext() {
- LoggingContext.save();
- LoggingContext.component("init");
- LoggingContext.partnerName("NA");
- LoggingContext.targetEntity(APP_NAME);
- LoggingContext.requestId(UUID.randomUUID().toString());
- LoggingContext.serviceName(APP_NAME);
- LoggingContext.targetServiceName("contextInitialized");
}
@PreDestroy
@@ -107,8 +104,8 @@ public class GraphAdminApp {
public static void main(String[] args) throws Exception {
loadDefaultProps();
+
ErrorLogHelper.loadProperties();
- initializeLoggingContext();
Environment env =null;
AAIConfig.init();
@@ -121,9 +118,6 @@ public class GraphAdminApp {
catch(Exception ex){
AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(ex);
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("Problems starting GraphAdminApp "+aai.getMessage());
ErrorLogHelper.logException(aai);
ErrorLogHelper.logError(aai.getCode(), ex.getMessage() + ", resolve and restart GraphAdmin");
throw aai;
@@ -141,7 +135,6 @@ public class GraphAdminApp {
System.setProperty("org.onap.aai.graphadmin.started", "true");
LOGGER.info("GraphAdmin MicroService Started");
- LOGGER.error("GraphAdmin MicroService Started");
LOGGER.debug("GraphAdmin MicroService Started");
System.out.println("GraphAdmin Microservice Started");
}
diff --git a/src/main/java/org/onap/aai/aailog/logs/AaiDBGraphadminMetricLog.java b/src/main/java/org/onap/aai/aailog/logs/AaiDBGraphadminMetricLog.java
new file mode 100644
index 0000000..5b761cc
--- /dev/null
+++ b/src/main/java/org/onap/aai/aailog/logs/AaiDBGraphadminMetricLog.java
@@ -0,0 +1,115 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.aai.aailog.logs;
+
+import org.onap.aai.util.AAIConstants;
+import org.onap.logging.filter.base.Constants;
+import org.onap.logging.filter.base.MDCSetup;
+import org.onap.logging.filter.base.ONAPComponents;
+import org.onap.logging.ref.slf4j.ONAPLogConstants;
+import org.slf4j.*;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.net.URI;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Optional;
+
+public class AaiDBGraphadminMetricLog extends MDCSetup {
+
+ protected static final Logger logger = LoggerFactory.getLogger(AaiDBGraphadminMetricLog.class);
+ private final String partnerName;
+ private static final Marker INVOKE_RETURN = MarkerFactory.getMarker("INVOKE-RETURN");
+ private static final String TARGET_ENTITY = ONAPComponents.AAI.toString() + ".DB";
+ public AaiDBGraphadminMetricLog(String subcomponent) {
+ partnerName = getPartnerName(subcomponent);
+ }
+
+
+ protected String getTargetServiceName(Optional<URI> uri) {
+ return (getServiceName(uri));
+ }
+
+ protected String getServiceName(Optional<URI> uri) {
+ String serviceName = Constants.DefaultValues.UNKNOWN;
+ if (uri.isPresent()) {
+ serviceName = uri.get().getPath();
+ if (serviceName != null && (!serviceName.isEmpty())) {
+ serviceName = serviceName.replaceAll(",", "\\\\,");
+ }
+ }
+ return serviceName;
+ }
+
+
+ protected String getTargetEntity(Optional<URI> uri) {
+ return TARGET_ENTITY;
+ }
+
+ protected String getPartnerName(@Value(AAIConstants.AAI_TRAVERSAL_MS) String subcomponent ) {
+ StringBuilder sb = new StringBuilder(ONAPComponents.AAI.toString()).append(subcomponent);
+ return (sb.toString());
+ }
+
+ public void pre(Optional<URI> uri) {
+ try {
+ setupMDC(uri);
+ setLogTimestamp();
+ logger.info(ONAPLogConstants.Markers.INVOKE, "Invoke");
+ } catch (Exception e) {
+ logger.warn("Error in AaiDBMetricLog pre", e.getMessage());
+ }
+ }
+
+ public void post() {
+ try {
+ setLogTimestamp();
+ setElapsedTimeInvokeTimestamp();
+ setResponseStatusCode(200);
+ setResponseDescription(200);
+ MDC.put(ONAPLogConstants.MDCs.RESPONSE_CODE, "200");
+ logger.info(INVOKE_RETURN, "InvokeReturn");
+ clearClientMDCs();
+ } catch (Exception e) {
+ logger.warn("Error in AaiDBMetricLog post", e.getMessage());
+ }
+ }
+
+ protected void setupMDC(Optional<URI> uri) {
+ MDC.put("InvokeTimestamp", ZonedDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
+ MDC.put("TargetServiceName", this.getTargetServiceName(uri));
+ MDC.put("StatusCode", ONAPLogConstants.ResponseStatus.INPROGRESS.toString());
+ this.setInvocationIdFromMDC();
+ if (MDC.get("TargetEntity") == null) {
+ String targetEntity = this.getTargetEntity(uri);
+ if (targetEntity != null) {
+ MDC.put("TargetEntity", targetEntity);
+ } else {
+ MDC.put("TargetEntity", "Unknown-Target-Entity");
+ }
+ }
+ if (MDC.get("ServiceName") == null) {
+ MDC.put("ServiceName", this.getServiceName(uri));
+ }
+ this.setServerFQDN();
+ }
+}
diff --git a/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java b/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java
index 6d97177..352e96f 100644
--- a/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java
+++ b/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java
@@ -20,27 +20,95 @@
*/
package org.onap.aai.config;
-import org.springframework.context.ApplicationContextInitializer;
-import org.springframework.context.ConfigurableApplicationContext;
-import org.springframework.core.env.ConfigurableEnvironment;
-import org.springframework.core.env.EnumerablePropertySource;
-import org.springframework.core.env.MapPropertySource;
-import org.springframework.core.env.PropertySource;
-
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.commons.io.IOUtils;
+import org.springframework.context.ApplicationContextInitializer;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.core.env.*;
+
public class PropertyPasswordConfiguration implements ApplicationContextInitializer<ConfigurableApplicationContext> {
private static final Pattern decodePasswordPattern = Pattern.compile("password\\((.*?)\\)");
-
private PasswordDecoder passwordDecoder = new JettyPasswordDecoder();
+ private static final Logger logger = LoggerFactory.getLogger(PropertyPasswordConfiguration.class.getName());
@Override
public void initialize(ConfigurableApplicationContext applicationContext) {
ConfigurableEnvironment environment = applicationContext.getEnvironment();
+ String certPath = environment.getProperty("server.certs.location");
+ File passwordFile = null;
+ File passphrasesFile = null;
+ InputStream passwordStream = null;
+ InputStream passphrasesStream = null;
+ Map<String, Object> sslProps = new LinkedHashMap<>();
+
+ // Override the passwords from application.properties if we find AAF certman files
+ if (certPath != null) {
+ try {
+ passwordFile = new File(certPath + ".password");
+ passwordStream = new FileInputStream(passwordFile);
+
+ if (passwordStream != null) {
+ String keystorePassword = null;
+
+ keystorePassword = IOUtils.toString(passwordStream);
+ if (keystorePassword != null) {
+ keystorePassword = keystorePassword.trim();
+ }
+ sslProps.put("server.ssl.key-store-password", keystorePassword);
+ sslProps.put("schema.service.ssl.key-store-password", keystorePassword);
+ } else {
+ logger.info("Not using AAF Certman password file");
+ }
+ } catch (IOException e) {
+ logger.warn("Not using AAF Certman password file, e=" + e.getMessage());
+ } finally {
+ if (passwordStream != null) {
+ try {
+ passwordStream.close();
+ } catch (Exception e) {
+ }
+ }
+ }
+ try {
+ passphrasesFile = new File(certPath + ".passphrases");
+ passphrasesStream = new FileInputStream(passphrasesFile);
+
+ if (passphrasesStream != null) {
+ String truststorePassword = null;
+ Properties passphrasesProps = new Properties();
+ passphrasesProps.load(passphrasesStream);
+ truststorePassword = passphrasesProps.getProperty("cadi_truststore_password");
+ if (truststorePassword != null) {
+ truststorePassword = truststorePassword.trim();
+ }
+ sslProps.put("server.ssl.trust-store-password", truststorePassword);
+ sslProps.put("schema.service.ssl.trust-store-password", truststorePassword);
+ } else {
+ logger.info("Not using AAF Certman passphrases file");
+ }
+ } catch (IOException e) {
+ logger.warn("Not using AAF Certman passphrases file, e=" + e.getMessage());
+ } finally {
+ if (passphrasesStream != null) {
+ try {
+ passphrasesStream.close();
+ } catch (Exception e) {
+ }
+ }
+ }
+ }
for (PropertySource<?> propertySource : environment.getPropertySources()) {
Map<String, Object> propertyOverrides = new LinkedHashMap<>();
decodePasswords(propertySource, propertyOverrides);
@@ -48,6 +116,12 @@ public class PropertyPasswordConfiguration implements ApplicationContextInitiali
PropertySource<?> decodedProperties = new MapPropertySource("decoded "+ propertySource.getName(), propertyOverrides);
environment.getPropertySources().addBefore(propertySource.getName(), decodedProperties);
}
+
+ }
+ if (!sslProps.isEmpty()) {
+ logger.info("Using AAF Certman files");
+ PropertySource<?> additionalProperties = new MapPropertySource("additionalProperties", sslProps);
+ environment.getPropertySources().addFirst(additionalProperties);
}
}
@@ -67,7 +141,7 @@ public class PropertyPasswordConfiguration implements ApplicationContextInitiali
private String decodePasswordsInString(String input) {
if (input == null) {
return null;
- };
+ }
StringBuffer output = new StringBuffer();
Matcher matcher = decodePasswordPattern.matcher(input);
while (matcher.find()) {
diff --git a/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java
index f4372c1..dff22a4 100644
--- a/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java
+++ b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java
@@ -28,26 +28,37 @@ import java.nio.file.attribute.FileTime;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
+import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
+import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LogFormatTools;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
+import org.onap.logging.filter.base.ONAPComponents;
+import org.onap.logging.ref.slf4j.ONAPLogConstants;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.PropertySource;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
@Component
@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties")
public class DataCleanupTasks {
-
- private static final EELFLogger logger = EELFManager.getInstance().getLogger(DataCleanupTasks.class);
+
+ @Autowired
+ private AaiScheduledTaskAuditLog auditLog;
+
+ private static final Logger logger = LoggerFactory.getLogger(DataCleanupTasks.class);
private final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd");
+
/**The function archives/deletes files that end in .out (Ie. dataGrooming.201511111305.out) that sit in our log/data directory structure.
logDir is the {project_home}/logs
archiveDir is the ARCHIVE directory where the files will be stored after 5 days.
@@ -56,8 +67,9 @@ public class DataCleanupTasks {
*/
@Scheduled(cron = "${datagroomingcleanup.cron}" )
public void dataGroomingCleanup() throws AAIException, Exception {
+ auditLog.logBefore("dataGroomingCleanup", ONAPComponents.AAI.toString() );
- logger.info("Started cron job dataGroomingCleanup @ " + simpleDateFormat.format(new Date()));
+ logger.debug("Started cron job dataGroomingCleanup @ " + simpleDateFormat.format(new Date()));
try {
String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs";
@@ -68,15 +80,15 @@ public class DataCleanupTasks {
File archivepath = new File(archiveDir);
File dataGroomingPath = new File(dataGroomingArcDir);
- logger.info("The logDir is " + logDir);
- logger.info("The dataGroomingDir is " + dataGroomingDir);
- logger.info("The archiveDir is " + archiveDir );
- logger.info("The dataGroomingArcDir is " + dataGroomingArcDir );
+ logger.debug("The logDir is " + logDir);
+ logger.debug("The dataGroomingDir is " + dataGroomingDir);
+ logger.debug("The archiveDir is " + archiveDir );
+ logger.debug("The dataGroomingArcDir is " + dataGroomingArcDir );
boolean exists = directoryExists(logDir);
- logger.info("Directory" + logDir + "exists: " + exists);
+ logger.debug("Directory" + logDir + "exists: " + exists);
if(!exists)
- logger.error("The directory" + logDir +"does not exists");
+ logger.debug("The directory" + logDir +"does not exists");
Integer ageZip = AAIConfig.getInt("aai.datagrooming.agezip");
Integer ageDelete = AAIConfig.getInt("aai.datagrooming.agedelete");
@@ -91,9 +103,9 @@ public class DataCleanupTasks {
continue;
}
if(listFile.isFile()){
- logger.info("The file name in dataGrooming: " +listFile.getName());
+ logger.debug("The file name in dataGrooming: " +listFile.getName());
Date fileCreateDate = fileCreationMonthDate(listFile);
- logger.info("The fileCreateDate in dataGrooming is " + fileCreateDate);
+ logger.debug("The fileCreateDate in dataGrooming is " + fileCreateDate);
if( fileCreateDate.compareTo(newAgeZip) < 0) {
archive(listFile,archiveDir,dataGroomingArcDir);
}
@@ -107,9 +119,9 @@ public class DataCleanupTasks {
if(listFilesArchive != null) {
for(File listFileArchive : listFilesArchive) {
if(listFileArchive.isFile()) {
- logger.info("The file name in ARCHIVE/dataGrooming: " +listFileArchive.getName());
+ logger.debug("The file name in ARCHIVE/dataGrooming: " +listFileArchive.getName());
Date fileCreateDate = fileCreationMonthDate(listFileArchive);
- logger.info("The fileCreateDate in ARCHIVE/dataGrooming is " + fileCreateDate);
+ logger.debug("The fileCreateDate in ARCHIVE/dataGrooming is " + fileCreateDate);
if(fileCreateDate.compareTo(newAgeDelete) < 0) {
delete(listFileArchive);
}
@@ -118,10 +130,11 @@ public class DataCleanupTasks {
}
}
catch (Exception e) {
- ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
- logger.info("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
- throw e;
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e));
+ logger.debug("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e));
}
+ logger.debug("Ended cron job dataGroomingCleanup @ " + simpleDateFormat.format(new Date()));
+ auditLog.logAfter();
}
/**
@@ -142,11 +155,11 @@ public class DataCleanupTasks {
public Date getZipDate(Integer days, Date date) throws Exception{
Calendar cal = Calendar.getInstance();
- logger.info("The current date is " + date );
+ logger.debug("The current date is " + date );
cal.setTime(date);
cal.add(Calendar.DATE, -days);
Date newAgeZip = cal.getTime();
- logger.info("The newAgeDate is " +newAgeZip);
+ logger.debug("The newAgeDate is " +newAgeZip);
return newAgeZip;
}
@@ -170,9 +183,9 @@ public class DataCleanupTasks {
*/
public void archive(File file, String archiveDir, String afterArchiveDir) throws AAIException, Exception {
- logger.info("Inside the archive folder");
+ logger.debug("Inside the archive folder");
String filename = file.getName();
- logger.info("file name is " +filename);
+ logger.debug("file name is " +filename);
File archivepath = new File(archiveDir);
String zipFile = afterArchiveDir + AAIConstants.AAI_FILESEP + filename;
@@ -180,13 +193,13 @@ public class DataCleanupTasks {
File dataGroomingPath = new File(afterArchiveDir);
boolean exists = directoryExists(archiveDir);
- logger.info("Directory" + archiveDir + "exists: " + exists);
+ logger.debug("Directory" + archiveDir + "exists: " + exists);
if(!exists) {
- logger.error("The directory" + archiveDir +"does not exists so will create a new archive folder");
+ logger.debug("The directory" + archiveDir +"does not exists so will create a new archive folder");
//Create an archive folder if does not exists
boolean flag = dataGroomingPath.mkdirs();
if(!flag)
- logger.error("Failed to create ARCHIVE folder");
+ logger.debug("Failed to create ARCHIVE folder");
}
try(FileOutputStream outputstream = new FileOutputStream(zipFile + ".gz");
ZipOutputStream zoutputstream = new ZipOutputStream(outputstream);
@@ -202,13 +215,8 @@ public class DataCleanupTasks {
zoutputstream.closeEntry();
//Delete the file after been added to archive folder
delete(file);
- logger.info("The file archived is " + file + " at " + afterArchiveDir );
+ logger.debug("The file archived is " + file + " at " + afterArchiveDir );
}
- catch (IOException e) {
- ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup " + e.getStackTrace());
- logger.info("AAI_4000", "Exception running cron job for DataCleanup", e);
- throw e;
- }
}
/**
@@ -217,10 +225,10 @@ public class DataCleanupTasks {
*/
public static void delete(File file) {
- logger.info("Deleting the file " + file);
+ logger.debug("Deleting the file " + file);
boolean deleteStatus = file.delete();
if(!deleteStatus){
- logger.error("Failed to delete the file" +file);
+ logger.debug("Failed to delete the file" +file);
}
}
@@ -233,7 +241,7 @@ public class DataCleanupTasks {
@Scheduled(cron = "${datasnapshotcleanup.cron}" )
public void dataSnapshotCleanup() throws AAIException, Exception {
- logger.info("Started cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date()));
+ logger.info(ONAPLogConstants.Markers.ENTRY, "Started cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date()));
try {
String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs";
@@ -244,15 +252,15 @@ public class DataCleanupTasks {
File archivepath = new File(archiveDir);
File dataSnapshotPath = new File(dataSnapshotArcDir);
- logger.info("The logDir is " + logDir);
- logger.info("The dataSnapshotDir is " + dataSnapshotDir);
- logger.info("The archiveDir is " + archiveDir );
- logger.info("The dataSnapshotArcDir is " + dataSnapshotArcDir );
+ logger.debug("The logDir is " + logDir);
+ logger.debug("The dataSnapshotDir is " + dataSnapshotDir);
+ logger.debug("The archiveDir is " + archiveDir );
+ logger.debug("The dataSnapshotArcDir is " + dataSnapshotArcDir );
boolean exists = directoryExists(logDir);
- logger.info("Directory" + logDir + "exists: " + exists);
+ logger.debug("Directory" + logDir + "exists: " + exists);
if(!exists)
- logger.error("The directory" + logDir +"does not exists");
+ logger.debug("The directory" + logDir +"does not exists");
Integer ageZipSnapshot = AAIConfig.getInt("aai.datasnapshot.agezip");
Integer ageDeleteSnapshot = AAIConfig.getInt("aai.datasnapshot.agedelete");
@@ -267,9 +275,9 @@ public class DataCleanupTasks {
continue;
}
if(listFile.isFile()){
- logger.info("The file name in dataSnapshot: " +listFile.getName());
+ logger.debug("The file name in dataSnapshot: " +listFile.getName());
Date fileCreateDate = fileCreationMonthDate(listFile);
- logger.info("The fileCreateDate in dataSnapshot is " + fileCreateDate);
+ logger.debug("The fileCreateDate in dataSnapshot is " + fileCreateDate);
if( fileCreateDate.compareTo(newAgeZip) < 0) {
archive(listFile,archiveDir,dataSnapshotArcDir);
}
@@ -283,9 +291,9 @@ public class DataCleanupTasks {
if(listFilesArchive != null) {
for(File listFileArchive : listFilesArchive) {
if(listFileArchive.isFile()) {
- logger.info("The file name in ARCHIVE/dataSnapshot: " +listFileArchive.getName());
+ logger.debug("The file name in ARCHIVE/dataSnapshot: " +listFileArchive.getName());
Date fileCreateDate = fileCreationMonthDate(listFileArchive);
- logger.info("The fileCreateDate in ARCHIVE/dataSnapshot is " + fileCreateDate);
+ logger.debug("The fileCreateDate in ARCHIVE/dataSnapshot is " + fileCreateDate);
if(fileCreateDate.compareTo(newAgeDelete) < 0) {
delete(listFileArchive);
}
@@ -294,9 +302,9 @@ public class DataCleanupTasks {
}
}
catch (Exception e) {
- ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
- logger.info("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
- throw e;
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e));
+ logger.debug("AAI_4000", "Exception running cron job for DataCleanup"+LogFormatTools.getStackTop(e));
}
+ logger.info(ONAPLogConstants.Markers.EXIT, "Ended cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date()));
}
}
diff --git a/src/main/java/org/onap/aai/dataexport/DataExportTasks.java b/src/main/java/org/onap/aai/dataexport/DataExportTasks.java
index 359e2ba..0131650 100644
--- a/src/main/java/org/onap/aai/dataexport/DataExportTasks.java
+++ b/src/main/java/org/onap/aai/dataexport/DataExportTasks.java
@@ -38,24 +38,27 @@ import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog;
import org.onap.aai.dbgen.DynamicPayloadGenerator;
import org.onap.aai.edges.EdgeIngestor;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LogFormatTools;
import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
+import org.onap.logging.filter.base.ONAPComponents;
+import org.onap.logging.ref.slf4j.ONAPLogConstants;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.PropertySource;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
import org.apache.commons.io.comparator.LastModifiedFileComparator;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.FileFileFilter;
@@ -69,8 +72,10 @@ import org.apache.commons.io.filefilter.RegexFileFilter;
@Component
@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties")
public class DataExportTasks {
+
+ private AaiScheduledTaskAuditLog auditLog;
- private static final EELFLogger LOGGER;
+ private static final Logger LOGGER;
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
private static final String GA_MS = "aai-graphadmin";
@@ -79,7 +84,7 @@ public class DataExportTasks {
Properties props = System.getProperties();
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
- LOGGER = EELFManager.getInstance().getLogger(DataExportTasks.class);
+ LOGGER = LoggerFactory.getLogger(DataExportTasks.class);
}
private LoaderFactory loaderFactory;
@@ -98,10 +103,12 @@ public class DataExportTasks {
*/
@Scheduled(cron = "${dataexporttask.cron}" )
public void export() {
+
try {
exportTask();
}
catch (Exception e) {
+ ErrorLogHelper.logError("AAI_8002", "Exception while running export "+ LogFormatTools.getStackTop(e));
}
}
/**
@@ -110,27 +117,20 @@ public class DataExportTasks {
* @throws AAIException, Exception
*/
public void exportTask() throws AAIException, Exception {
-
- LoggingContext.init();
- LoggingContext.requestId(UUID.randomUUID().toString());
- LoggingContext.partnerName("AAI");
- LoggingContext.targetEntity(GA_MS);
- LoggingContext.component("exportTask");
- LoggingContext.serviceName(GA_MS);
- LoggingContext.targetServiceName("exportTask");
- LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
-
+ auditLog = new AaiScheduledTaskAuditLog();
+ auditLog.logBefore("dataExportTask", ONAPComponents.AAI.toString());
+ LOGGER.info("Started exportTask: " + dateFormat.format(new Date()));
if (AAIConfig.get("aai.dataexport.enable").equalsIgnoreCase("false")) {
- LOGGER.info("Data Export is not enabled");
+ LOGGER.debug("Data Export is not enabled");
return;
}
// Check if the process was started via command line
if (isDataExportRunning()) {
- LOGGER.info("There is a dataExport process already running");
+ LOGGER.debug("There is a dataExport process already running");
return;
}
- LOGGER.info("Started exportTask: " + dateFormat.format(new Date()));
+ LOGGER.debug("Started exportTask: " + dateFormat.format(new Date()));
String enableSchemaValidation = AAIConfig.get("aai.dataexport.enable.schema.validation", "false");
String outputLocation = AAIConstants.AAI_HOME_BUNDLECONFIG + AAIConfig.get("aai.dataexport.output.location");
@@ -187,20 +187,20 @@ public class DataExportTasks {
String[] paramsArray = paramsList.toArray(new String[0]);
try {
DynamicPayloadGenerator.run(loaderFactory, edgeIngestor, schemaVersions, paramsArray, false);
- LOGGER.info("DynamicPaylodGenerator completed");
+ LOGGER.debug("DynamicPaylodGenerator completed");
// tar/gzip payload files
String[] command = new String[1];
command[0] = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "bin" + AAIConstants.AAI_FILESEP + "dynamicPayloadArchive.sh";
runScript(command);
}
catch (Exception e) {
- ErrorLogHelper.logError("AAI_8003", e.getMessage());
- LOGGER.info("Exception running dataExport task " + e.getMessage());
- throw e;
+ ErrorLogHelper.logError("AAI_8003", LogFormatTools.getStackTop(e));
+ LOGGER.debug("Exception running dataExport task " + LogFormatTools.getStackTop(e));
} finally {
- LOGGER.info("Completed dataExport task" );
- LoggingContext.clear();
+ LOGGER.debug("Ended exportTask: " + dateFormat.format(new Date()));
}
+ LOGGER.info("Ended exportTask: " + dateFormat.format(new Date()));
+ auditLog.logAfter();
}
/**
@@ -223,10 +223,10 @@ public class DataExportTasks {
}
int exitVal = process.waitFor();
- LOGGER.info("Check if dataExport is running returned: " + exitVal);
+ LOGGER.debug("Check if dataExport is running returned: " + exitVal);
} catch (Exception e) {
- ErrorLogHelper.logError("AAI_8002", "Exception while running the check to see if dataExport is running "+ e.getMessage());
- LOGGER.info("Exception while running the check to see if dataExport is running "+ e.getMessage());
+ ErrorLogHelper.logError("AAI_8002", "Exception while running the check to see if dataExport is running "+ LogFormatTools.getStackTop(e));
+ LOGGER.debug("Exception while running the check to see if dataExport is running "+ LogFormatTools.getStackTop(e));
}
if(count > 0){
@@ -249,7 +249,7 @@ public class DataExportTasks {
File[] allFilesArr = targetDirFile.listFiles((FileFilter) FileFileFilter.FILE);
if ( allFilesArr == null || allFilesArr.length == 0 ) {
ErrorLogHelper.logError("AAI_8001", "Unable to find data snapshots at " + targetDir);
- LOGGER.info ("Unable to find data snapshots at " + targetDir);
+ LOGGER.debug("Unable to find data snapshots at " + targetDir);
return (snapshot);
}
if ( allFilesArr.length > 1 ) {
@@ -287,7 +287,7 @@ public class DataExportTasks {
if ( allFilesArr == null || allFilesArr.length == 0 ) {
ErrorLogHelper.logError("AAI_8001", "Unable to find data snapshots at " + targetDir);
- LOGGER.info ("Unable to find data snapshots at " + targetDir);
+ LOGGER.debug("Unable to find data snapshots at " + targetDir);
return (null);
}
@@ -329,7 +329,7 @@ public class DataExportTasks {
snapshotName = snapshotName.substring(0,lastDot);
}
else {
- LOGGER.info ("Invalid snapshot file name format " + snapshotName);
+ LOGGER.debug("Invalid snapshot file name format " + snapshotName);
return null;
}
}
@@ -348,7 +348,7 @@ public class DataExportTasks {
File[] allFilesArr = targetDirFile.listFiles((FileFilter)DirectoryFileFilter.DIRECTORY);
if ( allFilesArr == null || allFilesArr.length == 0 ) {
- LOGGER.info ("No payload files found at " + targetDirFile.getPath());
+ LOGGER.debug("No payload files found at " + targetDirFile.getPath());
return;
}
for ( File f : allFilesArr ) {
@@ -357,7 +357,7 @@ public class DataExportTasks {
}
catch (IOException e) {
- LOGGER.info ("Unable to delete directory " + f.getAbsolutePath() + " " + e.getMessage());
+ LOGGER.debug("Unable to delete directory " + f.getAbsolutePath() + " " + e.getMessage());
}
}
@@ -372,10 +372,10 @@ public class DataExportTasks {
try {
process = new ProcessBuilder().command(script).start();
int exitVal = process.waitFor();
- LOGGER.info("dynamicPayloadArchive.sh returned: " + exitVal);
+ LOGGER.debug("dynamicPayloadArchive.sh returned: " + exitVal);
} catch (Exception e) {
- ErrorLogHelper.logError("AAI_8002", "Exception while running dynamicPayloadArchive.sh "+ e.getMessage());
- LOGGER.info("Exception while running dynamicPayloadArchive.sh" + e.getMessage());
+ ErrorLogHelper.logError("AAI_8002", "Exception while running dynamicPayloadArchive.sh "+ LogFormatTools.getStackTop(e));
+ LOGGER.debug("Exception while running dynamicPayloadArchive.sh" + LogFormatTools.getStackTop(e));
}
}
diff --git a/src/main/java/org/onap/aai/datagrooming/DataGrooming.java b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java
index 167ec21..29a588b 100644
--- a/src/main/java/org/onap/aai/datagrooming/DataGrooming.java
+++ b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java
@@ -49,6 +49,7 @@ import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.onap.aai.GraphAdminApp;
import org.onap.aai.config.PropertyPasswordConfiguration;
import org.onap.aai.util.GraphAdminConstants;
+import org.onap.logging.ref.slf4j.ONAPLogConstants;
import org.onap.aai.dbmap.AAIGraph;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.Introspector;
@@ -58,17 +59,15 @@ import org.onap.aai.introspection.ModelType;
import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
import org.onap.aai.logging.ErrorLogHelper;
import org.onap.aai.logging.LogFormatTools;
-import org.onap.aai.logging.LoggingContext;
import org.onap.aai.edges.enums.AAIDirection;
import org.onap.aai.edges.enums.EdgeProperty;
import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.setup.SchemaVersion;
import org.onap.aai.util.*;
-import org.onap.aai.logging.LoggingContext.StatusCode;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
@@ -78,8 +77,8 @@ import org.springframework.context.annotation.AnnotationConfigApplicationContext
public class DataGrooming {
- private static EELFLogger LOGGER = EELFManager.getInstance().getLogger(DataGrooming.class);
-
+ private static Logger LOGGER = LoggerFactory.getLogger(DataGrooming.class);
+ private static boolean historyEnabled;
private static final String FROMAPPID = "AAI-DB";
private static final String TRANSID = UUID.randomUUID().toString();
private int dupeGrpsDeleted = 0;
@@ -107,6 +106,15 @@ public class DataGrooming {
public void execute(String[] args){
String ver = "version"; // Placeholder
+
+ // Note - if execute() is called from DataGroomingTasks, ie. from the cron,
+ // then 'historyEnabled' will default to false. In History, dataGrooming
+ // is never called via the cron, but this check will prevent it from
+ // being called from the command line.
+ if( historyEnabled ) {
+ LOGGER.debug("ERROR: DataGrooming may not be used when history.enabled=true. ");
+ return;
+ }
// A value of 0 means that we will not have a time-window -- we will look
// at all nodes of the passed-in nodeType.
@@ -155,19 +163,19 @@ public class DataGrooming {
jCommander.setProgramName(DataGrooming.class.getSimpleName());
//Print Defaults
- LOGGER.info("EdgesOnlyFlag is [" + cArgs.edgesOnlyFlag + "]");
- LOGGER.info("DoAutoFix is [" + cArgs.doAutoFix + "]");
- LOGGER.info("skipHostCheck is [" + cArgs.skipHostCheck + "]");
- LOGGER.info("dontFixOrphansFlag is [" + cArgs.dontFixOrphansFlag + "]");
- LOGGER.info("dupeCheckOff is [" + cArgs.dupeCheckOff + "]");
- LOGGER.info("dupeFixOn is [" + cArgs.dupeFixOn + "]");
- LOGGER.info("ghost2CheckOff is [" + cArgs.ghost2CheckOff + "]");
- LOGGER.info("ghost2FixOn is [" + cArgs.ghost2FixOn + "]");
- LOGGER.info("neverUseCache is [" + cArgs.neverUseCache + "]");
- LOGGER.info("singleNodeType is [" + cArgs.singleNodeType + "]");
- LOGGER.info("skipEdgeChecks is [" + cArgs.skipEdgeCheckFlag + "]");
- LOGGER.info("skipIndexUpdateFix is [" + cArgs.skipIndexUpdateFix + "]");
- LOGGER.info("maxFix is [" + cArgs.maxRecordsToFix + "]");
+ LOGGER.debug("EdgesOnlyFlag is [" + cArgs.edgesOnlyFlag + "]");
+ LOGGER.debug("DoAutoFix is [" + cArgs.doAutoFix + "]");
+ LOGGER.debug("skipHostCheck is [" + cArgs.skipHostCheck + "]");
+ LOGGER.debug("dontFixOrphansFlag is [" + cArgs.dontFixOrphansFlag + "]");
+ LOGGER.debug("dupeCheckOff is [" + cArgs.dupeCheckOff + "]");
+ LOGGER.debug("dupeFixOn is [" + cArgs.dupeFixOn + "]");
+ LOGGER.debug("ghost2CheckOff is [" + cArgs.ghost2CheckOff + "]");
+ LOGGER.debug("ghost2FixOn is [" + cArgs.ghost2FixOn + "]");
+ LOGGER.debug("neverUseCache is [" + cArgs.neverUseCache + "]");
+ LOGGER.debug("singleNodeType is [" + cArgs.singleNodeType + "]");
+ LOGGER.debug("skipEdgeChecks is [" + cArgs.skipEdgeCheckFlag + "]");
+ LOGGER.debug("skipIndexUpdateFix is [" + cArgs.skipIndexUpdateFix + "]");
+ LOGGER.debug("maxFix is [" + cArgs.maxRecordsToFix + "]");
String windowTag = "FULL";
@@ -181,9 +189,8 @@ public class DataGrooming {
loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion());
}
catch (Exception ex){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
- LOGGER.error("ERROR - Could not create loader " + LogFormatTools.getStackTop(ex));
+ AAIException ae = new AAIException("AAI_6128", ex , "ERROR - Could not create loader "+args);
+ ErrorLogHelper.logException(ae);
AAISystemExitUtil.systemExitCloseAAIGraph(1);
}
@@ -191,7 +198,7 @@ public class DataGrooming {
if (!cArgs.prevFileName.equals("")) {
// They are trying to fix some data based on a data in a
// previous file.
- LOGGER.info(" Call doTheGrooming() with a previous fileName ["
+ LOGGER.debug(" Call doTheGrooming() with a previous fileName ["
+ prevFileName + "] for cleanup. ");
Boolean finalShutdownFlag = true;
Boolean cacheDbOkFlag = false;
@@ -201,7 +208,6 @@ public class DataGrooming {
finalShutdownFlag, cacheDbOkFlag,
cArgs.skipEdgeCheckFlag, cArgs.timeWindowMinutes,
cArgs.singleNodeType, cArgs.skipIndexUpdateFix );
-
} else if (cArgs.doAutoFix) {
// They want us to run the processing twice -- first to look for
// delete candidates, then after
@@ -209,8 +215,8 @@ public class DataGrooming {
// that were found by the first run.
// Note: we will produce a separate output file for each of the
// two runs.
- LOGGER.info(" Doing an auto-fix call to Grooming. ");
- LOGGER.info(" First, Call doTheGrooming() to look at what's out there. ");
+ LOGGER.debug(" Doing an auto-fix call to Grooming. ");
+ LOGGER.debug(" First, Call doTheGrooming() to look at what's out there. ");
Boolean finalShutdownFlag = false;
Boolean cacheDbOkFlag = true;
int fixCandCount = doTheGrooming("", cArgs.edgesOnlyFlag,
@@ -220,23 +226,23 @@ public class DataGrooming {
cArgs.skipEdgeCheckFlag, cArgs.timeWindowMinutes,
cArgs.singleNodeType, cArgs.skipIndexUpdateFix );
if (fixCandCount == 0) {
- LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. ");
+ LOGGER.debug(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. ");
} else {
// We'll sleep a little and then run a fix-pass based on the
// first-run's output file.
try {
- LOGGER.info("About to sleep for " + cArgs.sleepMinutes
+ LOGGER.debug("About to sleep for " + cArgs.sleepMinutes
+ " minutes.");
int sleepMsec = cArgs.sleepMinutes * 60 * 1000;
Thread.sleep(sleepMsec);
} catch (InterruptedException ie) {
- LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< ");
+ LOGGER.debug(" >>> Sleep Thread has been Interrupted <<< ");
AAISystemExitUtil.systemExitCloseAAIGraph(0);
}
dteStr = fd.getDateTime();
String secondGroomOutFileName = "dataGrooming." + windowTag + "." + dteStr + ".out";
- LOGGER.info(" Now, call doTheGrooming() a second time and pass in the name of the file "
+ LOGGER.debug(" Now, call doTheGrooming() a second time and pass in the name of the file "
+ "generated by the first pass for fixing: ["
+ groomOutFileName + "]");
finalShutdownFlag = true;
@@ -253,7 +259,7 @@ public class DataGrooming {
// Do the grooming - plain vanilla (no fix-it-file, no
// auto-fixing)
Boolean finalShutdownFlag = true;
- LOGGER.info(" Call doTheGrooming() ");
+ LOGGER.debug(" Call doTheGrooming() ");
Boolean cacheDbOkFlag = true;
if( cArgs.neverUseCache ){
// They have forbidden us from using a cached db connection.
@@ -267,11 +273,9 @@ public class DataGrooming {
cArgs.singleNodeType, cArgs.skipIndexUpdateFix );
}
} catch (Exception ex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("Exception while grooming data " + LogFormatTools.getStackTop(ex));
+ LOGGER.debug("Exception while grooming data " + LogFormatTools.getStackTop(ex));
}
- LOGGER.info(" Done! ");
+ LOGGER.debug(" Done! ");
AAISystemExitUtil.systemExitCloseAAIGraph(0);
}
@@ -285,16 +289,6 @@ public class DataGrooming {
// Set the logging file properties to be used by EELFManager
System.setProperty("aai.service.name", DataGrooming.class.getSimpleName());
- LoggingContext.init();
- LoggingContext.partnerName(FROMAPPID);
- LoggingContext.serviceName(GraphAdminApp.APP_NAME);
- LoggingContext.component("dataGrooming");
- LoggingContext.targetEntity(GraphAdminApp.APP_NAME);
- LoggingContext.targetServiceName("main");
- LoggingContext.requestId(TRANSID);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
-
Properties props = System.getProperties();
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
@@ -312,11 +306,12 @@ public class DataGrooming {
} catch (Exception e) {
AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
throw aai;
}
+
+ historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false"));
+
LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions");
DataGrooming dataGrooming = new DataGrooming(loaderFactory, schemaVersions);
@@ -349,7 +344,7 @@ public class DataGrooming {
Boolean skipEdgeCheckFlag, int timeWindowMinutes,
String singleNodeType, Boolean skipIndexUpdateFix ) {
- LOGGER.debug(" Entering doTheGrooming \n");
+ LOGGER.debug(" Entering doTheGrooming ");
int cleanupCandidateCount = 0;
long windowStartTime = 0; // Translation of the window into a starting timestamp
@@ -388,8 +383,6 @@ public class DataGrooming {
}
if (deleteCandidateList.size() > maxRecordsToFix) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(" >> WARNING >> Delete candidate list size ("
+ deleteCandidateList.size()
+ ") is too big. The maxFix we are using is: "
@@ -410,11 +403,11 @@ public class DataGrooming {
throw new AAIException("AAI_6124", emsg);
}
- LOGGER.info(" Will write to " + fullOutputFileName );
+ LOGGER.debug(" Will write to " + fullOutputFileName );
bw = new BufferedWriter(new FileWriter(groomOutFile.getAbsoluteFile()));
ErrorLogHelper.loadProperties();
- LOGGER.info(" ---- NOTE --- about to open graph (takes a little while)--------\n");
+ LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while)-------- ");
if( cacheDbOkFlag ){
// Since we're just reading (not deleting/fixing anything), we can use
@@ -478,10 +471,10 @@ public class DataGrooming {
Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet();
String ntList = "";
- LOGGER.info(" Starting DataGrooming Processing ");
+ LOGGER.debug(" Starting DataGrooming Processing ");
if (edgesOnlyFlag) {
- LOGGER.info(" NOTE >> Skipping Node processing as requested. Will only process Edges. << ");
+ LOGGER.debug(" NOTE >> Skipping Node processing as requested. Will only process Edges. << ");
}
else {
for (Entry<String, Introspector> entry : entrySet) {
@@ -632,16 +625,14 @@ public class DataGrooming {
}
} catch (Exception e) {
okFlag = false;
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("ERROR trying to delete delete Candidate VID = " + thisVid + " " + LogFormatTools.getStackTop(e));
+ LOGGER.debug("ERROR trying to delete delete Candidate VID = " + thisVid + " " + LogFormatTools.getStackTop(e));
}
if (okFlag){
if( updateOnlyFlag ) {
- LOGGER.info(" Updated Indexes for Delete Candidate VID = " + thisVid);
+ LOGGER.debug(" Updated Indexes for Delete Candidate VID = " + thisVid);
}
else {
- LOGGER.info(" DELETED Delete Candidate VID = " + thisVid);
+ LOGGER.debug(" DELETED Delete Candidate VID = " + thisVid);
}
}
} else {
@@ -653,7 +644,7 @@ public class DataGrooming {
if( ob == null ){
// Group this with missing-node-type guys - which
// we will delete more readily than orphans.
- LOGGER.info(" >> Encountered a missingAaiNodeType while looking for the parent of a [" + nType + "] node.");
+ LOGGER.debug(" >> Encountered a missingAaiNodeType while looking for the parent of a [" + nType + "] node.");
missingAaiNtNodeHash.put(thisVid, thisVtx);
}
else {
@@ -665,7 +656,7 @@ public class DataGrooming {
String checkDummyUid = thisVid + "dummy";
if( auid.equals(checkDummyUid) ){
// Group this with missing-node-type guys.
- LOGGER.info(" >> Encountered a missingAaiNodeType mid-fix-node while looking for the parent of a [" + nType + "] node.");
+ LOGGER.debug(" >> Encountered a missingAaiNodeType mid-fix-node while looking for the parent of a [" + nType + "] node.");
missingAaiNtNodeHash.put(thisVid, thisVtx);
}
else {
@@ -742,16 +733,14 @@ public class DataGrooming {
}
} catch (Exception e) {
okFlag = false;
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("ERROR trying to delete phantom VID = " + thisVid + " " + LogFormatTools.getStackTop(e));
+ LOGGER.debug("ERROR trying to delete phantom VID = " + thisVid + " " + LogFormatTools.getStackTop(e));
}
if (okFlag){
if( updateOnlyFlag ) {
- LOGGER.info(" Updated Indexes for Delete Candidate VID = " + thisVid);
+ LOGGER.debug(" Updated Indexes for Delete Candidate VID = " + thisVid);
}
else {
- LOGGER.info(" DELETED VID = " + thisVid);
+ LOGGER.debug(" DELETED VID = " + thisVid);
}
}
} else {
@@ -760,7 +749,7 @@ public class DataGrooming {
}
else if( (secondGetList.size() > 1) && depNodeOk && !dupeCheckOff ){
// Found some DUPLICATES - need to process them
- LOGGER.info(" - now check Dupes for this guy - ");
+ LOGGER.debug(" - now check Dupes for this guy - ");
List<String> tmpDupeGroups = checkAndProcessDupes(
TRANSID, FROMAPPID, g, source1, version,
nType, secondGetList, dupeFixOn,
@@ -769,28 +758,22 @@ public class DataGrooming {
while (dIter.hasNext()) {
// Add in any newly found dupes to our running list
String tmpGrp = dIter.next();
- LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
+ LOGGER.debug("Found set of dupes: [" + tmpGrp + "]");
dupeGroups.add(tmpGrp);
}
}
}
catch (AAIException e1) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1);
errArr.add(e1.getErrorObject().toString());
}
catch (Exception e2) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(" For nodeType = " + nType
+ " Caught exception", e2);
errArr.add(e2.getMessage());
}
}// try block to enclose looping over each single vertex
catch (Exception exx) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn("WARNING from inside the while-verts-loop ", exx);
}
@@ -809,7 +792,7 @@ public class DataGrooming {
Iterator<ArrayList<Vertex>> dsItr = nonDependentDupeSets.iterator();
while( dsItr.hasNext() ){
ArrayList<Vertex> dupeList = dsItr.next();
- LOGGER.info(" - now check Dupes for some non-dependent guys - ");
+ LOGGER.debug(" - now check Dupes for some non-dependent guys - ");
List<String> tmpDupeGroups = checkAndProcessDupes(
TRANSID, FROMAPPID, g, source1, version,
nType, dupeList, dupeFixOn,
@@ -818,7 +801,7 @@ public class DataGrooming {
while (dIter.hasNext()) {
// Add in any newly found dupes to our running list
String tmpGrp = dIter.next();
- LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
+ LOGGER.debug("Found set of dupes: [" + tmpGrp + "]");
dupeGroups.add(tmpGrp);
}
}
@@ -826,7 +809,7 @@ public class DataGrooming {
}// end of extra dupe check for non-dependent nodes
thisNtDeleteCount = 0; // Reset for the next pass
- LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total (in window) overall. " );
+ LOGGER.debug( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total (in window) overall. " );
}// While-loop for each node type
@@ -843,7 +826,7 @@ public class DataGrooming {
// --------------------------------------------------------------
// To do some strange checking - we need a second graph object
- LOGGER.debug(" ---- NOTE --- about to open a SECOND graph (takes a little while)--------\n");
+ LOGGER.debug(" ---- NOTE --- about to open a SECOND graph (takes a little while)-------- ");
// Note - graph2 just reads - but we want it to use a fresh connection to
// the database, so we are NOT using the CACHED DB CONFIG here.
@@ -854,7 +837,7 @@ public class DataGrooming {
String emsg = "null graph2 object in DataGrooming\n";
throw new AAIException("AAI_6101", emsg);
} else {
- LOGGER.debug("Got the graph2 object... \n");
+ LOGGER.debug("Got the graph2 object... ");
}
g2 = graph2.newTransaction();
if (g2 == null) {
@@ -873,7 +856,7 @@ public class DataGrooming {
int counter = 0;
int lastShown = 0;
Iterator<Vertex> vItor2 = vertList.iterator();
- LOGGER.info(" Checking for bad edges --- ");
+ LOGGER.debug(" Checking for bad edges --- ");
while (vItor2.hasNext()) {
Vertex v = null;
@@ -881,8 +864,6 @@ public class DataGrooming {
try {
v = vItor2.next();
} catch (Exception vex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 ");
continue;
}
@@ -892,14 +873,12 @@ public class DataGrooming {
try {
thisVertId = v.id().toString();
} catch (Exception ev) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list. ");
continue;
}
if (ghostNodeHash.containsKey(thisVertId)) {
// We already know that this is a phantom node, so don't bother checking it
- LOGGER.info(" >> Skipping edge check for edges from vertexId = "
+ LOGGER.debug(" >> Skipping edge check for edges from vertexId = "
+ thisVertId
+ ", since that guy is a Phantom Node");
continue;
@@ -920,7 +899,7 @@ public class DataGrooming {
if (counter == lastShown + 250) {
lastShown = counter;
- LOGGER.info("... Checking edges for vertex # "
+ LOGGER.debug("... Checking edges for vertex # "
+ counter);
}
Iterator<Edge> eItor = v.edges(Direction.BOTH);
@@ -931,8 +910,6 @@ public class DataGrooming {
try {
e = eItor.next();
} catch (Exception iex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex);
continue;
}
@@ -940,8 +917,6 @@ public class DataGrooming {
try {
vIn = e.inVertex();
} catch (Exception err) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err);
}
String vNtI = "";
@@ -967,8 +942,6 @@ public class DataGrooming {
if( ! ghost2CheckOff ){
Vertex connectedVert = g2.traversal().V(vIdLong).next();
if( connectedVert == null ) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
cantGetUsingVid = true;
@@ -980,8 +953,6 @@ public class DataGrooming {
ghost2 = g.traversal().V(vIdLong).next();
}
catch( Exception ex){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
}
if( ghost2 != null ){
@@ -991,8 +962,6 @@ public class DataGrooming {
}// end of the ghost2 checking
}
catch (Exception err) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err);
}
}
@@ -1017,13 +986,11 @@ public class DataGrooming {
deleteCount++;
} catch (Exception e1) {
okFlag = false;
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = "
+ vIdI, e1);
}
if (okFlag) {
- LOGGER.info(" DELETED vertex from bad edge = "
+ LOGGER.debug(" DELETED vertex from bad edge = "
+ vIdI);
}
} else {
@@ -1038,13 +1005,11 @@ public class DataGrooming {
// that this edge has already been
// removed
okFlag = false;
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn("WARNING when trying to delete edge = "
+ thisEid);
}
if (okFlag) {
- LOGGER.info(" DELETED edge = " + thisEid);
+ LOGGER.debug(" DELETED edge = " + thisEid);
}
}
} else {
@@ -1059,8 +1024,6 @@ public class DataGrooming {
try {
vOut = e.outVertex();
} catch (Exception err) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(">>> WARNING trying to get edge's Out-vertex ");
}
String vNtO = "";
@@ -1087,14 +1050,12 @@ public class DataGrooming {
Vertex connectedVert = g2.traversal().V(vIdLong).next();
if( connectedVert == null ) {
cantGetUsingVid = true;
- LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
+ LOGGER.debug( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
// If we can get this ghost with the other graph-object, then get it -- it's still a ghost
try {
ghost2 = g.traversal().V(vIdLong).next();
}
catch( Exception ex){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
}
if( ghost2 != null ){
@@ -1103,8 +1064,6 @@ public class DataGrooming {
}
}
} catch (Exception err) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err);
}
}
@@ -1128,13 +1087,11 @@ public class DataGrooming {
deleteCount++;
} catch (Exception e1) {
okFlag = false;
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = "
+ vIdO, e1);
}
if (okFlag) {
- LOGGER.info(" DELETED vertex from bad edge = "
+ LOGGER.debug(" DELETED vertex from bad edge = "
+ vIdO);
}
} else {
@@ -1149,13 +1106,11 @@ public class DataGrooming {
// that this edge has already been
// removed
okFlag = false;
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn("WARNING when trying to delete edge = "
+ thisEid, ex);
}
if (okFlag) {
- LOGGER.info(" DELETED edge = " + thisEid);
+ LOGGER.debug(" DELETED edge = " + thisEid);
}
}
} else {
@@ -1168,12 +1123,10 @@ public class DataGrooming {
}
}// End of while-edges-loop
} catch (Exception exx) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn("WARNING from in the while-verts-loop ", exx);
}
}// End of while-vertices-loop (the edge-checking)
- LOGGER.info(" Done checking for bad edges --- ");
+ LOGGER.debug(" Done checking for bad edges --- ");
} // end of -- if we're not skipping the edge-checking
@@ -1272,9 +1225,7 @@ public class DataGrooming {
bw.write(info + "\n");
}
} catch (Exception dex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("error trying to print detail info for a ghost-node: " + LogFormatTools.getStackTop(dex));
+ LOGGER.debug("error trying to print detail info for a ghost-node: " + LogFormatTools.getStackTop(dex));
}
}
@@ -1296,9 +1247,7 @@ public class DataGrooming {
bw.write(info + "\n");
}
} catch (Exception dex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("error trying to print detail info for a node missing its aai-node-type " + LogFormatTools.getStackTop(dex));
+ LOGGER.debug("error trying to print detail info for a node missing its aai-node-type " + LogFormatTools.getStackTop(dex));
}
}
@@ -1320,9 +1269,7 @@ public class DataGrooming {
bw.write(info + "\n");
}
} catch (Exception dex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("error trying to print detail info for a node with a bad aai-uri " + LogFormatTools.getStackTop(dex));
+ LOGGER.debug("error trying to print detail info for a node with a bad aai-uri " + LogFormatTools.getStackTop(dex));
}
}
@@ -1344,9 +1291,7 @@ public class DataGrooming {
bw.write(info + "\n");
}
} catch (Exception dex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("error trying to print detail info for a node with bad index " + LogFormatTools.getStackTop(dex));
+ LOGGER.debug("error trying to print detail info for a node with bad index " + LogFormatTools.getStackTop(dex));
}
}
@@ -1368,9 +1313,7 @@ public class DataGrooming {
bw.write(info + "\n");
}
} catch (Exception dex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge " + LogFormatTools.getStackTop(dex));
+ LOGGER.debug("error trying to print detail info for a Orphan Node /missing dependent edge " + LogFormatTools.getStackTop(dex));
}
}
@@ -1390,9 +1333,7 @@ public class DataGrooming {
+ propKey.value() + "]\n");
}
} catch (Exception pex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("error trying to print empty/bad vertex data: " + LogFormatTools.getStackTop(pex));
+ LOGGER.debug("error trying to print empty/bad vertex data: " + LogFormatTools.getStackTop(pex));
}
}
@@ -1471,9 +1412,7 @@ public class DataGrooming {
}// else last entry
}// for each vertex in a group
} catch (Exception dex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("error trying to print duplicate vertex data " + LogFormatTools.getStackTop(dex));
+ LOGGER.debug("error trying to print duplicate vertex data " + LogFormatTools.getStackTop(dex));
}
}// while - work on each group of dupes
@@ -1493,8 +1432,8 @@ public class DataGrooming {
bw.close();
- LOGGER.info("\n ------------- Done doing all the checks ------------ ");
- LOGGER.info("Output will be written to " + fullOutputFileName);
+ LOGGER.debug(" ------------- Done doing all the checks ------------ ");
+ LOGGER.debug("Output will be written to " + fullOutputFileName);
if (cleanupCandidateCount > 0 || badUriNodeCount > 0 || badIndexNodeCount > 0) {
// Technically, this is not an error -- but we're throwing this
@@ -1504,14 +1443,10 @@ public class DataGrooming {
+ "] and investigate delete candidates. ");
}
} catch (AAIException e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("Caught AAIException while grooming data");
+ LOGGER.debug("Caught AAIException while grooming data");
ErrorLogHelper.logException(e);
} catch (Exception ex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("Caught exception while grooming data");
+ LOGGER.debug("Caught exception while grooming data");
ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming");
} finally {
@@ -1519,34 +1454,26 @@ public class DataGrooming {
try {
bw.close();
} catch (IOException iox) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
- LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox);
+ LOGGER.warn("Got an IOException trying to close bufferedWriter() ", iox);
}
}
if (executeFinalCommit) {
// If we were holding off on commits till the end - then now is the time.
if( g == null ){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error(" >>>> ERROR <<<< Could not commit changes. graph was null when we wanted to commit.");
+ LOGGER.debug(" >>>> ERROR <<<< Could not commit changes. graph was null when we wanted to commit.");
}
else if( !g.tx().isOpen() ){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error(" >>>> ERROR <<<< Could not commit changes. Transaction was not open when we wanted to commit.");
+ LOGGER.debug(" >>>> ERROR <<<< Could not commit changes. Transaction was not open when we wanted to commit.");
}
else {
try {
- LOGGER.info("About to do the commit for "
+ LOGGER.debug("About to do the commit for "
+ deleteCount + " removes. ");
g.tx().commit();
- LOGGER.info("Commit was successful ");
+ LOGGER.debug("Commit was successful ");
} catch (Exception excom) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error(" >>>> ERROR <<<< Could not commit changes. " + LogFormatTools.getStackTop(excom));
+ LOGGER.debug(" >>>> ERROR <<<< Could not commit changes. " + LogFormatTools.getStackTop(excom));
deleteCount = 0;
}
}
@@ -1558,8 +1485,6 @@ public class DataGrooming {
g.tx().rollback();
} catch (Exception ex) {
// Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
LOGGER.warn("WARNING from final graphTransaction.rollback()", ex);
}
}
@@ -1570,8 +1495,6 @@ public class DataGrooming {
g2.tx().rollback();
} catch (Exception ex) {
// Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex);
}
}
@@ -1590,8 +1513,6 @@ public class DataGrooming {
}
} catch (Exception ex) {
// Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
LOGGER.warn("WARNING from final graph.shutdown()", ex);
}
@@ -1608,8 +1529,6 @@ public class DataGrooming {
}
} catch (Exception ex) {
// Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
LOGGER.warn("WARNING from final graph2.shutdown()", ex);
}
}
@@ -1629,7 +1548,7 @@ public class DataGrooming {
// NOTE -- as of 1902-P2, this is deprecated --------------
- LOGGER.info(" We will try to re-set the indexed properties for this node without changing any property values. VID = " + thisVidStr );
+ LOGGER.debug(" We will try to re-set the indexed properties for this node without changing any property values. VID = " + thisVidStr );
// These reserved-prop-names are all indexed for all nodes
ArrayList <String> propList = new ArrayList <String> ();
@@ -1644,7 +1563,7 @@ public class DataGrooming {
try {
Object valObj = thisVtx.<Object>property(propName).orElse(null);
if( valObj != null ){
- LOGGER.info(" We will try resetting prop [" + propName
+ LOGGER.debug(" We will try resetting prop [" + propName
+ "], to val = [" + valObj.toString() + "] for VID = " + thisVidStr);
thisVtx.property(propName, valObj);
}
@@ -1667,7 +1586,7 @@ public class DataGrooming {
// this pass. A future pass will just treat this node like a regular orphan
// and delete it (if appropriate).
- LOGGER.info(" We will be updating the indexed properties for this node to dummy values. VID = " + thisVidStr );
+ LOGGER.debug(" We will be updating the indexed properties for this node to dummy values. VID = " + thisVidStr );
String dummyPropValStr = thisVidStr + "dummy";
// These reserved-prop-names are all indexed for all nodes
thisVtx.property("aai-node-type",nType);
@@ -2327,8 +2246,6 @@ public class DataGrooming {
}
}
} catch (Exception e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e);
}
@@ -2454,9 +2371,7 @@ public class DataGrooming {
// like, "KeepVid=12345"
String[] prefArr = prefString.split("=");
if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("Bad format. Expecting KeepVid=999999");
+ LOGGER.debug("Bad format. Expecting KeepVid=999999");
return false;
} else {
String keepVidStr = prefArr[1];
@@ -2480,20 +2395,16 @@ public class DataGrooming {
} catch (Exception e) {
okFlag = false;
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("ERROR trying to delete VID = " + thisVid + " " + LogFormatTools.getStackTop(e));
+ LOGGER.debug("ERROR trying to delete VID = " + thisVid + " " + LogFormatTools.getStackTop(e));
}
if (okFlag) {
- LOGGER.info(" DELETED VID = " + thisVid);
+ LOGGER.debug(" DELETED VID = " + thisVid);
deletedSomething = true;
}
}
}
} else {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = ["
+ LOGGER.debug("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = ["
+ dupeInfoString + "]");
return false;
}
@@ -2563,9 +2474,7 @@ public class DataGrooming {
}
}
catch( Exception ex ){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error(" ERROR trying to get node with aai-uri: [" + aaiUriStr + "]" + LogFormatTools.getStackTop(ex));
+ LOGGER.debug(" ERROR trying to get node with aai-uri: [" + aaiUriStr + "]" + LogFormatTools.getStackTop(ex));
}
return true;
@@ -2636,9 +2545,7 @@ public class DataGrooming {
}
}
catch( Exception ex ){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]" + LogFormatTools.getStackTop(ex));
+ LOGGER.debug( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]" + LogFormatTools.getStackTop(ex));
}
if( verts != null ){
@@ -2979,7 +2886,7 @@ public class DataGrooming {
catch ( AAIException ae ){
String emsg = "Error trying to get node just by key " + ae.getMessage();
//System.out.println(emsg);
- LOGGER.error(emsg);
+ LOGGER.debug(emsg);
}
return returnVid;
diff --git a/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java b/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java
index 749a264..4309ece 100644
--- a/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java
+++ b/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java
@@ -20,31 +20,37 @@
package org.onap.aai.datagrooming;
import java.io.BufferedReader;
-import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LogFormatTools;
import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConfig;
+import org.onap.logging.filter.base.ONAPComponents;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.PropertySource;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-
@Component
@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties")
public class DataGroomingTasks {
+
+ private AaiScheduledTaskAuditLog auditLog;
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DataGroomingTasks.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(DataGroomingTasks.class);
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
@Autowired
@@ -55,25 +61,17 @@ public class DataGroomingTasks {
@Scheduled(cron = "${datagroomingtasks.cron}" )
public void groomingScheduleTask() throws AAIException, Exception {
-
- LoggingContext.init();
- LoggingContext.requestId(UUID.randomUUID().toString());
- LoggingContext.partnerName("AAI");
- LoggingContext.targetEntity("CronApp");
- LoggingContext.component("dataGrooming");
- LoggingContext.serviceName("groomingScheduleTask");
- LoggingContext.targetServiceName("groomingScheduleTask");
- LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
-
+ auditLog = new AaiScheduledTaskAuditLog();
+ auditLog.logBefore("dataGroomingTask", ONAPComponents.AAI.toString());
if(!"true".equals(AAIConfig.get("aai.disable.check.grooming.running", "false"))){
if(checkIfDataGroomingIsRunning()){
- LOGGER.info("Data Grooming is already running on the system");
+ LOGGER.debug("Data Grooming is already running on the system");
return;
}
}
-
- LOGGER.info("Started cron job dataGrooming @ " + dateFormat.format(new Date()));
+
+ LOGGER.debug("Started cron job dataGrooming @ " + dateFormat.format(new Date()));
Map<String, String> dataGroomingFlagMap = new HashMap<>();
append("enableautofix" , AAIConfig.get("aai.datagrooming.enableautofix"), dataGroomingFlagMap);
@@ -162,13 +160,12 @@ public class DataGroomingTasks {
}
}
catch (Exception e) {
- ErrorLogHelper.logError("AAI_4000", "Exception running cron job for dataGrooming"+e.toString());
- LOGGER.info("AAI_4000", "Exception running cron job for dataGrooming"+e.toString());
- throw e;
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for dataGrooming"+LogFormatTools.getStackTop(e));
+ LOGGER.debug("AAI_4000", "Exception running cron job for dataGrooming"+LogFormatTools.getStackTop(e));
} finally {
- LOGGER.info("Ended cron job dataGrooming @ " + dateFormat.format(new Date()));
- LoggingContext.clear();
+ LOGGER.debug("Ended cron job dataGrooming @ " + dateFormat.format(new Date()));
}
+ auditLog.logAfter();
}
private boolean checkIfDataGroomingIsRunning(){
@@ -187,9 +184,9 @@ public class DataGroomingTasks {
}
int exitVal = process.waitFor();
- LOGGER.info("Exit value of the dataGrooming check process: " + exitVal);
+ LOGGER.debug("Exit value of the dataGrooming check process: " + exitVal);
} catch (Exception e) {
- e.printStackTrace();
+ LOGGER.debug("AAI_4000", "Exception running dataGrooming check process "+LogFormatTools.getStackTop(e));
}
if(count > 0){
diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java
index e7ae5ec..217d6c0 100644
--- a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java
+++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java
@@ -25,7 +25,14 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.SequenceInputStream;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -33,34 +40,33 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.io.IoCore;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.util.JanusGraphCleanup;
import org.onap.aai.dbmap.AAIGraph;
import org.onap.aai.dbmap.AAIGraphConfig;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.logging.ErrorLogHelper;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
-import org.onap.aai.util.GraphAdminConstants;
import org.onap.aai.util.AAISystemExitUtil;
import org.onap.aai.util.FormatDate;
+import org.onap.aai.util.GraphAdminConstants;
import org.onap.aai.util.GraphAdminDBUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
-import org.janusgraph.core.JanusGraph;
-import org.janusgraph.core.JanusGraphFactory;
-import org.janusgraph.core.util.JanusGraphCleanup;
-
public class DataSnapshot {
- private static EELFLogger LOGGER;
+ private static Logger LOGGER;
/* Using realtime d */
private static final String REALTIME_DB = "realtime";
@@ -108,17 +114,16 @@ public class DataSnapshot {
public boolean executeCommand(String[] args, boolean success,
Boolean dbClearFlag, JanusGraph graph, String command,
String oldSnapshotFileName) {
-
+
// Set the logging file properties to be used by EELFManager
System.setProperty("aai.service.name", DataSnapshot.class.getSimpleName());
Properties props = System.getProperties();
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
- LOGGER = EELFManager.getInstance().getLogger(DataSnapshot.class);
+ LOGGER = LoggerFactory.getLogger(DataSnapshot.class);
cArgs = new CommandLineArgs();
String itemName = "aai.datasnapshot.threads.for.create";
-
try {
String val = AAIConfig.get(itemName);
if( val != null && !val.equals("") ){
@@ -128,9 +133,19 @@ public class DataSnapshot {
LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
}
int threadCount4Create = cArgs.threadCount;
-
+
+ itemName = "aai.datasnapshot.max.nodes.per.file.for.create";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.maxNodesPerFile = Long.parseLong(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+ long maxNodesPerFile4Create = cArgs.maxNodesPerFile;
+
cArgs.snapshotType = "graphson";
-
Long vertAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS;
itemName = "aai.datasnapshot.vertex.add.delay.ms";
try {
@@ -215,10 +230,10 @@ public class DataSnapshot {
jCommander = new JCommander(cArgs, args);
jCommander.setProgramName(DataSnapshot.class.getSimpleName());
} catch (ParameterException e1) {
- LOGGER.error("Error - invalid value passed to list of args - "+args);
+ AAIException ae = new AAIException("AAI_6128", e1 , "Error - invalid value passed to list of args - "+args);
+ ErrorLogHelper.logException(ae);
AAISystemExitUtil.systemExitCloseAAIGraph(1);
}
-
if (args.length >= 1) {
command = cArgs.command;
@@ -253,6 +268,23 @@ public class DataSnapshot {
}
LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount4Create );
+ try {
+ maxNodesPerFile4Create = cArgs.maxNodesPerFile;
+ }
+ catch ( NumberFormatException nfe ){
+ ErrorLogHelper.logError("AAI_6128", "Bad (non-long) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]");
+ LOGGER.debug("Bad (non-long) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ if( maxNodesPerFile4Create < 1000 || maxNodesPerFile4Create > 1000000 ){
+ ErrorLogHelper.logError("AAI_6128", "Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]");
+ LOGGER.debug("Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]");
+ LOGGER.debug("Out of range (1000-1000000) maxNodesPerFile >> Recommended value = 120000)");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ LOGGER.debug(" Will do Threaded Snapshot with maxNodesPerFile = " + maxNodesPerFile4Create );
+
// If doing a "threaded" snapshot, they need to specify how many threads to use
// They can also use debug mode if they pass the word "DEBUG" to do the nodes one at a time to see where it breaks.
if( cArgs.debugFlag.equals("DEBUG") ){
@@ -326,42 +358,45 @@ public class DataSnapshot {
}
+ threadCount4Create = cArgs.threadCount;
+ maxNodesPerFile4Create = cArgs.maxNodesPerFile;
//Print Defaults
- LOGGER.info("DataSnapshot command is [" + cArgs.command + "]");
- LOGGER.info("File name to reload snapshot [" + cArgs.oldFileName + "]");
- LOGGER.info("snapshotType is [" + cArgs.snapshotType + "]");
- LOGGER.info("Thread count is [" + cArgs.threadCount + "]");
- LOGGER.info("Debug Flag is [" + cArgs.debugFlag + "]");
- LOGGER.info("DebugAddDelayTimer is [" + cArgs.debugAddDelayTime + "]");
- LOGGER.info("VertAddDelayMs is [" + cArgs.vertAddDelayMs + "]");
- LOGGER.info("FailureDelayMs is [" + cArgs.failureDelayMs + "]");
- LOGGER.info("RetryDelayMs is [" + cArgs.retryDelayMs + "]");
- LOGGER.info("MaxErrorsPerThread is [" + cArgs.maxErrorsPerThread + "]");
- LOGGER.info("VertToEdgeProcDelay is [" + cArgs.vertToEdgeProcDelay + "]");
- LOGGER.info("StaggerThreadDelay is [" + cArgs.staggerThreadDelay + "]");
- LOGGER.info("Caller process is ["+ cArgs.caller + "]");
+ LOGGER.debug("DataSnapshot command is [" + cArgs.command + "]");
+ LOGGER.debug("File name to reload snapshot [" + cArgs.oldFileName + "]");
+ LOGGER.debug("snapshotType is [" + cArgs.snapshotType + "]");
+ LOGGER.debug("Thread count is [" + cArgs.threadCount + "]");
+ LOGGER.debug("Max Nodes Per File is [" + cArgs.maxNodesPerFile + "]");
+ LOGGER.debug("Debug Flag is [" + cArgs.debugFlag + "]");
+ LOGGER.debug("DebugAddDelayTimer is [" + cArgs.debugAddDelayTime + "]");
+ LOGGER.debug("VertAddDelayMs is [" + cArgs.vertAddDelayMs + "]");
+ LOGGER.debug("FailureDelayMs is [" + cArgs.failureDelayMs + "]");
+ LOGGER.debug("RetryDelayMs is [" + cArgs.retryDelayMs + "]");
+ LOGGER.debug("MaxErrorsPerThread is [" + cArgs.maxErrorsPerThread + "]");
+ LOGGER.debug("VertToEdgeProcDelay is [" + cArgs.vertToEdgeProcDelay + "]");
+ LOGGER.debug("StaggerThreadDelay is [" + cArgs.staggerThreadDelay + "]");
+ LOGGER.debug("Caller process is ["+ cArgs.caller + "]");
+
//Print non-default values
if (!AAIConfig.isEmpty(cArgs.fileName)){
- LOGGER.info("Snapshot file name (if not default) to use is [" + cArgs.fileName + "]");
+ LOGGER.debug("Snapshot file name (if not default) to use is [" + cArgs.fileName + "]");
}
if (!AAIConfig.isEmpty(cArgs.snapshotDir)){
- LOGGER.info("Snapshot file Directory path (if not default) to use is [" + cArgs.snapshotDir + "]");
+ LOGGER.debug("Snapshot file Directory path (if not default) to use is [" + cArgs.snapshotDir + "]");
}
if (!AAIConfig.isEmpty(cArgs.oldFileDir)){
- LOGGER.info("Directory path (if not default) to load the old snapshot file from is [" + cArgs.oldFileDir + "]");
+ LOGGER.debug("Directory path (if not default) to load the old snapshot file from is [" + cArgs.oldFileDir + "]");
}
+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
-
AAIConfig.init();
ErrorLogHelper.loadProperties();
LOGGER.debug("Command = " + command + ", oldSnapshotFileName = [" + oldSnapshotFileName + "].");
String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dataSnapshots";
// Make sure the dataSnapshots directory is there
new File(targetDir).mkdirs();
-
LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while) ");
if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT"))
@@ -370,7 +405,7 @@ public class DataSnapshot {
// They want to take a snapshot on a single thread and have it go in a single file
// NOTE - they can't use the DEBUG option in this case.
// -------------------------------------------------------------------------------
- LOGGER.debug("\n>>> Command = " + command );
+ LOGGER.debug(" Command = " + command );
verifyGraph(AAIGraph.getInstance().getGraph());
FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
String dteStr = fd.getDateTime();
@@ -390,19 +425,20 @@ public class DataSnapshot {
}
else if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT"))
- && threadCount4Create > 1 ){
+ && threadCount4Create > 1 ){
// ------------------------------------------------------------
// They want the creation of the snapshot to be spread out via
// threads and go to multiple files
// ------------------------------------------------------------
- LOGGER.debug("\n>>> Command = " + command );
+ LOGGER.debug(" Command = " + command );
String newSnapshotOutFname;
if (!AAIConfig.isEmpty(cArgs.fileName)){
newSnapshotOutFname = cArgs.fileName;
} else {
- FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
- String dteStr = fd.getDateTime();
- newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr;
+ FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
+ String dteStr = fd.getDateTime();
+ newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP
+ + "dataSnapshot.graphSON." + dteStr;
}
verifyGraph(AAIGraph.getInstance().getGraph());
graph = AAIGraph.getInstance().getGraph();
@@ -410,43 +446,57 @@ public class DataSnapshot {
GraphAdminDBUtils.logConfigs(graph.configuration());
long timeA = System.nanoTime();
- LOGGER.debug(" Need to divide vertexIds across this many threads: " + threadCount4Create );
- HashMap <String,ArrayList> vertListHash = new HashMap <String,ArrayList> ();
- for( int t = 0; t < threadCount4Create; t++ ){
- ArrayList <Vertex> vList = new ArrayList <Vertex> ();
- String tk = "" + t;
- vertListHash.put( tk, vList);
- }
LOGGER.debug("Count how many nodes are in the db. ");
long totalVertCount = graph.traversal().V().count().next();
- LOGGER.debug(" Total Count of Nodes in DB = " + totalVertCount + ".");
- long nodesPerFile = totalVertCount / threadCount4Create;
- LOGGER.debug(" Thread count = " + threadCount4Create + ", each file will get (roughly): " + nodesPerFile + " nodes.");
long timeA2 = System.nanoTime();
long diffTime = timeA2 - timeA;
long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
LOGGER.debug(" -- To count all vertices in DB it took: " +
minCount + " minutes, " + secCount + " seconds " );
+ LOGGER.debug(" Total Count of Nodes in DB = " + totalVertCount + ".");
- long vtxIndex = 0;
+ int fileCount4Create = figureOutFileCount( totalVertCount, threadCount4Create,
+ maxNodesPerFile4Create );
+ int threadPassesNeeded = (int) Math.ceil((double)fileCount4Create / (double)threadCount4Create);
+ long nodesPerFile = (long) Math.ceil((double)totalVertCount / (double)fileCount4Create);
+
+ LOGGER.debug(" We will run this many simultaneous threads: " + threadCount4Create );
+ LOGGER.debug(" Required number of passes: " + threadPassesNeeded );
+ LOGGER.debug(" Max Nodes per file: " + maxNodesPerFile4Create );
+ LOGGER.debug(" We will generate this many files: " + fileCount4Create );
+ LOGGER.debug(" Each file will have (roughly): " + nodesPerFile + " nodes.");
+ LOGGER.debug(" Now, divide vertexIds across this many files: " + fileCount4Create );
+
+ HashMap <String,ArrayList<Long>> vertIdListHash = new HashMap <String,ArrayList<Long>> ();
+ for( int t = 0; t < fileCount4Create; t++ ){
+ ArrayList <Long> vIdList = new ArrayList <Long> ();
+ String tk = "" + t;
+ vertIdListHash.put( tk, vIdList);
+ }
+
int currentTNum = 0;
String currentTKey = "0";
long thisThrIndex = 0;
- Iterator <Vertex> vtxItr = graph.vertices();
+ Iterator <Vertex> vtxItr = graph.vertices(); // Getting ALL vertices!
while( vtxItr.hasNext() ){
- // Divide up all the vertices so we can process them on different threads
- vtxIndex++;
+ // Divide up ALL the vertices so we can process them on different threads
thisThrIndex++;
- if( (thisThrIndex > nodesPerFile) && (currentTNum < threadCount4Create -1) ){
- // We will need to start adding to the Hash for the next thread
+ if( (thisThrIndex >= nodesPerFile) && (currentTNum < (fileCount4Create - 1)) ){
+ // We will need to start adding to the Hash for the next file
currentTNum++;
currentTKey = "" + currentTNum;
thisThrIndex = 0;
}
- (vertListHash.get(currentTKey)).add(vtxItr.next());
+ long vid = (long)(vtxItr.next()).id();
+ (vertIdListHash.get(currentTKey)).add(vid);
}
+ // close this graph instance thing here since we have all the ids
+ graph.tx().rollback();
+ graph.tx().close();
+
+
long timeB = System.nanoTime();
diffTime = timeB - timeA2;
minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
@@ -455,21 +505,38 @@ public class DataSnapshot {
minCount + " minutes, " + secCount + " seconds " );
// Need to print out each set of vertices using it's own thread
- ArrayList <Thread> threadArr = new ArrayList <Thread> ();
- for( int thNum = 0; thNum < threadCount4Create; thNum++ ){
- String thNumStr = "" + thNum;
- String subFName = newSnapshotOutFname + ".P" + thNumStr;
- Thread thr = new Thread(new PrintVertexDetails(graph, subFName, vertListHash.get(thNumStr),
- debug4Create, debugAddDelayTime, snapshotType) );
- thr.start();
- threadArr.add(thr);
- }
+ // NOTE - we may have more files to generate than number of threads - which
+ // just means that ALL the files won't necessarily be generated in parallel.
- // Make sure all the threads finish before moving on.
- for( int thNum = 0; thNum < threadCount4Create; thNum++ ){
- if( null != threadArr.get(thNum) ){
- (threadArr.get(thNum)).join();
+ int fileNo = 0;
+ for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){
+ ArrayList <Thread> threadArr = new ArrayList <Thread> ();
+ // For each Pass, kick off all the threads and wait until they finish
+ long timeP1 = System.nanoTime();
+ for( int thNum = 0; thNum < threadCount4Create; thNum++ ){
+ String fileNoStr = "" + fileNo;
+ String subFName = newSnapshotOutFname + ".P" + fileNoStr;
+ LOGGER.debug(" DEBUG >>> kick off pass # " + passNo + ", thread # " + thNum);
+ Thread thr = new Thread(new PrintVertexDetails(graph, subFName,
+ vertIdListHash.get(fileNoStr),
+ debug4Create, debugAddDelayTime,
+ snapshotType, LOGGER) );
+ thr.start();
+ threadArr.add(thr);
+ fileNo++;
+ }
+ // Make sure all the threads finish before considering this Pass finished.
+ for( int thNum = 0; thNum < threadCount4Create; thNum++ ){
+ if( null != threadArr.get(thNum) ){
+ (threadArr.get(thNum)).join();
+ }
}
+ long timeP2 = System.nanoTime();
+ diffTime = timeP2 - timeP1;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" Pass number " + passNo + " (out of " + threadPassesNeeded +
+ ") took " + minCount + " minutes, " + secCount + " seconds ");
}
long timeC = System.nanoTime();
@@ -487,7 +554,7 @@ public class DataSnapshot {
// snapshot is written to. Ie. if you have a single-file snapshot,
// then this will be single-threaded.
//
- LOGGER.debug("\n>>> Command = " + command );
+ LOGGER.debug(" Command = " + command );
if (cArgs.oldFileDir != null && cArgs.oldFileDir != ""){
targetDir = cArgs.oldFileDir;
@@ -497,7 +564,7 @@ public class DataSnapshot {
JanusGraph graph1 = AAIGraph.getInstance().getGraph();
long timeStart = System.nanoTime();
- HashMap <String,String> old2NewVertIdMap = new <String,String> HashMap ();
+ HashMap <String,String> old2NewVertIdMap = new HashMap <String,String> ();
// We're going to try loading in the vertices - without edges or properties
// using Separate threads
@@ -535,11 +602,13 @@ public class DataSnapshot {
}
catch (InterruptedException e) {
threadFailCount++;
- e.printStackTrace();
+ AAIException ae = new AAIException("AAI_6128", e , "InterruptedException");
+ ErrorLogHelper.logException(ae);
}
catch (ExecutionException e) {
threadFailCount++;
- e.printStackTrace();
+ AAIException ae = new AAIException("AAI_6128", e , "ExecutionException");
+ ErrorLogHelper.logException(ae);
}
}
@@ -602,11 +671,13 @@ public class DataSnapshot {
}
catch (InterruptedException e) {
threadFailCount++;
- e.printStackTrace();
+ AAIException ae = new AAIException("AAI_6128", e , "InterruptedException");
+ ErrorLogHelper.logException(ae);
}
catch (ExecutionException e) {
threadFailCount++;
- e.printStackTrace();
+ AAIException ae = new AAIException("AAI_6128", e , "ExecutionException");
+ ErrorLogHelper.logException(ae);
}
}
@@ -639,13 +710,13 @@ public class DataSnapshot {
// They are calling this to clear the db before re-loading it
// later
// ------------------------------------------------------------------
- LOGGER.debug("\n>>> Command = " + command );
+ LOGGER.debug(" Command = " + command );
// First - make sure the backup file(s) they will be using can be
// found and has(have) data.
// getFilesToProcess makes sure the file(s) exist and have some data.
getFilesToProcess(targetDir, oldSnapshotFileName, true);
- LOGGER.debug("\n>>> WARNING <<<< ");
+ LOGGER.debug(" WARNING <<<< ");
LOGGER.debug(">>> All data and schema in this database will be removed at this point. <<<");
LOGGER.debug(">>> Processing will begin in 5 seconds. <<<");
LOGGER.debug(">>> WARNING <<<< ");
@@ -660,7 +731,7 @@ public class DataSnapshot {
LOGGER.debug(" Begin clearing out old data. ");
String rtConfig = AAIConstants.REALTIME_DB_CONFIG;
- String serviceName = System.getProperty("aai.service.name", "NA");
+ String serviceName = System.getProperty("aai.service.name", DataSnapshot.class.getSimpleName());
LOGGER.debug("Getting new configs for clearig");
PropertiesConfiguration propertiesConfiguration = new AAIGraphConfig.Builder(rtConfig).forService(serviceName).withGraphType(REALTIME_DB).buildConfiguration();
LOGGER.debug("Open New Janus Graph");
@@ -681,7 +752,7 @@ public class DataSnapshot {
// of snapshot files. Either way, this command will restore via single
// threaded processing.
// ---------------------------------------------------------------------------
- LOGGER.debug("\n>>> Command = " + command );
+ LOGGER.debug(" Command = " + command );
verifyGraph(AAIGraph.getInstance().getGraph());
graph = AAIGraph.getInstance().getGraph();
GraphAdminDBUtils.logConfigs(graph.configuration());
@@ -771,14 +842,11 @@ public class DataSnapshot {
}
} catch (AAIException e) {
- ErrorLogHelper.logError("AAI_6128", e.getMessage());
- LOGGER.error("Encountered an exception during the datasnapshot: ", e);
- e.printStackTrace();
+ ErrorLogHelper.logException(e);
success = false;
} catch (Exception ex) {
- ErrorLogHelper.logError("AAI_6128", ex.getMessage());
- LOGGER.error("Encountered an exception during the datasnapshot: ", ex);
- ex.printStackTrace();
+ AAIException ae = new AAIException("AAI_6128", ex , "Encountered an exception during the datasnapshot");
+ ErrorLogHelper.logException(ae);
success = false;
} finally {
if (!dbClearFlag && graph != null && !MIGRATION_PROCESS_NAME.equalsIgnoreCase(source)) {
@@ -879,9 +947,28 @@ public class DataSnapshot {
}
- class CommandLineArgs {
-
+
+ public static int figureOutFileCount( long totalVertCount, int threadCount4Create,
+ long maxNodesPerFile ) {
+
+ // NOTE - we would always like to use all of our threads. That is, if
+ // we could process all the data with 16 threads, but our threadCount4Create is
+ // only 15, we will do two passes and use all 15 threads each pass which will
+ // create a total of 30 files. Each file will be a bit smaller so the overall
+ // time for the two passes should be faster.
+ if( totalVertCount <= 0 || threadCount4Create <= 0 || maxNodesPerFile <= 0) {
+ return 1;
+ }
+ long maxNodesPerPass = threadCount4Create * maxNodesPerFile;
+ int numberOfPasses = (int) Math.ceil( (double)totalVertCount / (double)maxNodesPerPass);
+ int fileCt = threadCount4Create * numberOfPasses;
+
+ return fileCt;
+ }
+
+
+ class CommandLineArgs {
@Parameter(names = "--help", help = true)
public boolean help;
@@ -897,6 +984,9 @@ public class DataSnapshot {
@Parameter(names = "-threadCount", description = "thread count for create")
public int threadCount = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_THREADS_FOR_CREATE;
+
+ @Parameter(names = "-maxNodesPerFile", description = "Max nodes per file")
+ public long maxNodesPerFile = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_NODES_PER_FILE_FOR_CREATE;
@Parameter(names = "-debugFlag", description = "DEBUG flag")
public String debugFlag = "";
@@ -924,7 +1014,7 @@ public class DataSnapshot {
@Parameter(names = "-staggerThreadDelay", description = "thread delay stagger time in ms")
public long staggerThreadDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_STAGGER_THREAD_DELAY_MS;
-
+
@Parameter(names = "-fileName", description = "file name for generating snapshot ")
public String fileName = "";
diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java
new file mode 100644
index 0000000..03ff51a
--- /dev/null
+++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot4HistInit.java
@@ -0,0 +1,1079 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datasnapshot;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.Map.Entry;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.onap.aai.config.PropertyPasswordConfiguration;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.tinkerpop.gremlin.structure.io.IoCore;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.util.JanusGraphCleanup;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+
+
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.LoaderUtil;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.GraphAdminConstants;
+import org.onap.aai.util.AAISystemExitUtil;
+import org.onap.aai.util.FormatDate;
+import org.onap.aai.util.GraphAdminDBUtils;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import com.att.eelf.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+
+public class DataSnapshot4HistInit {
+
+ private static Logger LOGGER;
+
+ /* Using realtime d */
+ private static final String REALTIME_DB = "realtime";
+
+ private static final Set<String> SNAPSHOT_RELOAD_COMMANDS = new HashSet<>();
+
+ private static final String MIGRATION_PROCESS_NAME = "migration";
+
+ private static boolean historyEnabled;
+
+ private LoaderFactory loaderFactory;
+ private SchemaVersions schemaVersions;
+
+ static {
+ SNAPSHOT_RELOAD_COMMANDS.add("RELOAD_DATA");
+ SNAPSHOT_RELOAD_COMMANDS.add("RELOAD_DATA_MULTI");
+ }
+
+ private CommandLineArgs cArgs;
+
+ public DataSnapshot4HistInit(LoaderFactory loaderFactory, SchemaVersions schemaVersions){
+ this.loaderFactory = loaderFactory;
+ this.schemaVersions = schemaVersions;
+ }
+
+ /**
+ * The main method.
+ *
+ * @param args
+ * the arguments
+ */
+ public static void main(String[] args) {
+
+ // Set the logging file properties to be used by EELFManager
+ System.setProperty("aai.service.name", DataSnapshot4HistInit.class.getSimpleName());
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+ LOGGER = LoggerFactory.getLogger(DataSnapshot4HistInit.class);
+
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
+ PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
+ initializer.initialize(ctx);
+ try {
+ ctx.scan(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+ ctx.refresh();
+ } catch (Exception e) {
+ AAIException ae = new AAIException("AAI_6128", e , "Error - Could not initialize context beans for DataSnapshot4HistInit.");
+ ErrorLogHelper.logException(ae);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false"));
+ if( !historyEnabled ) {
+ String emsg = "Error - DataSnapshot4HistInit may only be used when history.enabled=true. ";
+ System.out.println(emsg);
+ AAIException ae = new AAIException("AAI_6128", emsg);
+ ErrorLogHelper.logException(ae);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions");
+ DataSnapshot4HistInit dataSnapshotHI = new DataSnapshot4HistInit(loaderFactory, schemaVersions);
+
+ boolean success = dataSnapshotHI.executeCommand(args);
+ if(success){
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ } else {
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ }// End of main()
+
+
+ public boolean executeCommand(String[] args) {
+
+ Boolean dbClearFlag = false;
+ JanusGraph graph = null;
+ String command = "UNKNOWN";
+ String oldSnapshotFileName = "";
+ boolean success = true;
+
+ cArgs = new CommandLineArgs();
+ String itemName = "aai.datasnapshot.threads.for.create";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.threadCount = Integer.parseInt(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+ int threadCount4Create = cArgs.threadCount;
+
+ itemName = "aai.datasnapshot.max.nodes.per.file.for.create";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.maxNodesPerFile = Long.parseLong(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+ long maxNodesPerFile4Create = cArgs.maxNodesPerFile;
+
+ cArgs.snapshotType = "graphson";
+ Long vertAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS;
+ itemName = "aai.datasnapshot.vertex.add.delay.ms";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.vertAddDelayMs = Long.parseLong(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+
+ Long edgeAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_EDGE_ADD_DELAY_MS;
+ itemName = "aai.datasnapshot.edge.add.delay.ms";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.edgeAddDelayMs = Long.parseLong(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+
+ Long failureDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_FAILURE_DELAY_MS;
+ itemName = "aai.datasnapshot.failure.delay.ms";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.failureDelayMs = Long.parseLong(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+
+ Long retryDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_RETRY_DELAY_MS;
+ itemName = "aai.datasnapshot.retry.delay.ms";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.retryDelayMs = Long.parseLong(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+
+ int maxErrorsPerThread = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_ERRORS_PER_THREAD;
+ itemName = "aai.datasnapshot.max.errors.per.thread";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.maxErrorsPerThread = Integer.parseInt(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+
+ Long vertToEdgeProcDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_TO_EDGE_PROC_DELAY_MS;
+ itemName = "aai.datasnapshot.vertex.to.edge.proc.delay.ms";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.vertToEdgeProcDelay = Long.parseLong(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+
+ itemName = "aai.datasnapshot.stagger.thread.delay.ms";
+ try {
+ String val = AAIConfig.get(itemName);
+ if( val != null && !val.equals("") ){
+ cArgs.staggerThreadDelay = Long.parseLong(val);
+ }
+ }catch ( Exception e ){
+ LOGGER.warn("WARNING - could not get [" + itemName + "] value from aaiconfig.properties file. " + e.getMessage());
+ }
+
+ long debugAddDelayTime = 1; // Default to 1 millisecond
+ Boolean debug4Create = false; // By default we do not use debugging for snapshot creation
+ JCommander jCommander;
+ try {
+ jCommander = new JCommander(cArgs, args);
+ jCommander.setProgramName(DataSnapshot4HistInit.class.getSimpleName());
+ } catch (ParameterException e1) {
+ AAIException ae = new AAIException("AAI_6128", e1 , "Error - invalid value passed to list of args - "+args);
+ ErrorLogHelper.logException(ae);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+
+ if (args.length >= 1) {
+ command = cArgs.command;
+ }
+
+ String source = cArgs.caller;
+ String snapshotType = "graphson";
+ if( SNAPSHOT_RELOAD_COMMANDS.contains(cArgs.command)){
+ if (args.length >= 2) {
+ // If re-loading, they need to also pass the snapshot file name to use.
+ // We expected the file to be found in our snapshot directory.
+ oldSnapshotFileName = cArgs.oldFileName;
+ snapshotType = cArgs.snapshotType;
+ }
+ }
+ else if( command.equals("THREADED_SNAPSHOT") ){
+ if (args.length >= 2) {
+ // If doing a "threaded" snapshot, they need to specify how many threads to use
+ try {
+ threadCount4Create = cArgs.threadCount;
+ }
+ catch ( NumberFormatException nfe ){
+ ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) threadCount passed to DataSnapshot [" + cArgs.threadCount + "]");
+ LOGGER.debug("Bad (non-integer) threadCount passed to DataSnapshot [" + cArgs.threadCount + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ if( threadCount4Create < 1 || threadCount4Create > 100 ){
+ ErrorLogHelper.logError("AAI_6128", "Out of range (1-100) threadCount passed to DataSnapshot [" + cArgs.threadCount + "]");
+ LOGGER.debug("Out of range (1-100) threadCount passed to DataSnapshot [" + cArgs.threadCount + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount4Create );
+
+ if( maxNodesPerFile4Create < 1000 || maxNodesPerFile4Create > 1000000 ){
+ ErrorLogHelper.logError("AAI_6128", "Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]");
+ LOGGER.debug("Out of range (1000-1000000) maxNodesPerFile passed to DataSnapshot [" + cArgs.maxNodesPerFile + "]");
+ LOGGER.debug("Out of range (1000-1000000) maxNodesPerFile >> Recommended value = 120000)");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ LOGGER.debug(" Will do Threaded Snapshot with maxNodesPerFile = " + maxNodesPerFile4Create );
+
+ // If doing a "threaded" snapshot, they need to specify how many threads to use
+ // They can also use debug mode if they pass the word "DEBUG" to do the nodes one at a time to see where it breaks.
+ if( cArgs.debugFlag.equals("DEBUG") ){
+ debug4Create = true;
+ }
+ LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount4Create +
+ ", and DEBUG-flag set to: " + debug4Create );
+
+ if (debug4Create) {
+ // If doing a "threaded" snapshot, they need to specify how many threads to use (param 1)
+ // They can also use debug mode if they pass the word "DEBUG" to do the nodes one (param 2)
+ // They can also pass a delayTimer - how many milliseconds to put between each node's ADD (param 3)
+ try {
+ debugAddDelayTime = cArgs.debugAddDelayTime;
+ } catch (NumberFormatException nfe) {
+ ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) debugAddDelayTime passed to DataSnapshot ["
+ + cArgs.debugAddDelayTime + "]");
+ LOGGER.debug("Bad (non-integer) debugAddDelayTime passed to DataSnapshot ["+ cArgs.debugAddDelayTime + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ LOGGER.debug(" Will do Threaded Snapshot with threadCount = "+ threadCount4Create + ", DEBUG-flag set to: "
+ + debug4Create + ", and addDelayTimer = " + debugAddDelayTime + " mSec. ");
+ }
+ }
+ else {
+ ErrorLogHelper.logError("AAI_6128", "Wrong param count (should be 2,3 or 4) when using THREADED_SNAPSHOT.");
+ LOGGER.debug("Wrong param count (should be 2,3 or 4) when using THREADED_SNAPSHOT.");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ }
+ else if( command.equals("MULTITHREAD_RELOAD") ){
+ // Note - this will use as many threads as the snapshot file is
+ // broken up into. (up to a limit)
+ if (args.length >= 2) {
+ // Since they are re-loading, they need to pass the snapshot file name to use.
+ // We expected the file to be found in our snapshot directory. Note - if
+ // it is a multi-part snapshot, then this should be the root of the name.
+ // We will be using the default delay timers.
+ oldSnapshotFileName = cArgs.oldFileName;
+ vertAddDelayMs = cArgs.vertAddDelayMs;
+ edgeAddDelayMs = cArgs.edgeAddDelayMs;
+ failureDelayMs = cArgs.failureDelayMs;
+ retryDelayMs = cArgs.retryDelayMs;
+ try {
+ maxErrorsPerThread = cArgs.maxErrorsPerThread;
+ }
+ catch ( NumberFormatException nfe ){
+ ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) maxErrorsPerThread passed to DataSnapshot [" + cArgs.maxErrorsPerThread + "]");
+ LOGGER.debug("Bad (non-integer) maxErrorsPerThread passed to DataSnapshot [" + cArgs.maxErrorsPerThread + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ if( maxErrorsPerThread < 1 ){
+ ErrorLogHelper.logError("AAI_6128", "Out of range (>0) maxErrorsPerThread passed to DataSnapshot [" + cArgs.maxErrorsPerThread + "]");
+ LOGGER.debug("Out of range (>0) maxErrorsPerThread passed to DataSnapshot [" + cArgs.maxErrorsPerThread + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ }
+ else {
+ ErrorLogHelper.logError("AAI_6128", "Wrong param count (should be either 2 or 7) when using MUTLITHREAD_RELOAD.");
+ LOGGER.debug("Wrong param count (should be 2 or 7) when using MUTLITHREAD_RELOAD.");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ }
+ else if (command.equals("CLEAR_ENTIRE_DATABASE")) {
+ if (args.length >= 2) {
+ oldSnapshotFileName = cArgs.oldFileName;
+ }
+ }
+ long scriptStartTime = System.currentTimeMillis();
+
+ threadCount4Create = cArgs.threadCount;
+
+ //Print Defaults
+ LOGGER.debug("DataSnapshot4HistInit command is [" + cArgs.command + "]");
+ LOGGER.debug("File name to reload snapshot [" + cArgs.oldFileName + "]");
+ LOGGER.debug("snapshotType is [" + cArgs.snapshotType + "]");
+ LOGGER.debug("Thread count is [" + cArgs.threadCount + "]");
+ LOGGER.debug("Debug Flag is [" + cArgs.debugFlag + "]");
+ LOGGER.debug("DebugAddDelayTimer is [" + cArgs.debugAddDelayTime + "]");
+ LOGGER.debug("VertAddDelayMs is [" + cArgs.vertAddDelayMs + "]");
+ LOGGER.debug("FailureDelayMs is [" + cArgs.failureDelayMs + "]");
+ LOGGER.debug("RetryDelayMs is [" + cArgs.retryDelayMs + "]");
+ LOGGER.debug("MaxErrorsPerThread is [" + cArgs.maxErrorsPerThread + "]");
+ LOGGER.debug("VertToEdgeProcDelay is [" + cArgs.vertToEdgeProcDelay + "]");
+ LOGGER.debug("StaggerThreadDelay is [" + cArgs.staggerThreadDelay + "]");
+ LOGGER.debug("Caller process is ["+ cArgs.caller + "]");
+
+ //Print non-default values
+ if (!AAIConfig.isEmpty(cArgs.fileName)){
+ LOGGER.debug("Snapshot file name (if not default) to use is [" + cArgs.fileName + "]");
+ }
+ if (!AAIConfig.isEmpty(cArgs.snapshotDir)){
+ LOGGER.debug("Snapshot file Directory path (if not default) to use is [" + cArgs.snapshotDir + "]");
+ }
+ if (!AAIConfig.isEmpty(cArgs.oldFileDir)){
+ LOGGER.debug("Directory path (if not default) to load the old snapshot file from is [" + cArgs.oldFileDir + "]");
+ }
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ AAIConfig.init();
+ ErrorLogHelper.loadProperties();
+ LOGGER.debug("Command = " + command + ", oldSnapshotFileName = [" + oldSnapshotFileName + "].");
+ String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dataSnapshots";
+ // Make sure the dataSnapshots directory is there
+ new File(targetDir).mkdirs();
+
+ LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while) ");
+
+ if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT"))
+ && threadCount4Create == 1 ){
+ // -------------------------------------------------------------------------------
+ // They want to take a snapshot on a single thread and have it go in a single file
+ // NOTE - they can't use the DEBUG option in this case.
+ // -------------------------------------------------------------------------------
+ LOGGER.debug(" Command = " + command );
+ verifyGraph(AAIGraph.getInstance().getGraph());
+ FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
+ String dteStr = fd.getDateTime();
+ graph = AAIGraph.getInstance().getGraph();
+ GraphAdminDBUtils.logConfigs(graph.configuration());
+ String newSnapshotOutFname = null;
+ long timeA = System.nanoTime();
+ newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr;
+ graph.io(IoCore.graphson()).writeGraph(newSnapshotOutFname);
+ LOGGER.debug("Snapshot written to " + newSnapshotOutFname);
+ long timeB = System.nanoTime();
+ long diffTime = timeB - timeA;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- Single-Thread dataSnapshot took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ }
+ else if ( (command.equals("THREADED_SNAPSHOT") || command.equals("JUST_TAKE_SNAPSHOT"))
+ && threadCount4Create > 1 ){
+ // ------------------------------------------------------------
+ // They want the creation of the snapshot to be spread out via
+ // threads and go to multiple files
+ // ------------------------------------------------------------
+ LOGGER.debug(" Command = " + command );
+ String newSnapshotOutFname;
+ if (!AAIConfig.isEmpty(cArgs.fileName)){
+ newSnapshotOutFname = cArgs.fileName;
+ } else {
+ FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
+ String dteStr = fd.getDateTime();
+ newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr;
+ }
+ verifyGraph(AAIGraph.getInstance().getGraph());
+ graph = AAIGraph.getInstance().getGraph();
+ LOGGER.debug(" Successfully got the Graph instance. ");
+ GraphAdminDBUtils.logConfigs(graph.configuration());
+ long timeA = System.nanoTime();
+
+ LOGGER.debug("Count how many nodes are in the db. ");
+ long totalVertCount = graph.traversal().V().count().next();
+ LOGGER.debug(" Total Count of Nodes in DB = " + totalVertCount + ".");
+ long timeA2 = System.nanoTime();
+ long diffTime = timeA2 - timeA;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To count all vertices in DB it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ int fileCount4Create = figureOutFileCount( totalVertCount, threadCount4Create,
+ maxNodesPerFile4Create );
+ int threadPassesNeeded = (int) Math.ceil((double)fileCount4Create / (double)threadCount4Create);
+ long nodesPerFile = (long) Math.ceil((double)totalVertCount / (double)fileCount4Create);
+
+ LOGGER.debug(" We will run this many simultaneous threads: " + threadCount4Create );
+ LOGGER.debug(" Required number of passes: " + threadPassesNeeded );
+ LOGGER.debug(" Max Nodes per file: " + maxNodesPerFile4Create );
+ LOGGER.debug(" We will generate this many files: " + fileCount4Create );
+ LOGGER.debug(" Each file will have (roughly): " + nodesPerFile + " nodes.");
+ LOGGER.debug(" Now, divide vertexIds across this many files: " + fileCount4Create );
+
+ HashMap <String,ArrayList<Long>> vertIdListHash = new HashMap <String,ArrayList<Long>> ();
+ for( int t = 0; t < fileCount4Create; t++ ){
+ ArrayList <Long> vIdList = new ArrayList <Long> ();
+ String tk = "" + t;
+ vertIdListHash.put( tk, vIdList);
+ }
+
+ int currentTNum = 0;
+ String currentTKey = "0";
+ long thisThrIndex = 0;
+ Iterator <Vertex> vtxItr = graph.vertices(); // Getting ALL vertices!
+ while( vtxItr.hasNext() ){
+ // Divide up ALL the vertices so we can process them on different threads
+ thisThrIndex++;
+ if( (thisThrIndex >= nodesPerFile) && (currentTNum < (fileCount4Create - 1)) ){
+ // We will need to start adding to the Hash for the next file
+ currentTNum++;
+ currentTKey = "" + currentTNum;
+ thisThrIndex = 0;
+ }
+ long vid = (long)(vtxItr.next()).id();
+ (vertIdListHash.get(currentTKey)).add(vid);
+ }
+
+ // close this graph instance thing here since we have all the ids
+ graph.tx().rollback();
+ graph.tx().close();
+
+ long timeB = System.nanoTime();
+ diffTime = timeB - timeA2;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To Loop over all vertices, and put them into sub-Arrays it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ // Need to print out each set of vertices using it's own thread
+ // NOTE - we may have more files to generate than number of threads - which
+ // just means that ALL the files won't necessarily be generated in parallel.
+
+ int fileNo = 0;
+ for( int passNo = 1; passNo <= threadPassesNeeded; passNo++ ){
+ ArrayList <Thread> threadArr = new ArrayList <Thread> ();
+ // For each Pass, kick off all the threads and wait until they finish
+ long timeP1 = System.nanoTime();
+ for( int thNum = 0; thNum < threadCount4Create; thNum++ ){
+ String fileNoStr = "" + fileNo;
+ String subFName = newSnapshotOutFname + ".P" + fileNoStr;
+ LOGGER.debug(" DEBUG >>> kick off pass # " + passNo + ", thread # " + thNum);
+ Thread thr = new Thread(new PrintVertexDetails(graph, subFName,
+ vertIdListHash.get(fileNoStr),
+ debug4Create, debugAddDelayTime,
+ snapshotType, LOGGER) );
+ thr.start();
+ threadArr.add(thr);
+ fileNo++;
+ }
+ // Make sure all the threads finish before considering this Pass finished.
+ for( int thNum = 0; thNum < threadCount4Create; thNum++ ){
+ if( null != threadArr.get(thNum) ){
+ (threadArr.get(thNum)).join();
+ }
+ }
+ long timeP2 = System.nanoTime();
+ diffTime = timeP2 - timeP1;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" Pass number " + passNo + " (out of " + threadPassesNeeded +
+ ") took " + minCount + " minutes, " + secCount + " seconds ");
+ }
+
+ long timeC = System.nanoTime();
+ diffTime = timeC - timeB;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To write all the data out to snapshot files, it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+
+ } else if( command.equals("MULTITHREAD_RELOAD") ){
+ // ---------------------------------------------------------------------
+ // They want the RELOAD of the snapshot to be spread out via threads
+ // NOTE - it will only use as many threads as the number of files the
+ // snapshot is written to. Ie. if you have a single-file snapshot,
+ // then this will be single-threaded.
+ //
+ LOGGER.debug(" Command = " + command );
+ if (cArgs.oldFileDir != null && cArgs.oldFileDir != ""){
+ targetDir = cArgs.oldFileDir;
+ }
+ ArrayList <File> snapFilesArr = getFilesToProcess(targetDir, oldSnapshotFileName, false);
+ int fCount = snapFilesArr.size();
+ JanusGraph graph1 = AAIGraph.getInstance().getGraph();
+ GraphAdminDBUtils.logConfigs(graph1.configuration());
+ long timeStart = System.nanoTime();
+ HashMap <String,String> old2NewVertIdMap = new <String,String> HashMap ();
+ HashMap <String,ArrayList<String>> nodeKeyNames = new <String,ArrayList<String>> HashMap ();
+
+ try {
+ LOGGER.debug("call getNodeKeyNames ()" );
+ nodeKeyNames = getNodeKeyNames();
+ } catch (Exception e) {
+ AAIException ae = new AAIException("AAI_6128", e , "Error - Could not get node Key names "+e.getMessage());
+ ErrorLogHelper.logException(ae);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ // We're going to try loading in the vertices - without edges or properties
+ // using Separate threads
+
+ ExecutorService executor = Executors.newFixedThreadPool(fCount);
+ List<Future<HashMap<String,String>>> list = new ArrayList<Future<HashMap<String,String>>>();
+ for( int i=0; i < fCount; i++ ){
+ File f = snapFilesArr.get(i);
+ String fname = f.getName();
+ String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname;
+ Thread.sleep(cArgs.staggerThreadDelay); // Stagger the threads a bit
+ LOGGER.debug(" -- Read file: [" + fullSnapName + "]");
+ LOGGER.debug(" -- Call the PartialVertexLoader to just load vertices ----");
+ LOGGER.debug(" -- vertAddDelayMs = " + vertAddDelayMs
+ + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs
+ + ", maxErrorsPerThread = " + maxErrorsPerThread );
+ Callable <HashMap<String,String>> vLoader = new PartialVertexLoader(graph1, fullSnapName,
+ vertAddDelayMs, failureDelayMs, retryDelayMs, maxErrorsPerThread, LOGGER);
+ Future <HashMap<String,String>> future = (Future<HashMap<String, String>>) executor.submit(vLoader);
+
+ // add Future to the list, we can get return value using Future
+ list.add(future);
+ LOGGER.debug(" -- Starting PartialDbLoad VERT_ONLY thread # "+ i );
+ }
+
+ int threadCount4Reload = 0;
+ int threadFailCount = 0;
+ for(Future<HashMap<String,String>> fut : list){
+ threadCount4Reload++;
+ try {
+ old2NewVertIdMap.putAll(fut.get());
+ LOGGER.debug(" -- back from PartialVertexLoader. returned thread # " + threadCount4Reload +
+ ", current size of old2NewVertMap is: " + old2NewVertIdMap.size() );
+ }
+ catch (InterruptedException e) {
+ threadFailCount++;
+ AAIException ae = new AAIException("AAI_6128", e , "InterruptedException");
+ ErrorLogHelper.logException(ae);
+ }
+ catch (ExecutionException e) {
+ threadFailCount++;
+ AAIException ae = new AAIException("AAI_6128", e , "ExecutionException");
+ ErrorLogHelper.logException(ae);
+ }
+ }
+ executor.shutdown();
+
+ if( threadFailCount > 0 ) {
+ String emsg = " FAILURE >> " + threadFailCount + " Vertex-loader thread(s) failed to complete successfully. ";
+ LOGGER.debug(emsg);
+ throw new Exception( emsg );
+ }
+
+ long timeX = System.nanoTime();
+ long diffTime = timeX - timeStart;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To reload just the vertex ids from the snapshot files, it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ // Give the DB a little time to chew on all those vertices
+ Thread.sleep(vertToEdgeProcDelay);
+
+ // ----------------------------------------------------------------------------------------
+ LOGGER.debug("\n\n\n -- Now do the edges/props ----------------------");
+ // ----------------------------------------------------------------------------------------
+
+ // We're going to try loading in the edges and missing properties
+ // Note - we're passing the whole oldVid2newVid mapping to the PartialPropAndEdgeLoader
+ // so that the String-updates to the GraphSON will happen in the threads instead of
+ // here in the un-threaded calling method.
+ executor = Executors.newFixedThreadPool(fCount);
+ ArrayList<Future<ArrayList<String>>> listEdg = new ArrayList<Future<ArrayList<String>>>();
+ for( int i=0; i < fCount; i++ ){
+ File f = snapFilesArr.get(i);
+ String fname = f.getName();
+ String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname;
+ Thread.sleep(cArgs.staggerThreadDelay); // Stagger the threads a bit
+ LOGGER.debug(" -- Read file: [" + fullSnapName + "]");
+ LOGGER.debug(" -- Call the PartialPropAndEdgeLoader4HistInit for Properties and EDGEs ----");
+ LOGGER.debug(" -- edgeAddDelayMs = " + vertAddDelayMs
+ + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs
+ + ", maxErrorsPerThread = " + maxErrorsPerThread );
+
+
+ Callable eLoader = new PartialPropAndEdgeLoader4HistInit(graph1, fullSnapName,
+ edgeAddDelayMs, failureDelayMs, retryDelayMs,
+ old2NewVertIdMap, maxErrorsPerThread, LOGGER,
+ scriptStartTime, nodeKeyNames);
+ Future <ArrayList<String>> future = (Future<ArrayList<String>>) executor.submit(eLoader);
+
+ //add Future to the list, we can get return value using Future
+ listEdg.add(future);
+ LOGGER.debug(" -- Starting PartialPropAndEdge thread # "+ i );
+ }
+ threadCount4Reload = 0;
+ for(Future<ArrayList<String>> fut : listEdg){
+ threadCount4Reload++;
+ try{
+ fut.get(); // DEBUG -- should be doing something with the return value if it's not empty - ie. errors
+ LOGGER.debug(" -- back from PartialPropAndEdgeLoader. thread # " + threadCount4Reload );
+ }
+ catch (InterruptedException e) {
+ threadFailCount++;
+ AAIException ae = new AAIException("AAI_6128", e , "InterruptedException");
+ ErrorLogHelper.logException(ae);
+ }
+ catch (ExecutionException e) {
+ threadFailCount++;
+ AAIException ae = new AAIException("AAI_6128", e , "ExecutionException");
+ ErrorLogHelper.logException(ae);
+ }
+ }
+
+ executor.shutdown();
+ if( threadFailCount > 0 ) {
+ String emsg = " FAILURE >> " + threadFailCount + " Property/Edge-loader thread(s) failed to complete successfully. ";
+ LOGGER.debug(emsg);
+ throw new Exception( emsg );
+ }
+
+ // This is needed so we can see the data committed by the called threads
+ graph1.tx().commit();
+
+ long timeEnd = System.nanoTime();
+ diffTime = timeEnd - timeX;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To reload the edges and properties from snapshot files, it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ long totalDiffTime = timeEnd - timeStart;
+ long totalMinCount = TimeUnit.NANOSECONDS.toMinutes(totalDiffTime);
+ long totalSecCount = TimeUnit.NANOSECONDS.toSeconds(totalDiffTime) - (60 * totalMinCount);
+ LOGGER.debug(" -- TOTAL multi-threaded reload time: " +
+ totalMinCount + " minutes, " + totalSecCount + " seconds " );
+ } else if (command.equals("CLEAR_ENTIRE_DATABASE")) {
+ // ------------------------------------------------------------------
+ // They are calling this to clear the db before re-loading it
+ // later
+ // ------------------------------------------------------------------
+ LOGGER.debug(" Command = " + command );
+ // First - make sure the backup file(s) they will be using can be
+ // found and has(have) data.
+ // getFilesToProcess makes sure the file(s) exist and have some data.
+ getFilesToProcess(targetDir, oldSnapshotFileName, true);
+ LOGGER.debug(" WARNING <<<< ");
+ LOGGER.debug(">>> All data and schema in this database will be removed at this point. <<<");
+ LOGGER.debug(">>> Processing will begin in 5 seconds. <<<");
+ LOGGER.debug(">>> WARNING <<<< ");
+
+ try {
+ // Give them a chance to back out of this
+ Thread.sleep(5000);
+ } catch (java.lang.InterruptedException ie) {
+ LOGGER.debug(" DB Clearing has been aborted. ");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ LOGGER.debug(" Begin clearing out old data. ");
+ String rtConfig = AAIConstants.REALTIME_DB_CONFIG;
+ String serviceName = System.getProperty("aai.service.name", DataSnapshot4HistInit.class.getSimpleName());
+ LOGGER.debug("Getting new configs for clearig");
+
+ PropertiesConfiguration propertiesConfiguration = new AAIGraphConfig.Builder(rtConfig).forService(serviceName).withGraphType(REALTIME_DB).buildConfiguration();
+ LOGGER.debug("Open New Janus Graph");
+
+ JanusGraph janusGraph = JanusGraphFactory.open(propertiesConfiguration);
+ verifyGraph(janusGraph);
+ GraphAdminDBUtils.logConfigs(janusGraph.configuration());
+ janusGraph.close();
+ JanusGraphCleanup.clear(janusGraph);
+ LOGGER.debug(" Done clearing data. ");
+ LOGGER.debug(">>> IMPORTANT - NOTE >>> you need to run the SchemaGenerator (use GenTester) before ");
+ LOGGER.debug(" reloading data or the data will be put in without indexes. ");
+ dbClearFlag = true;
+ LOGGER.debug("All done clearing DB");
+
+ } else if (command.equals("RELOAD_DATA")) {
+ // ---------------------------------------------------------------------------
+ // They want to restore the database from either a single file, or a group
+ // of snapshot files. Either way, this command will restore via single
+ // threaded processing.
+ // ---------------------------------------------------------------------------
+ LOGGER.debug(" Command = " + command );
+ verifyGraph(AAIGraph.getInstance().getGraph());
+ graph = AAIGraph.getInstance().getGraph();
+ GraphAdminDBUtils.logConfigs(graph.configuration());
+ if (oldSnapshotFileName.equals("")) {
+ String emsg = "No oldSnapshotFileName passed to DataSnapshot when RELOAD_DATA used.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ long timeA = System.nanoTime();
+
+ ArrayList <File> snapFilesArr = new ArrayList <File> ();
+
+ // First, see if this is a single file (ie. no ".P#" suffix)
+ String onePieceSnapshotFname = targetDir + AAIConstants.AAI_FILESEP + oldSnapshotFileName;
+ File sf = new File(onePieceSnapshotFname);
+ if( sf.exists() ){
+ snapFilesArr.add(sf);
+ }
+ else {
+ // If it's a multi-part snapshot, then collect all the files for it
+ String thisSnapPrefix = oldSnapshotFileName + ".P";
+ File fDir = new File(targetDir); // Snapshot directory
+ File[] allFilesArr = fDir.listFiles();
+ for (File snapFile : allFilesArr) {
+ String snapFName = snapFile.getName();
+ if( snapFName.startsWith(thisSnapPrefix)){
+ snapFilesArr.add(snapFile);
+ }
+ }
+ }
+
+ if( snapFilesArr.isEmpty() ){
+ String emsg = "oldSnapshotFile " + onePieceSnapshotFname + "(with or without .P0) could not be found.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ int fCount = snapFilesArr.size();
+ Vector<InputStream> inputStreamsV = new Vector<>();
+ for( int i = 0; i < fCount; i++ ){
+ File f = snapFilesArr.get(i);
+ String fname = f.getName();
+ if (!f.canRead()) {
+ String emsg = "oldSnapshotFile " + fname + " could not be read.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ } else if (f.length() == 0) {
+ String emsg = "oldSnapshotFile " + fname + " had no data.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ String fullFName = targetDir + AAIConstants.AAI_FILESEP + fname;
+ InputStream fis = new FileInputStream(fullFName);
+ inputStreamsV.add(fis);
+ }
+
+ // Now add inputStreams.elements() to the Vector,
+ // inputStreams.elements() will return Enumerations
+ InputStream sis = new SequenceInputStream(inputStreamsV.elements());
+ LOGGER.debug("Begin loading data from " + fCount + " files -----");
+ if("gryo".equalsIgnoreCase(snapshotType)){
+ graph.io(IoCore.gryo()).reader().create().readGraph(sis, graph);
+ } else {
+ graph.io(IoCore.graphson()).reader().create().readGraph(sis, graph);
+ }
+ LOGGER.debug("Completed the inputGraph command, now try to commit()... ");
+ graph.tx().commit();
+ LOGGER.debug("Completed reloading data.");
+
+ long vCount = graph.traversal().V().count().next();
+ LOGGER.debug("A little after repopulating from an old snapshot, we see: " + vCount + " vertices in the db.");
+
+ long timeB = System.nanoTime();
+ long diffTime = timeB - timeA;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To Reload this snapshot, it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ LOGGER.debug("A little after repopulating from an old snapshot, we see: " + vCount + " vertices in the db.");
+
+ } else {
+ String emsg = "Bad command passed to DataSnapshot: [" + command + "]";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ } catch (AAIException e) {
+ AAIException ae = new AAIException("AAI_6128", e , "Encountered an exception during the datasnapshot:"+e.getMessage());
+ ErrorLogHelper.logException(ae);
+ success = false;
+ } catch (Exception ex) {
+ AAIException ae = new AAIException("AAI_6128", ex , "Encountered an exception during the datasnapshot");
+ ErrorLogHelper.logException(ae);
+ success = false;
+ } finally {
+ if (!dbClearFlag && graph != null && !MIGRATION_PROCESS_NAME.equalsIgnoreCase(source)) {
+ // Any changes that worked correctly should have already done
+ // thier commits.
+ if(!"true".equals(System.getProperty("org.onap.aai.graphadmin.started"))) {
+ if (graph.isOpen()) {
+ graph.tx().rollback();
+ graph.close();
+ }
+ }
+ }
+ try {
+ baos.close();
+ } catch (IOException iox) {
+ }
+ }
+
+ return success;
+ }
+
+ public static HashMap <String,ArrayList<String>> getNodeKeyNames() {
+
+ HashMap <String,ArrayList<String>> keyNameHash = new HashMap <String,ArrayList<String>> ();
+ Loader loader = LoaderUtil.getLatestVersion();
+ Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet();
+ // Get a collection of the names of the key properties for each nodeType
+ for (Entry<String, Introspector> entry : entrySet) {
+ String nType = entry.getKey();
+ Set <String> keyPropsSet = entry.getValue().getKeys();
+ ArrayList <String> keyProps = new ArrayList <String> ();
+ keyProps.addAll(keyPropsSet);
+ keyNameHash.put(nType, keyProps);
+ }
+ return keyNameHash;
+ }
+
+
+ private static ArrayList <File> getFilesToProcess(String targetDir, String oldSnapshotFileName, boolean doingClearDb)
+ throws Exception {
+
+ if( oldSnapshotFileName == null || oldSnapshotFileName.equals("") ){
+ String emsg = "No oldSnapshotFileName passed to DataSnapshot for Reload. ";
+ if( doingClearDb ) {
+ emsg = "No oldSnapshotFileName passed to DataSnapshot. Needed when Clearing the db in case we need a backup. ";
+ }
+ LOGGER.debug(emsg);
+ throw new Exception( emsg );
+ }
+
+ ArrayList <File> snapFilesArrList = new ArrayList <File> ();
+
+ // First, we'll assume that this is a multi-file snapshot and
+ // look for names based on that.
+ String thisSnapPrefix = oldSnapshotFileName + ".P";
+ File fDir = new File(targetDir); // Snapshot directory
+ File[] allFilesArr = fDir.listFiles();
+ for (File snapFile : allFilesArr) {
+ String snapFName = snapFile.getName();
+ if( snapFName.startsWith(thisSnapPrefix)){
+ if (!snapFile.canRead()) {
+ String emsg = "oldSnapshotFile " + snapFName + " could not be read.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ } else if (snapFile.length() == 0) {
+ String emsg = "oldSnapshotFile " + snapFName + " had no data.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ }
+ snapFilesArrList.add(snapFile);
+ }
+ }
+
+ if( snapFilesArrList.isEmpty() ){
+ // Multi-file snapshot check did not find files, so this may
+ // be a single-file snapshot.
+ String oldSnapshotFullFname = targetDir + AAIConstants.AAI_FILESEP + oldSnapshotFileName;
+ File f = new File(oldSnapshotFullFname);
+ if (!f.exists()) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be found.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ } else if (!f.canRead()) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be read.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ } else if (f.length() == 0) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " had no data.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ }
+ snapFilesArrList.add(f);
+ }
+
+ if( snapFilesArrList.isEmpty() ){
+ // Still haven't found anything.. that was not a good file name.
+ String fullFName = targetDir + AAIConstants.AAI_FILESEP + thisSnapPrefix;
+ String emsg = "oldSnapshotFile " + fullFName + "* could not be found.";
+ LOGGER.debug(emsg);
+ throw new Exception(emsg);
+ }
+
+ return snapFilesArrList;
+ }
+
+
+ public static void verifyGraph(JanusGraph graph) {
+
+ if (graph == null) {
+ String emsg = "Not able to get a graph object in DataSnapshot.java\n";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ }
+
+
+ public static int figureOutFileCount( long totalVertCount, int threadCount4Create,
+ long maxNodesPerFile ) {
+
+ // NOTE - we would always like to use all of our threads. That is, if
+ // we could process all the data with 16 threads, but our threadCount4Create is
+ // only 15, we will do two passes and use all 15 threads each pass which will
+ // create a total of 30 files. Each file will be a bit smaller so the overall
+ // time for the two passes should be faster.
+ if( totalVertCount <= 0 || threadCount4Create <= 0 || maxNodesPerFile <= 0) {
+ return 1;
+ }
+
+ long maxNodesPerPass = threadCount4Create * maxNodesPerFile;
+ int numberOfPasses = (int) Math.ceil( (double)totalVertCount / (double)maxNodesPerPass);
+ int fileCt = threadCount4Create * numberOfPasses;
+
+ return fileCt;
+ }
+
+
+ class CommandLineArgs {
+
+ @Parameter(names = "--help", help = true)
+ public boolean help;
+
+ @Parameter(names = "-c", description = "command for taking data snapshot")
+ public String command = "JUST_TAKE_SNAPSHOT";
+
+ @Parameter(names = "-f", description = "previous snapshot file to reload")
+ public String oldFileName = "";
+
+ @Parameter(names = "-snapshotType", description = "snapshot type of gryo or graphson")
+ public String snapshotType = "graphson";
+
+ @Parameter(names = "-threadCount", description = "thread count for create")
+ public int threadCount = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_THREADS_FOR_CREATE;
+
+ @Parameter(names = "-maxNodesPerFile", description = "Max nodes per file")
+ public long maxNodesPerFile = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_NODES_PER_FILE_FOR_CREATE;
+
+ @Parameter(names = "-debugFlag", description = "DEBUG flag")
+ public String debugFlag = "";
+
+ @Parameter(names = "-debugAddDelayTime", description = "delay in ms between each Add for debug mode")
+ public long debugAddDelayTime = 1L;
+
+ @Parameter(names = "-vertAddDelayMs", description = "delay in ms while adding each vertex")
+ public long vertAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS.longValue();
+
+ @Parameter(names = "-edgeAddDelayMs", description = "delay in ms while adding each edge")
+ public long edgeAddDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_EDGE_ADD_DELAY_MS.longValue();
+
+ @Parameter(names = "-failureDelayMs", description = "delay in ms when failure to load vertex or edge in snapshot")
+ public long failureDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_FAILURE_DELAY_MS.longValue();
+
+ @Parameter(names = "-retryDelayMs", description = "time in ms after which load snapshot is retried")
+ public long retryDelayMs = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_FAILURE_DELAY_MS.longValue();
+
+ @Parameter(names = "-maxErrorsPerThread", description = "max errors allowed per thread")
+ public int maxErrorsPerThread = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_MAX_ERRORS_PER_THREAD;
+
+ @Parameter(names = "-vertToEdgeProcDelay", description = "vertex to edge processing delay in ms")
+ public long vertToEdgeProcDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_VERTEX_TO_EDGE_PROC_DELAY_MS.longValue();
+
+ @Parameter(names = "-staggerThreadDelay", description = "thread delay stagger time in ms")
+ public long staggerThreadDelay = GraphAdminConstants.AAI_SNAPSHOT_DEFAULT_STAGGER_THREAD_DELAY_MS;
+
+ @Parameter(names = "-fileName", description = "file name for generating snapshot ")
+ public String fileName = "";
+
+ @Parameter(names = "-snapshotDir", description = "file path for generating snapshot ")
+ public String snapshotDir = "";
+
+ @Parameter(names = "-oldFileDir", description = "directory containing the old snapshot file for reloading")
+ public String oldFileDir = "";
+
+ @Parameter(names = "-caller", description = "process invoking the dataSnapshot")
+ public String caller = "";
+
+ }
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java
index 7092aa8..15fff4b 100644
--- a/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java
+++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java
@@ -25,61 +25,58 @@ import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.*;
+import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog;
import org.onap.aai.datagrooming.DataGrooming;
import org.onap.aai.datagrooming.DataGroomingTasks;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.logging.ErrorLogHelper;
import org.onap.aai.logging.LogFormatTools;
-import org.onap.aai.logging.LoggingContext;
import org.onap.aai.util.AAIConfig;
+import org.onap.logging.filter.base.ONAPComponents;
+import org.onap.logging.ref.slf4j.ONAPLogConstants;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.PropertySource;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
@Component
@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties")
public class DataSnapshotTasks {
-
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DataSnapshotTasks.class);
+
+ private AaiScheduledTaskAuditLog auditLog;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(DataSnapshotTasks.class);
private final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
+
@Scheduled(cron = "${datasnapshottasks.cron}" )
public void snapshotScheduleTask() throws AAIException, Exception {
-
- LoggingContext.init();
- LoggingContext.requestId(UUID.randomUUID().toString());
- LoggingContext.partnerName("AAI");
- LoggingContext.targetEntity("CronApp");
- LoggingContext.component("dataSnapshot");
- LoggingContext.serviceName("snapshotScheduleTask");
- LoggingContext.targetServiceName("snapshotScheduleTask");
- LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
-
+ auditLog = new AaiScheduledTaskAuditLog();
+ auditLog.logBefore("dataSnapshotTask", ONAPComponents.AAI.toString() );
if(!"true".equals(AAIConfig.get("aai.disable.check.snapshot.running", "false"))){
if(checkIfDataSnapshotIsRunning()){
- LOGGER.info("Data Snapshot is already running on the system");
+ LOGGER.debug("Data Snapshot is already running on the system");
return;
}
}
-
- LOGGER.info("Started cron job dataSnapshot @ " + dateFormat.format(new Date()));
+ LOGGER.debug("Started cron job dataSnapshot @ " + dateFormat.format(new Date()));
try {
if (AAIConfig.get("aai.cron.enable.dataSnapshot").equals("true")) {
String [] dataSnapshotParms = {"-c",AAIConfig.get("aai.datasnapshot.params", "JUST_TAKE_SNAPSHOT")};
- LOGGER.info("DataSnapshot Params {}", Arrays.toString(dataSnapshotParms));
+ LOGGER.debug("DataSnapshot Params {}", Arrays.toString(dataSnapshotParms));
DataSnapshot.main(dataSnapshotParms);
}
}
catch (Exception e) {
- ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataSnapshot"+e.toString());
- LOGGER.info("AAI_4000", "Exception running cron job for DataSnapshot"+e.toString());
- throw e;
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataSnapshot"+LogFormatTools.getStackTop(e));
+ LOGGER.debug("AAI_4000", "Exception running cron job for DataSnapshot"+LogFormatTools.getStackTop(e));
} finally {
- LOGGER.info("Ended cron job dataSnapshot @ " + dateFormat.format(new Date()));
- LoggingContext.clear();
+ LOGGER.debug("Ended cron job dataSnapshot @ " + dateFormat.format(new Date()));
}
+ auditLog.logAfter();
}
@@ -98,9 +95,9 @@ public class DataSnapshotTasks {
}
int exitVal = process.waitFor();
- LOGGER.info("Exit value of the dataSnapshot check process: " + exitVal);
+ LOGGER.debug("Exit value of the dataSnapshot check process: " + exitVal);
} catch (Exception e) {
- LOGGER.error("Exception in checkIfDataSnapshotIsRunning" + LogFormatTools.getStackTop(e));
+ ErrorLogHelper.logError("AAI_4000", "Exception in checkIfDataSnapshotIsRunning" + LogFormatTools.getStackTop(e));
}
return count > 0;
diff --git a/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java
index 0f03ee0..6f9065a 100644
--- a/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java
+++ b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java
@@ -31,14 +31,15 @@ import org.apache.tinkerpop.gremlin.structure.Edge;
import org.json.JSONArray;
import org.json.JSONObject;
-import com.att.eelf.configuration.EELFLogger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PartialPropAndEdgeLoader implements Callable <ArrayList<String>>{
- private EELFLogger LOGGER;
+ private Logger LOGGER;
private JanusGraph jg;
private String fName;
@@ -51,7 +52,7 @@ public class PartialPropAndEdgeLoader implements Callable <ArrayList<String>>{
public PartialPropAndEdgeLoader (JanusGraph graph, String fn, Long edgeDelay, Long failureDelay, Long retryDelay,
- HashMap<String,String> vidMap, int maxErrors, EELFLogger elfLog ){
+ HashMap<String,String> vidMap, int maxErrors, Logger elfLog ){
jg = graph;
fName = fn;
edgeAddDelayMs = edgeDelay;
diff --git a/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader4HistInit.java b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader4HistInit.java
new file mode 100644
index 0000000..fa8483e
--- /dev/null
+++ b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader4HistInit.java
@@ -0,0 +1,454 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datasnapshot;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.concurrent.Callable;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+
+
+public class PartialPropAndEdgeLoader4HistInit implements Callable <ArrayList<String>>{
+
+ private Logger LOGGER;
+
+ private JanusGraph jg;
+ private String fName;
+ private Long edgeAddDelayMs;
+ private Long retryDelayMs;
+ private Long failureDelayMs;
+ private HashMap<String,String> old2NewVidMap;
+ private int maxAllowedErrors;
+ private long currentTs;
+ private HashMap <String,ArrayList<String>> keyNamesHash;
+
+
+ public PartialPropAndEdgeLoader4HistInit (JanusGraph graph, String fn, Long edgeDelay, Long failureDelay, Long retryDelay,
+ HashMap<String,String> vidMap, int maxErrors, Logger elfLog,
+ Long cTs, HashMap <String,ArrayList<String>> kNHash){
+ jg = graph;
+ fName = fn;
+ edgeAddDelayMs = edgeDelay;
+ failureDelayMs = failureDelay;
+ retryDelayMs = retryDelay;
+ old2NewVidMap = vidMap;
+ maxAllowedErrors = maxErrors;
+ LOGGER = elfLog;
+ currentTs = cTs;
+ keyNamesHash = kNHash;
+ }
+
+
+ public ArrayList<String> call() throws Exception {
+
+ // This is a partner to the "PartialVertexLoader" code.
+ // That code loads in vertex-id's/vertex-label's for a
+ // multi-file data snapshot.
+ // This code assumes that the all vertex-id's are now in the target db.
+ // This code loads vertex properties and edges for a
+ // multi-file data snapshot (the same one that loaded
+ // the vertex-ids).
+ //
+
+
+ // NOTE - We will be loading parameters and edges for one node at a time so that problems can be
+ // identified or ignored or re-tried instead of causing the entire load to fail.
+ //
+ // Return an arrayList of Strings to give info on what nodes encountered problems
+
+ int entryCount = 0;
+ int retryCount = 0;
+ int failureCount = 0;
+ int retryFailureCount = 0;
+ HashMap <String,String> failedAttemptHash = new HashMap <String,String> ();
+ ArrayList <String> failedAttemptInfo = new ArrayList <String> ();
+
+ int passNum = 1;
+ try( BufferedReader br = new BufferedReader(new FileReader(fName))) {
+ // loop through the file lines and do PUT for each vertex or the edges depending on what the loadtype is
+ for(String origLine; (origLine = br.readLine()) != null; ) {
+ entryCount++;
+ Thread.sleep(edgeAddDelayMs); // Space the edge requests out a little
+
+ String errInfoStr = processThisLine(origLine, passNum);
+ if( !errInfoStr.equals("") ){
+ // There was a problem with this line
+ String vidStr = getTheVidForThisLine(origLine);
+ // We'll use the failedAttemptHash to reTry this item
+ failedAttemptHash.put(vidStr,origLine);
+ failedAttemptInfo.add(errInfoStr);
+ failureCount++;
+ if( failureCount > maxAllowedErrors ) {
+ LOGGER.debug(">>> Abandoning PartialPropAndEdgeLoader() because " +
+ "Max Allowed Error count was exceeded for this thread. (max = " +
+ maxAllowedErrors + ". ");
+ throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". ");
+ }
+ Thread.sleep(failureDelayMs); // take a little nap if it failed
+ }
+ } // End of looping over each line
+ if( br != null ){
+ br.close();
+ }
+ }
+ catch (Exception e) {
+ LOGGER.debug(" --- Failed in the main loop for Buffered-Reader item # " + entryCount +
+ ", fName = " + fName );
+ LOGGER.debug(" --- msg = " + e.getMessage() );
+ throw e;
+ }
+
+ // ---------------------------------------------------------------------------
+ // Now Re-Try any failed requests that might have Failed on the first pass.
+ // ---------------------------------------------------------------------------
+ passNum++;
+ try {
+ for (String failedVidStr : failedAttemptHash.keySet()) {
+ // Take a little nap, and retry this failed attempt
+ LOGGER.debug("DEBUG >> We will sleep for " + retryDelayMs + " and then RETRY any failed edge/property ADDs. ");
+ Thread.sleep(retryDelayMs);
+ retryCount++;
+ Long failedVidL = Long.parseLong(failedVidStr);
+ // When an Edge/Property Add fails, we store the whole (translated) graphSON line as the data in the failedAttemptHash
+ // We're really just doing a GET of this one vertex here...
+ String jsonLineToRetry = failedAttemptHash.get(failedVidStr);
+ String errInfoStr = processThisLine(jsonLineToRetry, passNum);
+ if( !errInfoStr.equals("") ){
+ // There was a problem with this line
+ String translatedVidStr = getTheVidForThisLine(jsonLineToRetry);
+ failedAttemptHash.put(translatedVidStr,jsonLineToRetry);
+ failedAttemptInfo.add(errInfoStr);
+ retryFailureCount++;
+ if( retryFailureCount > maxAllowedErrors ) {
+ LOGGER.debug(">>> Abandoning PartialPropAndEdgeLoader() because " +
+ "Max Allowed Error count was exceeded while doing retries for this thread. (max = " +
+ maxAllowedErrors + ". ");
+ throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". ");
+ }
+ Thread.sleep(failureDelayMs); // take a little nap if it failed
+ }
+ } // End of looping over each failed line
+ }
+ catch (Exception e) {
+ LOGGER.debug(" -- error in RETRY block. ErrorMsg = [" + e.getMessage() + "]" );
+ throw e;
+ }
+
+ LOGGER.debug(">>> After Processing in PartialPropAndEdgeLoader() " +
+ entryCount + " records processed. " + failureCount + " records failed. " +
+ retryCount + " RETRYs processed. " + retryFailureCount + " RETRYs failed. ");
+
+ return failedAttemptInfo;
+
+ }// end of call()
+
+
+
+ private String translateThisVid(String oldVid) throws Exception {
+
+ if( old2NewVidMap == null ){
+ throw new Exception(" ERROR - null old2NewVidMap found in translateThisVid. ");
+ }
+
+ if( old2NewVidMap.containsKey(oldVid) ){
+ return old2NewVidMap.get(oldVid);
+ }
+ else {
+ throw new Exception(" ERROR - could not find VID translation for original VID = " + oldVid );
+ }
+ }
+
+
+ private String getTheVidForThisLine(String graphSonLine) throws Exception {
+
+ if( graphSonLine == null ){
+ throw new Exception(" ERROR - null graphSonLine passed to getTheVidForThisLine. ");
+ }
+
+ // We are assuming that the graphSonLine has the vertexId as the first ID:
+ // {"id":100995128,"label":"vertex","inE":{"hasPinterface":[{"id":"7lgg0e-2... etc...
+
+ // The vertexId for this line is the numeric part after the initial {"id":xxxxx up to the first comma
+ int x = graphSonLine.indexOf(':') + 1;
+ int y = graphSonLine.indexOf(',');
+ String initialVid = graphSonLine.substring(x,y);
+ if( initialVid != null && !initialVid.isEmpty() && initialVid.matches("^[0-9]+$") ){
+ return initialVid;
+ }
+ else {
+ throw new Exception(" ERROR - could not determine initial VID for graphSonLine: " + graphSonLine );
+ }
+ }
+
+
+ private String processThisLine(String graphSonLine, int passNum){
+
+ String passInfo = "";
+ if( passNum > 1 ) {
+ passInfo = " >> RETRY << pass # " + passNum + " ";
+ }
+
+ JSONObject jObj = new JSONObject();
+ String originalVid = "";
+
+ try{
+ jObj = new JSONObject(graphSonLine);
+ originalVid = jObj.get("id").toString();
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- Could not convert line to JsonObject [ " + graphSonLine + "]" );
+ LOGGER.debug(" -- ErrorMsg = [" +e.getMessage() + "]");
+
+ return(" JSON translation or getVid exception when processing this line [" + graphSonLine + "]");
+ }
+
+ // -----------------------------------------------------------------------------------------
+ // Note - this assumes that any vertices referred to by an edge will already be in the DB.
+ // -----------------------------------------------------------------------------------------
+ Vertex dbVtx = null;
+
+ String newVidStr = "";
+ Long newVidL = 0L;
+ try {
+ newVidStr = translateThisVid(originalVid);
+ newVidL = Long.parseLong(newVidStr);
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " translate VertexId before adding edges failed for this: vtxId = "
+ + originalVid + ". ErrorMsg = [" +e.getMessage() + "]");
+
+ return(" VID-translation error when processing this line ---");
+ }
+
+ try {
+ dbVtx = getVertexFromDbForVid(newVidStr);
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " READ Vertex from DB before adding edges failed for this: vtxId = " + originalVid
+ + ", newVidId = " + newVidL + ". ErrorMsg = [" +e.getMessage() + "]");
+
+ return(" ERROR getting Vertex based on VID = " + newVidStr + "]");
+ }
+
+
+ String edResStr = processEdgesForVtx( jObj, dbVtx, passInfo, originalVid );
+ if( edResStr.equals("") ){
+ // We will commit the edges by themselves in case the properties stuff below fails
+ try {
+ jg.tx().commit();
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " COMMIT FAILED adding EDGES for this vertex: vtxId = "
+ + originalVid + ". ErrorMsg = [" +e.getMessage() + "]");
+ return(" ERROR with committing edges for vertexId = " + originalVid );
+ }
+ }
+
+ // Add the properties that we didn't have when we added the 'bare-bones' vertex
+ String pResStr = processPropertiesForVtx( jObj, dbVtx, passInfo, originalVid );
+ if( pResStr.equals("") ){
+ try {
+ jg.tx().commit();
+ return "";
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " COMMIT FAILED adding Properties for this vertex: vtxId = "
+ + originalVid + ". ErrorMsg = [" +e.getMessage() + "]");
+ return(" ERROR with committing properties for vertexId = " + originalVid );
+ }
+ }
+ else {
+ LOGGER.debug("DEBUG " + passInfo + " Error processing Properties for this vertex: vtxId = "
+ + originalVid + ", [" + pResStr + "]");
+ return(" ERROR processing properties for vertexId = " + originalVid + ", [" + pResStr + "]");
+ }
+ }
+
+
+ private String processPropertiesForVtx( JSONObject jObj, Vertex dbVtx, String passInfo, String originalVid ){
+
+ try {
+ JSONObject propsOb = (JSONObject) jObj.get("properties");
+ String thisVNType = "unknown";
+ String thisVSot = "N/A";
+ Iterator <String> propsItr = propsOb.keys();
+ // First, loop through to get the nodeType and the node-level source-of-truth
+ while( propsItr.hasNext() ){
+ String pKey = propsItr.next();
+ JSONArray propsDetArr = propsOb.getJSONArray(pKey);
+ for( int i=0; i< propsDetArr.length(); i++ ){
+ JSONObject prop = propsDetArr.getJSONObject(i);
+ if( pKey.equals("aai-node-type")) {
+ thisVNType = (String) prop.get("value");
+ }
+ else if( pKey.equals("source-of-truth") ){
+ thisVSot = (String) prop.get("value");
+ }
+ }
+ }
+
+ // Get the key(s) for this nodeType
+ ArrayList <String> popSotArr = new ArrayList<String>();
+ if( keyNamesHash.containsKey(thisVNType) ) {
+ // This one is the node's key prop, so it will get SOT.
+ popSotArr = keyNamesHash.get(thisVNType);
+ }
+ // other required properties that we can give the SOT
+ popSotArr.add("aai-node-type");
+ popSotArr.add("aai-uri");
+ popSotArr.add("source-of-truth"); // odd, but they want it...
+
+ // 2nd pass, add each property to the db vertex.
+ propsItr = propsOb.keys();
+ while( propsItr.hasNext() ){
+ String pKey = propsItr.next();
+ JSONArray propsDetArr = propsOb.getJSONArray(pKey);
+ for( int i=0; i< propsDetArr.length(); i++ ){
+ JSONObject prop = propsDetArr.getJSONObject(i);
+ Object val = prop.get("value");
+ String sotVal = "N/A"; // default
+ if( popSotArr.contains(pKey) ){
+ sotVal = thisVSot; // This prop will have SOT populated.
+ }
+ dbVtx.property(pKey, val,"start-ts", currentTs,"source-of-truth",sotVal);
+ }
+ // The vertex itself gets the history data also - and since we're at
+ // the node level, we give "start-ts" it's own sub-properties just so it
+ // will match everybody else.
+ // Note - All vertices have always had a 'source-of-truth' property.
+ dbVtx.property("start-ts", currentTs,"start-ts", currentTs,"source-of-truth",thisVSot);
+ }
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " failure getting/setting properties for: vtxId = "
+ + originalVid + ". ErrorMsg = [" + e.getMessage() + "]");
+ return(" error processing properties for vtxId = " + originalVid);
+ }
+
+ return "";
+ }
+
+
+ private Vertex getVertexFromDbForVid( String vtxIdStr ) throws Exception {
+ Vertex thisVertex = null;
+ Long vtxIdL = 0L;
+
+ try {
+ vtxIdL = Long.parseLong(vtxIdStr);
+ Iterator <Vertex> vItr = jg.vertices(vtxIdL);
+ // Note - we only expect to find one vertex found for this ID.
+ while( vItr.hasNext() ){
+ thisVertex = vItr.next();
+ }
+ }
+ catch ( Exception e ){
+ String emsg = "Error finding vertex for vid = " + vtxIdStr + "[" + e.getMessage() + "]";
+ throw new Exception ( emsg );
+ }
+
+ if( thisVertex == null ){
+ String emsg = "Could not find vertex for passed vid = " + vtxIdStr;
+ throw new Exception ( emsg );
+ }
+
+ return thisVertex;
+ }
+
+
+ private String processEdgesForVtx( JSONObject jObj, Vertex dbVtx, String passInfo, String originalVid ){
+
+ // Process the edges for this vertex -- but, just the "OUT" ones so edges don't get added twice (once from
+ // each side of the edge).
+ JSONObject edOb = null;
+ try {
+ edOb = (JSONObject) jObj.get("outE");
+ }
+ catch (Exception e){
+ // There were no OUT edges. This is OK.
+ return "";
+ }
+
+ try {
+ if( edOb == null ){
+ // There were no OUT edges. This is OK. Not all nodes have out edges.
+ return "";
+ }
+ Iterator <String> edItr = edOb.keys();
+ while( edItr.hasNext() ){
+ String eLabel = edItr.next();
+ JSONArray edArr = edOb.getJSONArray(eLabel);
+ for( int i=0; i< edArr.length(); i++ ){
+ JSONObject eObj = edArr.getJSONObject(i);
+ String inVidStr = eObj.get("inV").toString();
+ String translatedInVidStr = translateThisVid(inVidStr);
+ Vertex newInVertex = getVertexFromDbForVid(translatedInVidStr);
+
+ // Note - addEdge automatically adds the edge in the OUT direction from the
+ // 'anchor' node that the call is being made from.
+ Edge tmpE = dbVtx.addEdge(eLabel, newInVertex);
+ JSONObject ePropsOb = null;
+ try {
+ ePropsOb = (JSONObject) eObj.get("properties");
+ }
+ catch (Exception e){
+ // NOTE - model definition related edges do not have edge properties. That is OK.
+ // Ie. when a model-element node has an "isA" edge to a "model-ver" node, that edge does
+ // not have edge properties on it.
+ }
+ if( ePropsOb != null ){
+ Iterator <String> ePropsItr = ePropsOb.keys();
+ while( ePropsItr.hasNext() ){
+ String pKey = ePropsItr.next();
+ tmpE.property(pKey, ePropsOb.get(pKey));
+ }
+ // For History add start-ts and source-of-truth
+ tmpE.property("start-ts", currentTs);
+ tmpE.property("source-of-truth", "N/A");
+ }
+ }
+ }
+ }
+ catch ( Exception e ){
+ String msg = " -- " + passInfo + " failure adding edge for: original vtxId = "
+ + originalVid + ". ErrorMsg = [" +e.getMessage() + "]";
+ LOGGER.debug( " -- " + msg );
+ LOGGER.debug(" -- now going to return/bail out of processEdgesForVtx" );
+ return(" >> " + msg );
+ }
+
+ return "";
+ }
+
+
+}
+
+
diff --git a/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java b/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java
index 3afd295..22ca0a2 100644
--- a/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java
+++ b/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java
@@ -26,8 +26,10 @@ import java.util.concurrent.Callable;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
import org.janusgraph.core.JanusGraph;
-
-import com.att.eelf.configuration.EELFLogger;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
@@ -35,7 +37,7 @@ import com.google.gson.JsonParser;
public class PartialVertexLoader implements Callable<HashMap<String,String>>{
- private EELFLogger LOGGER;
+ private Logger LOGGER;
private JanusGraph jg;
private String fName;
@@ -45,7 +47,7 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{
private int maxAllowedErrors;
public PartialVertexLoader (JanusGraph graph, String fn, Long vertDelay, Long failurePause,
- Long retryDelay, int maxErrors, EELFLogger elfLog ){
+ Long retryDelay, int maxErrors, Logger elfLog ){
jg = graph;
fName = fn;
vertAddDelayMs = vertDelay;
@@ -137,7 +139,8 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{
LOGGER.debug(" --- Failed in the main loop for Buffered-Reader item # " + entryCount +
", fName = " + fName );
LOGGER.debug(" --- msg = " + e.getMessage() );
- e.printStackTrace();
+ AAIException ae = new AAIException("AAI_6128", e , "Failed in the main loop for Buffered-Reader item");
+ ErrorLogHelper.logException(ae);
throw e;
}
@@ -165,8 +168,9 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{
LOGGER.debug(" -- addVertex FAILED for RETRY for vtxId = " +
failedVidStr + ", label = [" + failedLabel +
"]. ErrorMsg = [" +e.getMessage() + "]" );
- e.printStackTrace();
- if( retryFailureCount > maxAllowedErrors ) {
+ AAIException ae = new AAIException("AAI_6128", e , "addVertex FAILED for RETRY");
+ ErrorLogHelper.logException(ae);
+ if( retryFailureCount > maxAllowedErrors ) {
LOGGER.debug(">>> Abandoning PartialVertexLoader() because " +
"Max Allowed Error count was exceeded for this thread. (max = " +
maxAllowedErrors + ". ");
@@ -189,7 +193,8 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{
+ ", label = [" + failedLabel + "]. ErrorMsg = [" + e.getMessage()
+ "]. This vertex will not be tried again. ");
- e.printStackTrace();
+ AAIException ae = new AAIException("AAI_6128", e , "--POSSIBLE ERROR-- COMMIT FAILED for RETRY");
+ ErrorLogHelper.logException(ae);
if( retryFailureCount > maxAllowedErrors ) {
LOGGER.debug(">>> Abandoning PartialVertexLoader() because " +
"Max Allowed Error count was exceeded for this thread. (max = " +
@@ -205,7 +210,8 @@ public class PartialVertexLoader implements Callable<HashMap<String,String>>{
}
catch ( Exception e ){
LOGGER.debug(" -- error in RETRY block. ErrorMsg = [" +e.getMessage() + "]" );
- e.printStackTrace();
+ AAIException ae = new AAIException("AAI_6128", e , " -- error in RETRY block.");
+ ErrorLogHelper.logException(ae);
throw e;
}
diff --git a/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java b/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java
index 493678b..12b3099 100644
--- a/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java
+++ b/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java
@@ -22,100 +22,149 @@ package org.onap.aai.datasnapshot;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.Iterator;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
+import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.janusgraph.core.JanusGraph;
+import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.logging.filter.base.ONAPComponents;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PrintVertexDetails implements Runnable{
+ private Logger LOGGER;
+
private JanusGraph jg;
private String fname;
- private ArrayList<Vertex> vtxList;
+ private ArrayList<Long> vtxIdList;
private Boolean debugOn;
private long debugDelayMs;
private String snapshotType;
static final byte[] newLineBytes = "\n".getBytes();
+
+ private AaiScheduledTaskAuditLog auditLog;
+
- public PrintVertexDetails (JanusGraph graph, String fn, ArrayList<Vertex> vL, Boolean debugFlag, long debugAddDelayTime, String snapshotType){
+ public PrintVertexDetails (JanusGraph graph, String fn, ArrayList<Long> vIdL, Boolean debugFlag,
+ long debugAddDelayTime, String snapshotType, Logger elfLog){
jg = graph;
fname = fn;
- vtxList = vL;
+ vtxIdList = vIdL;
debugOn = debugFlag;
debugDelayMs = debugAddDelayTime;
this.snapshotType = snapshotType;
+ LOGGER = elfLog;
+ this.auditLog = new AaiScheduledTaskAuditLog();
}
-
- public void run(){
- if( debugOn ){
- // This is much slower, but sometimes we need to find out which single line is causing a failure
- try{
- int okCount = 0;
- int failCount = 0;
- Long debugDelayMsL = new Long(debugDelayMs);
- FileOutputStream subFileStr = new FileOutputStream(fname);
- Iterator <Vertex> vSubItr = vtxList.iterator();
- GraphWriter graphWriter = null;
- if("gryo".equalsIgnoreCase(snapshotType)){
- graphWriter = jg.io(IoCore.gryo()).writer().create();
- } else {
- graphWriter = jg.io(IoCore.graphson()).writer().create();
- }
- while( vSubItr.hasNext() ){
- Long vertexIdL = 0L;
- String aaiNodeType = "";
- String aaiUri = "";
- String aaiUuid = "";
- try {
- Vertex tmpV = vSubItr.next();
- vertexIdL = (Long) tmpV.id();
- aaiNodeType = (String) tmpV.property("aai-node-type").orElse(null);
- aaiUri = (String) tmpV.property("aai-uri").orElse(null);
- aaiUuid = (String) tmpV.property("aai-uuid").orElse(null);
-
- Thread.sleep(debugDelayMsL); // Make sure it doesn't bump into itself
- graphWriter.writeVertex(subFileStr, tmpV, Direction.BOTH);
- subFileStr.write(newLineBytes);
- okCount++;
+
+
+ public void run(){
+ LOGGER = LoggerFactory.getLogger(PrintVertexDetails.class);
+ auditLog.logBefore("printVertexDetails", ONAPComponents.AAI.toString());
+ try {
+ if (debugOn) {
+ // This is much slower, but sometimes we need to find out which single line is
+ // causing a failure
+ try {
+ int okCount = 0;
+ int failCount = 0;
+ Long debugDelayMsL = new Long(debugDelayMs);
+ FileOutputStream subFileStr = new FileOutputStream(fname);
+
+ GraphWriter graphWriter = null;
+ if ("gryo".equalsIgnoreCase(snapshotType)) {
+ graphWriter = jg.io(IoCore.gryo()).writer().create();
+ } else {
+ graphWriter = jg.io(IoCore.graphson()).writer().create();
}
- catch(Exception e) {
- failCount++;
- System.out.println(" >> DEBUG MODE >> Failed at: VertexId = [" + vertexIdL +
- "], aai-node-type = [" + aaiNodeType +
- "], aai-uuid = [" + aaiUuid +
- "], aai-uri = [" + aaiUri + "]. " );
- e.printStackTrace();
+
+ GraphTraversalSource gts = jg.traversal();
+ ArrayList<Vertex> vtxList = new ArrayList<Vertex> ();
+ GraphTraversal<Vertex, Vertex> gt = gts.V(vtxIdList);
+ while( gt.hasNext() ) {
+ vtxList.add(gt.next());
}
+ Iterator<Vertex> vSubItr = vtxList.iterator();
+ while (vSubItr.hasNext()) {
+ Long vertexIdL = 0L;
+ String aaiNodeType = "";
+ String aaiUri = "";
+ String aaiUuid = "";
+ try {
+ Vertex tmpV = vSubItr.next();
+ vertexIdL = (Long) tmpV.id();
+ aaiNodeType = (String) tmpV.property("aai-node-type").orElse(null);
+ aaiUri = (String) tmpV.property("aai-uri").orElse(null);
+ aaiUuid = (String) tmpV.property("aai-uuid").orElse(null);
+
+ Thread.sleep(debugDelayMsL); // Make sure it doesn't bump into itself
+ graphWriter.writeVertex(subFileStr, tmpV, Direction.BOTH);
+ subFileStr.write(newLineBytes);
+ okCount++;
+ } catch (Exception e) {
+ failCount++;
+ String fmsg = " >> DEBUG MODE >> Failed at: VertexId = [" + vertexIdL
+ + "], aai-node-type = [" + aaiNodeType + "], aai-uuid = [" + aaiUuid
+ + "], aai-uri = [" + aaiUri + "]. ";
+ System.out.println(fmsg);
+ LOGGER.debug(" PrintVertexDetails " + fmsg);
+ // e.printStackTrace();
+ }
+ }
+ System.out.println(" -- Printed " + okCount + " vertexes out to " + fname + ", with " + failCount
+ + " failed.");
+ subFileStr.close();
+ } catch (Exception e) {
+ AAIException ae = new AAIException("AAI_6128", e , "Error running PrintVertexDetails in debugon");
+ ErrorLogHelper.logException(ae);
}
- System.out.println(" -- Printed " + okCount + " vertexes out to " + fname +
- ", with " + failCount + " failed.");
- subFileStr.close();
- }
- catch(Exception e){
- e.printStackTrace();
- }
- }
- else {
- // Not in DEBUG mode, so we'll do all the nodes in one group
- try{
- int count = vtxList.size();
- Iterator <Vertex> vSubItr = vtxList.iterator();
- FileOutputStream subFileStr = new FileOutputStream(fname);
- if ("gryo".equalsIgnoreCase(snapshotType)) {
- jg.io(IoCore.gryo()).writer().create().writeVertices(subFileStr, vSubItr, Direction.BOTH);
- } else {
- jg.io(IoCore.graphson()).writer().create().writeVertices(subFileStr, vSubItr, Direction.BOTH);
+ } else {
+ // Not in DEBUG mode, so we'll do all the nodes in one group
+ GraphTraversalSource gts = jg.traversal();
+ ArrayList<Vertex> vtxList = new ArrayList<Vertex> ();
+ GraphTraversal<Vertex, Vertex> gt = gts.V(vtxIdList);
+ while( gt.hasNext() ) {
+ vtxList.add(gt.next());
+ }
+
+ try {
+ int count = vtxList.size();
+ Iterator<Vertex> vSubItr = vtxList.iterator();
+ FileOutputStream subFileStr = new FileOutputStream(fname);
+ if ("gryo".equalsIgnoreCase(snapshotType)) {
+ jg.io(IoCore.gryo()).writer().create().writeVertices(subFileStr, vSubItr, Direction.BOTH);
+ } else {
+ jg.io(IoCore.graphson()).writer().create().writeVertices(subFileStr, vSubItr, Direction.BOTH);
+ }
+ subFileStr.close();
+ String pmsg = " -- Printed " + count + " vertexes out to " + fname;
+ System.out.println(pmsg);
+ LOGGER.debug(" PrintVertexDetails " + pmsg);
+ } catch (Exception e) {
+ AAIException ae = new AAIException("AAI_6128", e , "Error running PrintVertexDetails in else");
+ ErrorLogHelper.logException(ae);
}
- subFileStr.close();
- System.out.println(" -- Printed " + count + " vertexes out to " + fname);
}
- catch(Exception e){
- e.printStackTrace();
- }
}
+ catch(Exception e){
+ AAIException ae = new AAIException("AAI_6128", e , "Error running PrintVertexDetails");
+ ErrorLogHelper.logException(ae);
+ }
+ finally {
+ // Make sure the transaction this thread was using is freed up.
+ jg.tx().commit();
+ jg.tx().close();
+ }
+ auditLog.logAfter();
}
} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/db/schema/AuditOXM.java b/src/main/java/org/onap/aai/db/schema/AuditOXM.java
index d7f7314..e976bed 100644
--- a/src/main/java/org/onap/aai/db/schema/AuditOXM.java
+++ b/src/main/java/org/onap/aai/db/schema/AuditOXM.java
@@ -19,8 +19,8 @@
*/
package org.onap.aai.db.schema;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Multimap;
import org.janusgraph.core.Cardinality;
import org.janusgraph.core.Multiplicity;
@@ -43,7 +43,7 @@ import java.util.stream.Collectors;
public class AuditOXM extends Auditor {
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(AuditOXM.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(AuditOXM.class);
private Set<Introspector> allObjects;
private EdgeIngestor ingestor;
diff --git a/src/main/java/org/onap/aai/db/schema/ScriptDriver.java b/src/main/java/org/onap/aai/db/schema/ScriptDriver.java
index 6b36adc..a634e0b 100644
--- a/src/main/java/org/onap/aai/db/schema/ScriptDriver.java
+++ b/src/main/java/org/onap/aai/db/schema/ScriptDriver.java
@@ -33,8 +33,6 @@ import org.onap.aai.edges.EdgeIngestor;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.logging.ErrorLogHelper;
import org.onap.aai.logging.ErrorObjectFormatException;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.logging.LoggingContext.StatusCode;
import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.ExceptionTranslator;
@@ -56,15 +54,6 @@ public class ScriptDriver {
public static void main (String[] args) throws AAIException, IOException, ConfigurationException, ErrorObjectFormatException {
CommandLineArgs cArgs = new CommandLineArgs();
- LoggingContext.init();
- LoggingContext.component("DBSchemaScriptDriver");
- LoggingContext.partnerName("NA");
- LoggingContext.targetEntity("AAI");
- LoggingContext.requestId(UUID.randomUUID().toString());
- LoggingContext.serviceName("AAI");
- LoggingContext.targetServiceName("main");
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
ErrorLogHelper.loadProperties();
new JCommander(cArgs, args);
@@ -84,8 +73,6 @@ public class ScriptDriver {
} catch (Exception e) {
AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
throw aai;
}
@@ -136,4 +123,4 @@ class CommandLineArgs {
public String type = "graph";
-}
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/dbgen/DupeTool.java b/src/main/java/org/onap/aai/dbgen/DupeTool.java
index 800e9a4..4164ee8 100644
--- a/src/main/java/org/onap/aai/dbgen/DupeTool.java
+++ b/src/main/java/org/onap/aai/dbgen/DupeTool.java
@@ -20,8 +20,8 @@
package org.onap.aai.dbgen;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
@@ -40,8 +40,6 @@ import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.introspection.ModelType;
import org.onap.aai.logging.ErrorLogHelper;
import org.onap.aai.logging.LogFormatTools;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.logging.LoggingContext.StatusCode;
import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
@@ -57,7 +55,7 @@ import java.util.Map.Entry;
public class DupeTool {
- private static final EELFLogger logger = EELFManager.getInstance().getLogger(DupeTool.class.getSimpleName());
+ private static final Logger logger = LoggerFactory.getLogger(DupeTool.class.getSimpleName());
private static final String FROMAPPID = "AAI-DB";
private static final String TRANSID = UUID.randomUUID().toString();
@@ -87,14 +85,12 @@ public class DupeTool {
public void execute(String[] args){
- String defVersion = "v16";
+ String defVersion = "v18";
try {
defVersion = AAIConfig.get(AAIConstants.AAI_DEFAULT_API_VERSION_PROP);
} catch (AAIException ae) {
String emsg = "Error trying to get default API Version property \n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(emsg);
exit(0);
}
@@ -104,8 +100,6 @@ public class DupeTool {
try {
loader = loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion());
} catch (Exception ex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
logger.error("ERROR - Could not do the moxyMod.init() " + LogFormatTools.getStackTop(ex));
exit(1);
}
@@ -155,8 +149,6 @@ public class DupeTool {
if (thisArg.equals("-nodeType")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" No value passed with -nodeType option. ");
exit(0);
}
@@ -165,8 +157,6 @@ public class DupeTool {
} else if (thisArg.equals("-sleepMinutes")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error("No value passed with -sleepMinutes option.");
exit(0);
}
@@ -174,8 +164,6 @@ public class DupeTool {
try {
sleepMinutes = Integer.parseInt(nextArg);
} catch (Exception e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error("Bad value passed with -sleepMinutes option: ["
+ nextArg + "]");
exit(0);
@@ -184,8 +172,6 @@ public class DupeTool {
} else if (thisArg.equals("-maxFix")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error("No value passed with -maxFix option.");
exit(0);
}
@@ -193,8 +179,6 @@ public class DupeTool {
try {
maxRecordsToFix = Integer.parseInt(nextArg);
} catch (Exception e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error("Bad value passed with -maxFix option: ["
+ nextArg + "]");
exit(0);
@@ -203,8 +187,6 @@ public class DupeTool {
} else if (thisArg.equals("-timeWindowMinutes")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error("No value passed with -timeWindowMinutes option.");
exit(0);
}
@@ -212,8 +194,6 @@ public class DupeTool {
try {
timeWindowMinutes = Integer.parseInt(nextArg);
} catch (Exception e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error("Bad value passed with -timeWindowMinutes option: ["
+ nextArg + "]");
exit(0);
@@ -228,8 +208,6 @@ public class DupeTool {
} else if (thisArg.equals("-userId")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" No value passed with -userId option. ");
exit(0);
}
@@ -238,16 +216,12 @@ public class DupeTool {
} else if (thisArg.equals("-params4Collect")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" No value passed with -params4Collect option. ");
exit(0);
}
filterParams = args[i];
argStr4Msg = argStr4Msg + " " + filterParams;
} else {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" Unrecognized argument passed to DupeTool: ["
+ thisArg + "]. ");
logger.error(" Valid values are: -action -userId -vertexId -edgeId -overRideProtection ");
@@ -260,8 +234,6 @@ public class DupeTool {
if ((userIdVal.length() < 6) || userIdVal.toUpperCase().equals("AAIADMIN")) {
String emsg = "userId parameter is required. [" + userIdVal + "] passed to DupeTool(). userId must be not empty and not aaiadmin \n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(emsg);
exit(0);
}
@@ -270,8 +242,6 @@ public class DupeTool {
if (nodeTypeVal.equals("")) {
String emsg = " nodeType is a required parameter for DupeTool().\n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(emsg);
exit(0);
} else {
@@ -287,7 +257,7 @@ public class DupeTool {
String msg = "";
msg = "DupeTool called with these params: [" + argStr4Msg + "]";
System.out.println(msg);
- logger.info(msg);
+ logger.debug(msg);
// Determine what the key fields are for this nodeType (and we want them ordered)
ArrayList<String> keyPropNamesArr = new ArrayList<String>(obj.getKeys());
@@ -311,8 +281,6 @@ public class DupeTool {
} catch (AAIException ae) {
String emsg = "Error trying to get initial set of nodes to check. \n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(emsg);
exit(0);
}
@@ -321,13 +289,13 @@ public class DupeTool {
msg = " No vertices found to check. Used nodeType = [" + nodeTypeVal
+ "], windowMinutes = " + timeWindowMinutes
+ ", filterData = [" + filterParams + "].";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
exit(0);
} else {
msg = " Found " + verts2Check.size() + " nodes of type " + nodeTypeVal
+ " to check using passed filterParams and windowStartTime. ";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
}
@@ -349,15 +317,15 @@ public class DupeTool {
}
msg = " Found " + firstPassDupeSets.size() + " sets of duplicates for this request. ";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
if (firstPassDupeSets.size() > 0) {
msg = " Here is what they look like: ";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
for (int x = 0; x < firstPassDupeSets.size(); x++) {
msg = " Set " + x + ": [" + firstPassDupeSets.get(x) + "] ";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
showNodeDetailsForADupeSet(gt1, firstPassDupeSets.get(x), logger);
}
@@ -367,7 +335,7 @@ public class DupeTool {
ArrayList<String> dupeSetsToFix = new ArrayList<String>();
if (autoFix && firstPassDupeSets.size() == 0) {
msg = "AutoFix option is on, but no dupes were found on the first pass. Nothing to fix.";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
} else if (autoFix) {
// We will try to fix any dupes that we can - but only after sleeping for a
@@ -375,13 +343,13 @@ public class DupeTool {
try {
msg = "\n\n----------- About to sleep for " + sleepMinutes + " minutes."
+ " -----------\n\n";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
int sleepMsec = sleepMinutes * 60 * 1000;
Thread.sleep(sleepMsec);
} catch (InterruptedException ie) {
msg = "\n >>> Sleep Thread has been Interrupted <<< ";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
exit(0);
}
@@ -401,16 +369,16 @@ public class DupeTool {
dupeSetsToFix = figureWhichDupesStillNeedFixing(firstPassDupeSets, secondPassDupeSets, logger);
msg = "\nAfter running a second pass, there were " + dupeSetsToFix.size()
+ " sets of duplicates that we think can be deleted. ";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
if (dupeSetsToFix.size() > 0) {
msg = " Here is what the sets look like: ";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
for (int x = 0; x < dupeSetsToFix.size(); x++) {
msg = " Set " + x + ": [" + dupeSetsToFix.get(x) + "] ";
- logger.info(msg);
+ logger.debug(msg);
System.out.println(msg);
showNodeDetailsForADupeSet(gt2, dupeSetsToFix.get(x), logger);
}
@@ -425,7 +393,7 @@ public class DupeTool {
+ ". No nodes will be deleted. (use the"
+ " -maxFix option to override this limit.)";
System.out.println(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
} else {
// Call the routine that fixes known dupes
didSomeDeletesFlag = deleteNonKeepers(gt2, dupeSetsToFix, logger);
@@ -500,15 +468,6 @@ public class DupeTool {
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
MDC.put("logFilenameAppender", DupeTool.class.getSimpleName());
- LoggingContext.init();
- LoggingContext.partnerName(FROMAPPID);
- LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.component("dupeTool");
- LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.targetServiceName("main");
- LoggingContext.requestId(TRANSID);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
@@ -522,8 +481,6 @@ public class DupeTool {
} catch (Exception e) {
AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
logger.error("Problems running DupeTool "+aai.getMessage());
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
throw aai;
}
@@ -550,7 +507,7 @@ public class DupeTool {
String fromAppId, Graph g, String version, String nType,
ArrayList<Vertex> passedVertList,
ArrayList<String> keyPropNamesArr,
- Boolean specialTenantRule, Loader loader, EELFLogger logger) {
+ Boolean specialTenantRule, Loader loader, Logger logger) {
ArrayList<String> returnList = new ArrayList<String>();
@@ -643,14 +600,14 @@ public class DupeTool {
* @param dbMaps the db maps
* @param keyPropNamesArr Array (ordered) of keyProperty names
* @param specialTenantRule flag
- * @param EELFLogger the logger
+ * @param Logger the logger
* @return the array list
*/
private ArrayList<String> getDupeSets4DependentNodes(String transId,
String fromAppId, Graph g, String version, String nType,
ArrayList<Vertex> passedVertList,
ArrayList<String> keyPropNamesArr, Loader loader,
- Boolean specialTenantRule, EELFLogger logger) {
+ Boolean specialTenantRule, Logger logger) {
// This is for nodeTypes that DEPEND ON A PARENT NODE FOR UNIQUNESS
@@ -766,15 +723,13 @@ public class DupeTool {
}// End of getDupeSets4DependentNodes()
- private Graph getGraphTransaction(JanusGraph graph, EELFLogger logger) {
+ private Graph getGraphTransaction(JanusGraph graph, Logger logger) {
Graph gt = null;
try {
if (graph == null) {
String emsg = "could not get graph object in DupeTool. \n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
logger.error(emsg);
exit(0);
}
@@ -787,15 +742,11 @@ public class DupeTool {
} catch (AAIException e1) {
String msg = e1.getErrorObject().toString();
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(msg);
exit(0);
} catch (Exception e2) {
String msg = e2.toString();
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
logger.error(msg);
exit(0);
}
@@ -805,39 +756,35 @@ public class DupeTool {
}// End of getGraphTransaction()
- public void showNodeInfo(EELFLogger logger, Vertex tVert, Boolean displayAllVidsFlag) {
+ public void showNodeInfo(Logger logger, Vertex tVert, Boolean displayAllVidsFlag) {
try {
Iterator<VertexProperty<Object>> pI = tVert.properties();
String infStr = ">>> Found Vertex with VertexId = " + tVert.id() + ", properties: ";
System.out.println(infStr);
- logger.info(infStr);
+ logger.debug(infStr);
while (pI.hasNext()) {
VertexProperty<Object> tp = pI.next();
infStr = " [" + tp.key() + "|" + tp.value() + "] ";
System.out.println(infStr);
- logger.info(infStr);
+ logger.debug(infStr);
}
ArrayList<String> retArr = collectEdgeInfoForNode(logger, tVert, displayAllVidsFlag);
for (String infoStr : retArr) {
System.out.println(infoStr);
- logger.info(infoStr);
+ logger.debug(infoStr);
}
} catch (Exception e) {
String warnMsg = " -- Error -- trying to display edge info. [" + e.getMessage() + "]";
System.out.println(warnMsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
logger.warn(warnMsg);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
}
}// End of showNodeInfo()
- public ArrayList<String> collectEdgeInfoForNode(EELFLogger logger, Vertex tVert, boolean displayAllVidsFlag) {
+ public ArrayList<String> collectEdgeInfoForNode(Logger logger, Vertex tVert, boolean displayAllVidsFlag) {
ArrayList<String> retArr = new ArrayList<String>();
Direction dir = Direction.OUT;
for (int i = 0; i <= 1; i++) {
@@ -909,7 +856,7 @@ public class DupeTool {
* @throws AAIException the AAI exception
*/
public ArrayList<Vertex> getNodeJustUsingKeyParams(String transId, String fromAppId, Graph graph, String nodeType,
- HashMap<String, Object> keyPropsHash, String apiVersion, EELFLogger logger) throws AAIException {
+ HashMap<String, Object> keyPropsHash, String apiVersion, Logger logger) throws AAIException {
ArrayList<Vertex> retVertList = new ArrayList<Vertex>();
@@ -956,11 +903,7 @@ public class DupeTool {
throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n");
}
} catch (Exception ex) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(" ERROR trying to get node for: [" + propsAndValuesForMsg + "] " + LogFormatTools.getStackTop(ex));
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
}
if (verts != null) {
@@ -995,7 +938,7 @@ public class DupeTool {
*/
public ArrayList<Vertex> figureOutNodes2Check(String transId, String fromAppId,
Graph graph, String nodeType, long windowStartTime,
- String propsString, EELFLogger logger) throws AAIException {
+ String propsString, Logger logger) throws AAIException {
ArrayList<Vertex> retVertList = new ArrayList<Vertex>();
String msg = "";
@@ -1008,8 +951,6 @@ public class DupeTool {
if (firstPipeLoc <= 0) {
msg = "Bad props4Collect passed: [" + propsString + "]. \n Expecting a format like, 'propName1|propVal1,propName2|propVal2'";
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(msg);
exit(0);
}
@@ -1021,8 +962,6 @@ public class DupeTool {
if (pipeLoc <= 0) {
msg = "Bad propsString passed: [" + propsString + "]. \n Expecting a format like, 'propName1|propVal1,propName2|propVal2'";
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(msg);
exit(0);
} else {
@@ -1037,8 +976,6 @@ public class DupeTool {
if (tgQ == null) {
msg = "Bad JanusGraphQuery object. ";
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
logger.error(msg);
exit(0);
} else {
@@ -1081,14 +1018,14 @@ public class DupeTool {
* @param g the g
* @param dupeVertexList the dupe vertex list
* @param ver the ver
- * @param EELFLogger the logger
+ * @param Logger the logger
* @return Vertex
* @throws AAIException the AAI exception
*/
public Vertex getPreferredDupe(String transId,
String fromAppId, Graph g,
ArrayList<Vertex> dupeVertexList, String ver,
- Boolean specialTenantRule, Loader loader, EELFLogger logger)
+ Boolean specialTenantRule, Loader loader, Logger logger)
throws AAIException {
// This method assumes that it is being passed a List of
@@ -1168,13 +1105,13 @@ public class DupeTool {
* @param vtxB the vtx B
* @param ver the ver
* @param boolean specialTenantRuleFlag flag
- * @param EELFLogger the logger
+ * @param Logger the logger
* @return Vertex
* @throws AAIException the AAI exception
*/
public Vertex pickOneOfTwoDupes(String transId,
String fromAppId, GraphTraversalSource gts, Vertex vtxA,
- Vertex vtxB, String ver, Boolean specialTenantRule, Loader loader, EELFLogger logger) throws AAIException {
+ Vertex vtxB, String ver, Boolean specialTenantRule, Loader loader, Logger logger) throws AAIException {
Vertex nullVtx = null;
Vertex preferredVtx = null;
@@ -1352,13 +1289,13 @@ public class DupeTool {
String infMsg = " WARNING >>> we are using the special tenant rule to choose to " +
" delete tenant vtxId = " + vidA + ", and keep tenant vtxId = " + vidB;
System.out.println(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
preferredVtx = vtxB;
} else if (nodeTypesConn2B.containsKey("vserver") && nodeTypesConn2A.containsKey("service-subscription")) {
String infMsg = " WARNING >>> we are using the special tenant rule to choose to " +
" delete tenant vtxId = " + vidB + ", and keep tenant vtxId = " + vidA;
System.out.println(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
preferredVtx = vtxA;
}
}
@@ -1474,11 +1411,11 @@ public class DupeTool {
*
* @param g the g
* @param dupeInfoList the dupe info string
- * @param logger the EELFLogger
+ * @param logger the Logger
* @return the boolean
*/
private Boolean deleteNonKeepers(Graph g,
- ArrayList<String> dupeInfoList, EELFLogger logger) {
+ ArrayList<String> dupeInfoList, Logger logger) {
// This assumes that each dupeInfoString is in the format of
// pipe-delimited vid's followed by either "keepVid=xyz" or "keepVid=UNDETERMINED"
@@ -1501,11 +1438,11 @@ public class DupeTool {
*
* @param g the g
* @param dupeSetStr the dupe string
- * @param logger the EELFLogger
+ * @param logger the Logger
* @return the boolean
*/
private Boolean deleteNonKeeperForOneSet(Graph g,
- String dupeInfoString, EELFLogger logger) {
+ String dupeInfoString, Logger logger) {
Boolean deletedSomething = false;
// This assumes that each dupeInfoString is in the format of
@@ -1536,11 +1473,7 @@ public class DupeTool {
if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
String emsg = "Bad format. Expecting KeepVid=999999";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(emsg);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
return false;
} else {
String keepVidStr = prefArr[1];
@@ -1556,21 +1489,17 @@ public class DupeTool {
Vertex vtx = g.traversal().V(longVertId).next();
String msg = "--->>> We will delete node with VID = " + thisVid + " <<<---";
System.out.println(msg);
- logger.info(msg);
+ logger.debug(msg);
vtx.remove();
} catch (Exception e) {
okFlag = false;
String emsg = "ERROR trying to delete VID = " + thisVid + ", [" + e + "]";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(emsg);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
}
if (okFlag) {
String infMsg = " DELETED VID = " + thisVid;
- logger.info(infMsg);
+ logger.debug(infMsg);
System.out.println(infMsg);
deletedSomething = true;
}
@@ -1578,11 +1507,7 @@ public class DupeTool {
} else {
String emsg = "ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = ["
+ dupeInfoString + "]";
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(emsg);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
System.out.println(emsg);
return false;
}
@@ -1601,11 +1526,11 @@ public class DupeTool {
*
* @param tvx the vertex to pull the properties from
* @param keyPropertyNames ArrayList (ordered) of key prop names
- * @param logger the EELFLogger
+ * @param logger the Logger
* @return a hashMap of the propertyNames/values
*/
private HashMap<String, Object> getNodeKeyVals(Vertex tvx,
- ArrayList<String> keyPropNamesArr, EELFLogger logger) {
+ ArrayList<String> keyPropNamesArr, Logger logger) {
HashMap<String, Object> retHash = new HashMap<String, Object>();
Iterator<String> propItr = keyPropNamesArr.iterator();
@@ -1629,11 +1554,11 @@ public class DupeTool {
* @param fromAppId the from app id
* @param graph the graph
* @param vtx
- * @param EELFLogger
+ * @param Logger
* @return true if aai-uri is populated and the aai-uri-index points to this vtx
* @throws AAIException the AAI exception
*/
- private Boolean checkAaiUriOk( GraphTraversalSource graph, Vertex origVtx, EELFLogger eLogger )
+ private Boolean checkAaiUriOk( GraphTraversalSource graph, Vertex origVtx, Logger eLogger )
throws AAIException{
String aaiUriStr = "";
try {
@@ -1677,8 +1602,6 @@ public class DupeTool {
}
}
catch( Exception ex ){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
eLogger.error(" ERROR trying to get node with aai-uri: [" + aaiUriStr + "]" + LogFormatTools.getStackTop(ex));
}
return true;
@@ -1691,11 +1614,11 @@ public class DupeTool {
*
* @param tvx the vertex to pull the properties from
* @param keyPropertyNames collection of key prop names
- * @param logger the EELFLogger
+ * @param logger the Logger
* @return a String of concatenated values
*/
private String getNodeKeyValString(Vertex tvx,
- ArrayList<String> keyPropNamesArr, EELFLogger logger) {
+ ArrayList<String> keyPropNamesArr, Logger logger) {
// -- NOTE -- for what we're using this for, we would need to
// guarantee that the properties are always in the same order
@@ -1719,11 +1642,11 @@ public class DupeTool {
*
* @param firstPassDupeSets from the first pass
* @param secondPassDupeSets from the second pass
- * @param EELFLogger logger
+ * @param Logger logger
* @return commonDupeSets that are common to both passes and have a determined keeper
*/
private ArrayList<String> figureWhichDupesStillNeedFixing(ArrayList<String> firstPassDupeSets,
- ArrayList<String> secondPassDupeSets, EELFLogger logger) {
+ ArrayList<String> secondPassDupeSets, Logger logger) {
ArrayList<String> common2BothSet = new ArrayList<String>();
@@ -1815,7 +1738,7 @@ public class DupeTool {
private HashMap<String, ArrayList<String>> makeKeeperHashOfDupeStrings(ArrayList<String> dupeSets,
- ArrayList<String> excludeSets, EELFLogger logger) {
+ ArrayList<String> excludeSets, Logger logger) {
HashMap<String, ArrayList<String>> keeperHash = new HashMap<String, ArrayList<String>>();
@@ -1852,7 +1775,7 @@ public class DupeTool {
String infMsg = "Bad format in figureWhichDupesStillNeedFixing(). Expecting " +
" KeepVid=999999 but string looks like: [" + tmpSetStr + "]";
System.out.println(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
} else {
keeperHash.put(prefArr[0], delIdArr);
}
@@ -1871,10 +1794,10 @@ public class DupeTool {
*
* @param g the g
* @param dupeInfoString
- * @param logger the EELFLogger
+ * @param logger the Logger
* @return void
*/
- private void showNodeDetailsForADupeSet(Graph g, String dupeInfoString, EELFLogger logger) {
+ private void showNodeDetailsForADupeSet(Graph g, String dupeInfoString, Logger logger) {
// dang... parsing this string once again...
@@ -1894,7 +1817,7 @@ public class DupeTool {
if (prefString.equals("KeepVid=UNDETERMINED")) {
String msg = " Our algorithm cannot choose from among these, so they will all be kept. -------\n";
System.out.println(msg);
- logger.info(msg);
+ logger.debug(msg);
} else {
// If we know which to keep, then the prefString should look
// like, "KeepVid=12345"
@@ -1902,16 +1825,12 @@ public class DupeTool {
if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
String emsg = "Bad format. Expecting KeepVid=999999";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(emsg);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
} else {
String keepVidStr = prefArr[1];
String msg = " vid = " + keepVidStr + " is the one that we would KEEP. ------\n";
System.out.println(msg);
- logger.info(msg);
+ logger.debug(msg);
}
}
}
@@ -1921,7 +1840,7 @@ public class DupeTool {
private int graphIndex = 1;
- public JanusGraph setupGraph(EELFLogger logger) {
+ public JanusGraph setupGraph(Logger logger) {
JanusGraph JanusGraph = null;
@@ -1945,7 +1864,7 @@ public class DupeTool {
return JanusGraph;
}
- public void closeGraph(JanusGraph graph, EELFLogger logger) {
+ public void closeGraph(JanusGraph graph, Logger logger) {
try {
if ("inmemory".equals(graphType)) {
diff --git a/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java b/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java
index dec6a85..2e1bc4b 100644
--- a/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java
+++ b/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java
@@ -1,906 +1,899 @@
-/**
- * ============LICENSE_START=======================================================
- * org.onap.aai
- * ================================================================================
- * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END=========================================================
- */
-package org.onap.aai.dbgen;
-
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import org.apache.tinkerpop.gremlin.process.traversal.P;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
-import org.apache.tinkerpop.gremlin.structure.Element;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.node.ObjectNode;
-import org.codehaus.jackson.type.TypeReference;
-import org.onap.aai.config.PropertyPasswordConfiguration;
-import org.onap.aai.db.props.AAIProperties;
-import org.onap.aai.dbmap.DBConnectionType;
-import org.onap.aai.dbmap.InMemoryGraph;
-import org.onap.aai.edges.EdgeIngestor;
-import org.onap.aai.edges.EdgeRule;
-import org.onap.aai.edges.EdgeRuleQuery;
-import org.onap.aai.edges.enums.AAIDirection;
-import org.onap.aai.edges.enums.EdgeType;
-import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException;
-import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException;
-import org.onap.aai.exceptions.AAIException;
-import org.onap.aai.introspection.Introspector;
-import org.onap.aai.introspection.Loader;
-import org.onap.aai.introspection.LoaderFactory;
-import org.onap.aai.introspection.ModelType;
-import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
-import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.logging.LogFormatTools;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.parsers.uri.URIToObject;
-import org.onap.aai.serialization.db.DBSerializer;
-import org.onap.aai.serialization.engines.InMemoryDBEngine;
-import org.onap.aai.serialization.engines.QueryStyle;
-import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
-import org.onap.aai.setup.SchemaVersion;
-import org.onap.aai.setup.SchemaVersions;
-import org.onap.aai.util.AAIConfig;
-import org.onap.aai.util.AAIConstants;
-import org.onap.aai.util.AAISystemExitUtil;
-import org.onap.aai.util.ExceptionTranslator;
-import org.slf4j.MDC;
-import org.springframework.context.annotation.AnnotationConfigApplicationContext;
-
-import java.io.*;
-import java.net.URI;
-import java.nio.file.Files;
-import java.nio.file.InvalidPathException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/*
- * The Class ListEndpoints.
- */
-public class DynamicPayloadGenerator {
-
- /*
- * Create a Dynamic memory graph instance which should not affect the
- * AAIGraph
- */
- private InMemoryGraph inMemGraph = null;
-
- private InMemoryDBEngine dbEngine;
- private InputStream sequenceInputStreams;
- /*
- * Loader, QueryStyle, ConnectionType for the Serializer
- */
- private Loader loader;
- private String urlBase;
- private BufferedWriter bw = null;
- private boolean exitFlag = true;
- private CommandLineArgs cArgs;
-
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DynamicPayloadGenerator.class);
-
- private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL;
- private static final DBConnectionType type = DBConnectionType.CACHED;
- private static final ModelType introspectorFactoryType = ModelType.MOXY;
- private final LoaderFactory loaderFactory;
- private final EdgeIngestor edgeRules;
- private final SchemaVersions schemaVersions;
- private final SchemaVersion version;
-
- public DynamicPayloadGenerator(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions){
- this.loaderFactory = loaderFactory;
- this.edgeRules = edgeIngestor;
- this.schemaVersions = schemaVersions;
- this.version = schemaVersions.getDefaultVersion();
- }
-
- /**
- * The run method.
- *
- * @param args
- * the arguments
- * @param exitFlag true if running from a shell script to call system exit, false if running from scheduled task
- * @throws AAIException
- * @throws Exception
- */
-
- public static void run (LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions, String[] args, boolean isSystemExit) {
- //
- MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName());
- DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator(loaderFactory, edgeIngestor, schemaVersions);
- payloadgen.exitFlag = isSystemExit;
- try {
- payloadgen.init(args);
-
- payloadgen.generatePayloads();
- } catch (AAIException e) {
- LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
- } catch (IOException e) {
- LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
- }
- if ( isSystemExit ) {
- AAISystemExitUtil.systemExitCloseAAIGraph(1);
- }
- else {
- AAISystemExitUtil.systemExitCloseAAIGraph(0);
- }
-
- }
- public static void main(String[] args) throws AAIException {
- AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
- PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
- initializer.initialize(ctx);
- try {
- ctx.scan(
- "org.onap.aai.config",
- "org.onap.aai.setup"
- );
- ctx.refresh();
- } catch (Exception e) {
- AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
- LOGGER.error("Problems running tool "+aai.getMessage());
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
- throw aai;
-
- }
- LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
- EdgeIngestor edgeIngestor = ctx.getBean(EdgeIngestor.class);
- SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions");
- run (loaderFactory, edgeIngestor, schemaVersions, args, true);
- }
-
-
- public void taskExit() {
- if ( this.exitFlag ) {
- AAISystemExitUtil.systemExitCloseAAIGraph(1);
- }
- else {
- AAISystemExitUtil.systemExitCloseAAIGraph(0);
- }
- }
- public void init(String[] args) throws AAIException {
- cArgs = new CommandLineArgs();
- JCommander jCommander = new JCommander(cArgs, args);
- jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName());
- LOGGER.info("Snapshot file " + cArgs.dataSnapshot);
-
-
- // TODO- How to add dynamic.properties
-
- LOGGER.info("output file " + cArgs.output);
- LOGGER.info("format file " + cArgs.format);
- LOGGER.info("schema enabled " + cArgs.schemaEnabled);
- LOGGER.info("Multiple snapshots " + cArgs.isMultipleSnapshot);
- LOGGER.info("Is Partial Graph " + cArgs.isPartialGraph);
-
- if (cArgs.config.isEmpty())
- cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties";
-
- LOGGER.info("config file " + cArgs.config);
- if (cArgs.nodePropertyFile.isEmpty())
- cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json";
- LOGGER.info("nodePropertyFile file " + cArgs.nodePropertyFile);
-
- if (cArgs.inputFilterPropertyFile.isEmpty())
- cArgs.inputFilterPropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/inputFilters.json";
- LOGGER.info("inputFilterPropertyFile file " + cArgs.inputFilterPropertyFile);
-
- if (cArgs.isPartialGraph)
- cArgs.dataSnapshot = cArgs.dataSnapshot+".partial";
-
- if (!cArgs.isMultipleSnapshot) {
- validateFile(cArgs.dataSnapshot);
- } else {
- // for multiple snapshots dataSnapshot + ".P" is the prefix of the
- // files
- sequenceInputStreams = validateMultipleSnapshots(cArgs.dataSnapshot);
- }
-
- LOGGER.info("Datasnapshot file " + cArgs.dataSnapshot);
- AAIConfig.init();
-
- urlBase = AAIConfig.get("aai.server.url.base", "");
-
- }
-
- public void generatePayloads() throws AAIException, IOException {
-
- List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile);
- /*
- * Read the inputFilters which will include for each node-type the regex that needs to be
- * applied and the filtered-node-type
- * For eg: complex --> apply regex on cloud-region and then traverse to complex
- * complex --> filtered-node-type: cloud-region, filters: include regex on cloud-region
- */
- /*
- * Example:
- * { "cloud-region" :
- * {"filtered-node-type":"cloud-region",
- * "filters": [ { "property": "cloud-owner", "regex": "att-aic" },
- * { "property": "cloud-region-id", "regex": "M*" },
- * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" }
- * ] },
- * "complex" : {
- * "filtered-node-type":"cloud-region",
- * "filters": [ { "property": "cloud-owner", "regex": "att-aic" },
- * { "property": "cloud-region-id", "regex": "M*" },
- * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" }
- * ] },
- *
- * } }
- */
- Map<String, Map<String, String>> inputFilters = readInputFilterPropertyFile(cArgs.inputFilterPropertyFile);
- Map<String, String> filteredNodeTypes = findFilteredNodeTypes(cArgs.inputFilterPropertyFile);
- // Read the input filter criteria
- LOGGER.info("Load the Graph");
-
- this.loadGraph();
- LOGGER.info("Generate payload");
- this.generatePayload(nodeFilters, inputFilters, filteredNodeTypes);
- LOGGER.info("Close graph");
- this.closeGraph();
-
- }
-
- private List<Map<String, List<String>>> readFile(String inputFile) throws IOException {
-
- // validate that we can read the inputFile
- validateFile(inputFile);
-
- InputStream is = new FileInputStream(inputFile);
- Scanner scanner = new Scanner(is);
- String jsonFile = scanner.useDelimiter("\\Z").next();
- scanner.close();
-
- List<Map<String, List<String>>> allNodes = new ArrayList<>();
- Map<String, List<String>> filterCousins = new HashMap<>();
- Map<String, List<String>> filterParents = new HashMap<>();
-
- ObjectMapper mapper = new ObjectMapper();
-
- JsonNode rootNode = mapper.readTree(jsonFile);
-
- Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
-
- while (nodeFields.hasNext()) {
- Entry<String, JsonNode> entry = nodeFields.next();
- String nodeType = entry.getKey();
- JsonNode nodeProperty = entry.getValue();
-
- JsonNode cousinFilter = nodeProperty.path("cousins");
- JsonNode parentFilter = nodeProperty.path("parents");
- List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(),
- new TypeReference<ArrayList<String>>() {
- });
-
- List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(),
- new TypeReference<ArrayList<String>>() {
- });
- for (String cousin : cousins) {
- LOGGER.info("Cousins-Filtered " + cousin);
- }
- for (String parent : parents) {
- LOGGER.info("Parents-Filtered " + parent);
- }
- filterCousins.put(nodeType, cousins);
- filterParents.put(nodeType, parents);
-
- }
-
- allNodes.add(filterCousins);
- allNodes.add(filterParents);
- return allNodes;
-
- }
-
- /* Example:
-{
- "cloud-region" : {
- "filtered-node-type" :"cloud-region",
- "filters": [
- {
- "property": "cloud-owner",
- "regex": "att-aic"
- },
- {
- "property": "cloud-region-id",
- "regex": "M*"
- },
- {
- "property": "cloud-region-version",
- "regex": "aic2.5|aic3.0"
- }
- ]
- },
- "complex" : {
- "filters":[
- ]
-
- }
-}
-*/
- private Map<String, Map<String, String>> readInputFilterPropertyFile(String inputFile) throws IOException {
-
- validateFile(inputFile);
-
- InputStream is = new FileInputStream(inputFile);
- Scanner scanner = new Scanner(is);
- String jsonFile = scanner.useDelimiter("\\Z").next();
- scanner.close();
-
- Map<String, Map<String, String>> propToRegex = new HashMap<String, Map<String, String>>();
-
- ObjectMapper mapper = new ObjectMapper();
-
- JsonNode rootNode = mapper.readTree(jsonFile);
-
- Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
-
- while (nodeFields.hasNext()) {
- Entry<String, JsonNode> entry = nodeFields.next();
- String nodeType = entry.getKey();
- JsonNode nodeProperty = entry.getValue();
-
- JsonNode filter = nodeProperty.path("filters");
- List<JsonNode> filterMap = new ObjectMapper().readValue(filter.traverse(),
- new TypeReference<ArrayList<JsonNode>>() {
- });
- HashMap<String, String> filterMaps = new HashMap<String, String>();
- for (JsonNode n : filterMap) {
- filterMaps.put(n.get("property").asText(), n.get("regex").asText());
- }
-
- propToRegex.put(nodeType, filterMaps);
- }
- return (propToRegex);
- }
-
- private Map<String, String> findFilteredNodeTypes(String inputFile) throws IOException {
-
- validateFile(inputFile);
-
- InputStream is = new FileInputStream(inputFile);
- Scanner scanner = new Scanner(is);
- String jsonFile = scanner.useDelimiter("\\Z").next();
- scanner.close();
-
- Map<String, String> filteredNodeTypes = new HashMap<String, String>();
-
- ObjectMapper mapper = new ObjectMapper();
-
- JsonNode rootNode = mapper.readTree(jsonFile);
-
- Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
-
- while (nodeFields.hasNext()) {
- Entry<String, JsonNode> entry = nodeFields.next();
- String nodeType = entry.getKey();
- JsonNode nodeProperty = entry.getValue();
-
- JsonNode filter = nodeProperty.path("filtered-node-type");
-
- filteredNodeTypes.put(nodeType, filter.asText());
- }
- return (filteredNodeTypes);
- }
-
- public void loadGraph() throws IOException {
-
- loadGraphIntoMemory();
- buildDbEngine();
-
- }
-
- private void loadGraphIntoMemory() throws IOException {
- if (!(cArgs.isMultipleSnapshot)) {
- inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled,
- cArgs.isPartialGraph);
- } else {
- inMemGraph = new InMemoryGraph.Builder().build(sequenceInputStreams, cArgs.config, cArgs.schemaEnabled,
- cArgs.isPartialGraph);
- }
- }
-
- private void buildDbEngine() {
- // TODO : parametrise version
- loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
-
- dbEngine = new InMemoryDBEngine(queryStyle, type, loader, inMemGraph.getGraph());
- dbEngine.startTransaction();
- }
-
- private void generatePayload(List<Map<String, List<String>>> nodeFilters,
- Map<String, Map<String, String>> inputFilters, Map<String, String> filteredNodeTypes)
- throws AAIException, IOException {
-
- Map<String, List<String>> filterCousinsMap = nodeFilters.get(0);
- Map<String, List<String>> filterParentsMap = nodeFilters.get(1);
- Set<String> nodeTypes = filterCousinsMap.keySet();
-
- for (String nodeType : nodeTypes) {
- if ("DMAAP-MR".equals(cArgs.format)) {
- bw = createFile(nodeType + ".json");
- }
- List<String> filterCousins = filterCousinsMap.get(nodeType);
- List<String> filterParents = filterParentsMap.get(nodeType);
- Map<String, String> nodeInputFilterMap = inputFilters.get(nodeType);
- String filteredNodeType = nodeType;
- if(filteredNodeTypes.get(nodeType) != null && !filteredNodeTypes.get(nodeType).isEmpty())
- filteredNodeType = filteredNodeTypes.get(nodeType);
- readVertices(nodeType, filterCousins, filterParents, nodeInputFilterMap, filteredNodeType);
- if(bw != null)
- bw.close();
- LOGGER.info("All Done-" + nodeType);
- }
-
- }
-
- private BufferedWriter createFile(String outfileName) throws IOException {
- // FileLocation
- String fileName = outfileName;
- File outFile = new File(fileName);
- FileWriter fw = null;
- LOGGER.info(" Will write to " + fileName);
- try {
- fw = new FileWriter(outFile.getAbsoluteFile());
- } catch (IOException i) {
- String emsg = "Unable to write to " + fileName + " Exception = " + i.getMessage();
- LOGGER.error(emsg);
- System.out.println(emsg);
- throw i;
- }
- return new BufferedWriter(fw);
- }
-
- private void createDirectory(String dirName) throws IOException {
- // FileLocation
- Path pathDir = null;
- try {
- pathDir = Paths.get(dirName);
- } catch (InvalidPathException i) {
- String emsg = "Directory " + dirName + " could not be found.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- try {
- Files.createDirectories(pathDir);
- } catch (Exception e) {
- String emsg = "Directory " + dirName + " could not be created. Exception = " + e.getMessage();
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- }
-
- public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents,
- Map<String, String> nodeInputFilters, String filteredNodeType) throws AAIException, IOException {
-
- DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
-
- /*
- * Start with nodeType you need to filter and then traverse to the actual nodeType
- */
- GraphTraversal<Vertex, Vertex> gtraversal = inMemGraph.getGraph().traversal().V().has("aai-node-type",
- filteredNodeType);
-
-
- // input regex
- if (nodeInputFilters != null && (!nodeInputFilters.isEmpty())) {
- for (Map.Entry<String, String> entry : nodeInputFilters.entrySet()) {
- String property = entry.getKey();
- String regex = entry.getValue();
- Pattern pa = Pattern.compile(regex);
-
- gtraversal = gtraversal.has(property, P.test((t, p) -> {
- Matcher m = ((Pattern) p).matcher((CharSequence) t);
- boolean b = m.matches();
- return b;
- }, pa));
- }
- }
-
- /*
- * Tenant, AZ, Complex, Zone, pserver come here
- */
- if (!filteredNodeType.equals(nodeType)) {
-
- EdgeRuleQuery treeEdgeRuleQuery = new EdgeRuleQuery
- .Builder(filteredNodeType, nodeType)
- .edgeType(EdgeType.TREE)
- .build();
-
- EdgeRuleQuery cousinEdgeQuery = new EdgeRuleQuery
- .Builder(filteredNodeType, nodeType)
- .edgeType(EdgeType.COUSIN)
- .build();
-
- EdgeRule rule = null;
- boolean hasTreeEdgeRule = true;
-
- try {
- rule = edgeRules.getRule(treeEdgeRuleQuery);
- } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
- hasTreeEdgeRule = false;
- }
-
- if(!hasTreeEdgeRule) {
- try {
- rule = edgeRules.getRule(cousinEdgeQuery);
- } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
- LOGGER.error("Unable to get a tree or cousin edge between {} and {}", filteredNodeType, nodeType);
- return;
- }
- }
-
- if (rule.getDirection().toString().equals(AAIDirection.OUT.toString())) {
- gtraversal.out(rule.getLabel()).has("aai-node-type", nodeType);
- } else {
- gtraversal.in(rule.getLabel()).has("aai-node-type", nodeType);
- }
-
- }
-
- String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP;
- createDirectory(dirName);
- // TODO: Formatter
-
- if ("DMAAP-MR".equals(cArgs.format)) {
- while (gtraversal.hasNext()) {
- if (bw != null)
- bw = createFile(nodeType + ".json");
- Vertex node = gtraversal.next();
- Introspector nodeObj = serializer.getLatestVersionView(node);
- createPayloadForDmaap(node, nodeObj);
- }
- } else {
- if ("PAYLOAD".equals(cArgs.format)) {
- int counter = 0;
- while (gtraversal.hasNext()) {
- Vertex node = gtraversal.next();
- try {
- counter++;
- String filename = dirName + counter + "-" + nodeType + ".json";
- bw = createFile(filename);
- Introspector obj = loader.introspectorFromName(nodeType);
- Set<Vertex> seen = new HashSet<>();
- int depth = AAIProperties.MAXIMUM_DEPTH;
- boolean nodeOnly = false;
-
- Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly);
- TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree);
- serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins,
- filterParents);
- createPayloadForPut(obj);
- if(bw != null)
- bw.close();
-
- URI uri = serializer.getURIForVertex(node);
- String filenameWithUri = dirName + counter + "-" + nodeType + ".txt";
- bw = createFile(filenameWithUri);
- bw.write(uri.toString());
- bw.newLine();
- bw.close();
- } catch (Exception e) {
- String emsg = "Caught exception while processing [" + counter + "-" + nodeType + "] continuing";
- System.out.println(emsg);
- LOGGER.error(emsg);
-
- }
- }
- }
- }
-
- }
-
- public void createPayloadForPut(Introspector nodeObj) throws IOException {
-
- String entityJson = nodeObj.marshal(false);
- ObjectMapper mapper = new ObjectMapper();
-
- ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson);
- rootNode.remove("resource-version");
-
- bw.newLine();
- bw.write(rootNode.toString());
- bw.newLine();
- }
-
- public void createPayloadForDmaap(Vertex node, Introspector nodeObj)
- throws AAIException, UnsupportedEncodingException {
-
- DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
-
- URI uri = serializer.getURIForVertex(node);
-
- String sourceOfTruth = "";
- HashMap<String, Introspector> relatedVertices = new HashMap<>();
- List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node);
-
- for (Vertex vertex : vertexChain) {
- try {
-
- Introspector vertexObj = serializer.getVertexProperties(vertex);
-
- relatedVertices.put(vertexObj.getObjectId(), vertexObj);
- } catch (AAIUnknownObjectException e) {
- LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
- }
-
- }
-
- String transactionId = "TXID";
- createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices);
-
- }
-
- public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj,
- Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException {
-
- String action = "CREATE";
- final Introspector notificationEvent = loader.introspectorFromName("notification-event");
-
- try {
- Introspector eventHeader = loader.introspectorFromName("notification-event-header");
- URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects);
-
- String entityLink = urlBase + version + uri;
-
- notificationEvent.setValue("cambria-partition", "AAI");
-
- eventHeader.setValue("entity-link", entityLink);
- eventHeader.setValue("action", action);
- eventHeader.setValue("entity-type", obj.getDbName());
- eventHeader.setValue("top-entity-type", parser.getTopEntityName());
- eventHeader.setValue("source-name", sourceOfTruth);
- eventHeader.setValue("version", version.toString());
- eventHeader.setValue("id", transactionId);
- eventHeader.setValue("event-type", "AAI-BASELINE");
- if (eventHeader.getValue("domain") == null) {
- eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK"));
- }
-
- if (eventHeader.getValue("sequence-number") == null) {
- eventHeader.setValue("sequence-number",
- AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK"));
- }
-
- if (eventHeader.getValue("severity") == null) {
- eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK"));
- }
-
- if (eventHeader.getValue("id") == null) {
- eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString());
-
- }
-
- if (eventHeader.getValue("timestamp") == null) {
- eventHeader.setValue("timestamp", genDate());
- }
-
- List<Object> parentList = parser.getParentList();
- parentList.clear();
-
- if (!parser.getTopEntity().equals(parser.getEntity())) {
- Introspector child;
- String json = obj.marshal(false);
- child = parser.getLoader().unmarshal(parser.getEntity().getName(), json);
- parentList.add(child.getUnderlyingObject());
- }
-
- final Introspector eventObject;
-
- String json = "";
- if (parser.getTopEntity().equals(parser.getEntity())) {
- json = obj.marshal(false);
- eventObject = loader.unmarshal(obj.getName(), json);
- } else {
- json = parser.getTopEntity().marshal(false);
-
- eventObject = loader.unmarshal(parser.getTopEntity().getName(), json);
- }
- notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject());
- notificationEvent.setValue("entity", eventObject.getUnderlyingObject());
-
- String entityJson = notificationEvent.marshal(false);
-
- bw.newLine();
- bw.write(entityJson);
-
- } catch (AAIUnknownObjectException e) {
- LOGGER.error("Fatal error - notification-event-header object not found!");
- } catch (Exception e) {
- LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e));
- }
- }
-
- private void closeGraph() {
- inMemGraph.getGraph().tx().rollback();
- inMemGraph.getGraph().close();
- }
-
- public static String genDate() {
- Date date = new Date();
- DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS");
- return formatter.format(date);
- }
-
- public static String genDate2() {
- Date date = new Date();
- DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
- return formatter.format(date);
- }
-
- private void validateFile(String filename) {
- File f = new File(filename);
- if (!f.exists()) {
- String emsg = "File " + filename + " could not be found.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- } else if (!f.canRead()) {
- String emsg = "File " + filename + " could not be read.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- } else if (f.length() == 0) {
- String emsg = "File " + filename + " had no data.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- }
-
- private InputStream validateMultipleSnapshots(String filenamePrefix) {
- if (filenamePrefix == null || filenamePrefix.length() == 0) {
- String emsg = "No snapshot path was provided.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- String targetDir = ".";
- int lastSeparator = filenamePrefix.lastIndexOf(File.separator);
-
- LOGGER.info("File separator=[" + File.separator + "] lastSeparator=" + lastSeparator + " filenamePrefix="
- + filenamePrefix);
- if (lastSeparator >= 0) {
- targetDir = filenamePrefix.substring(0, lastSeparator);
- LOGGER.info("targetDir=" + targetDir);
- }
- if (targetDir.length() == 0) {
- String emsg = "No snapshot directory was found in path:" + filenamePrefix;
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- String prefix = filenamePrefix.substring(lastSeparator + 1);
- if (prefix == null || prefix.length() == 0) {
- String emsg = "No snapshot file prefix was provided.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- long timeA = System.nanoTime();
-
- ArrayList<File> snapFilesArr = new ArrayList<File>();
- String thisSnapPrefix = prefix + ".P";
- File fDir = new File(targetDir); // Snapshot directory
- File[] allFilesArr = fDir.listFiles();
- for (File snapFile : allFilesArr) {
- String snapFName = snapFile.getName();
- if (snapFName.startsWith(thisSnapPrefix)) {
- snapFilesArr.add(snapFile);
- }
- }
-
- if (snapFilesArr.isEmpty()) {
- String fullFName = targetDir + AAIConstants.AAI_FILESEP + thisSnapPrefix;
- String emsg = "Snapshot files " + fullFName + "* could not be found.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
-
- int fCount = snapFilesArr.size();
- Iterator<File> fItr = snapFilesArr.iterator();
- Vector<InputStream> inputStreamsV = new Vector<>();
- for (int i = 0; i < fCount; i++) {
- File f = snapFilesArr.get(i);
- String fname = f.getName();
- if (!f.canRead()) {
- String emsg = "Snapshot file " + fname + " could not be read.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- } else if (f.length() == 0) {
- String emsg = "Snapshot file " + fname + " had no data.";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- String fullFName = targetDir + AAIConstants.AAI_FILESEP + fname;
- InputStream fis = null;
- try {
- fis = new FileInputStream(fullFName);
- } catch (FileNotFoundException e) {
- // should not happen at this point
- String emsg = "Snapshot file " + fullFName + " could not be found";
- LOGGER.error(emsg);
- System.out.println(emsg);
- taskExit();
- }
- inputStreamsV.add(fis);
- }
- // Now add inputStreams.elements() to the Vector,
- InputStream sis = new SequenceInputStream(inputStreamsV.elements());
- return (sis);
- }
-
- public InMemoryGraph getInMemGraph() {
- return inMemGraph;
- }
-
- public void setInMemGraph(InMemoryGraph inMemGraph) {
- this.inMemGraph = inMemGraph;
- }
-}
-
-class CommandLineArgs {
-
- @Parameter(names = "--help", help = true)
- public boolean help;
-
- @Parameter(names = "-d", description = "snapshot file to be loaded", required = true)
- public String dataSnapshot;
-
- @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1)
- public boolean schemaEnabled = true;
-
- @Parameter(names = "-c", description = "location of configuration file")
- public String config = "";
-
- @Parameter(names = "-o", description = "output location")
- public String output = "";
-
- @Parameter(names = "-f", description = "format of output")
- public String format = "PAYLOAD";
-
- @Parameter(names = "-n", description = "Node input file")
- public String nodePropertyFile = "";
-
- @Parameter(names = "-m", description = "multipe snapshots or not", arity = 1)
- public boolean isMultipleSnapshot = false;
-
- @Parameter(names = "-i", description = "input filter configuration file")
- public String inputFilterPropertyFile = "";
-
- @Parameter(names = "-p", description = "Use the partial graph", arity = 1)
- public boolean isPartialGraph = true;
-
-}
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
+import org.apache.tinkerpop.gremlin.structure.Element;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.node.ObjectNode;
+import org.codehaus.jackson.type.TypeReference;
+import org.onap.aai.config.PropertyPasswordConfiguration;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.dbmap.InMemoryGraph;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.EdgeRule;
+import org.onap.aai.edges.EdgeRuleQuery;
+import org.onap.aai.edges.enums.AAIDirection;
+import org.onap.aai.edges.enums.EdgeType;
+import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException;
+import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LogFormatTools;
+import org.onap.aai.parsers.uri.URIToObject;
+import org.onap.aai.serialization.db.DBSerializer;
+import org.onap.aai.serialization.engines.InMemoryDBEngine;
+import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.AAISystemExitUtil;
+import org.onap.aai.util.ExceptionTranslator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import java.io.*;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/*
+ * The Class ListEndpoints.
+ */
+public class DynamicPayloadGenerator {
+
+ /*
+ * Create a Dynamic memory graph instance which should not affect the
+ * AAIGraph
+ */
+ private InMemoryGraph inMemGraph = null;
+
+ private InMemoryDBEngine dbEngine;
+ private InputStream sequenceInputStreams;
+ /*
+ * Loader, QueryStyle, ConnectionType for the Serializer
+ */
+ private Loader loader;
+ private String urlBase;
+ private BufferedWriter bw = null;
+ private boolean exitFlag = true;
+ private CommandLineArgs cArgs;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(DynamicPayloadGenerator.class);
+
+ private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL;
+ private static final ModelType introspectorFactoryType = ModelType.MOXY;
+ private final LoaderFactory loaderFactory;
+ private final EdgeIngestor edgeRules;
+ private final SchemaVersions schemaVersions;
+ private final SchemaVersion version;
+
+ public DynamicPayloadGenerator(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions){
+ this.loaderFactory = loaderFactory;
+ this.edgeRules = edgeIngestor;
+ this.schemaVersions = schemaVersions;
+ this.version = schemaVersions.getDefaultVersion();
+ }
+
+ /**
+ * The run method.
+ *
+ * @param args
+ * the arguments
+ * @param exitFlag true if running from a shell script to call system exit, false if running from scheduled task
+ * @throws AAIException
+ * @throws Exception
+ */
+
+ public static void run (LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, SchemaVersions schemaVersions, String[] args, boolean isSystemExit) {
+ //
+ MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName());
+ DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator(loaderFactory, edgeIngestor, schemaVersions);
+ payloadgen.exitFlag = isSystemExit;
+ try {
+ payloadgen.init(args);
+
+ payloadgen.generatePayloads();
+ } catch (AAIException | IOException e) {
+ LOGGER.error("Exception {}", LogFormatTools.getStackTop(e));
+ }
+ if ( isSystemExit ) {
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ else {
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+
+ }
+ public static void main(String[] args) throws AAIException {
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
+ PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
+ initializer.initialize(ctx);
+ try {
+ ctx.scan(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+ ctx.refresh();
+ } catch (Exception e) {
+ AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
+ LOGGER.error("Problems running tool {}", aai.getMessage());
+ ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
+ throw aai;
+
+ }
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ EdgeIngestor edgeIngestor = ctx.getBean(EdgeIngestor.class);
+ SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions");
+ run (loaderFactory, edgeIngestor, schemaVersions, args, true);
+ }
+
+
+ public void taskExit() {
+ if ( this.exitFlag ) {
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ else {
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ }
+ public void init(String[] args) throws AAIException {
+ cArgs = new CommandLineArgs();
+ JCommander jCommander = new JCommander(cArgs, args);
+ jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName());
+ LOGGER.debug("Snapshot file " + cArgs.dataSnapshot);
+
+
+ // TODO- How to add dynamic.properties
+
+ LOGGER.debug("output file " + cArgs.output);
+ LOGGER.debug("format file " + cArgs.format);
+ LOGGER.debug("schema enabled " + cArgs.schemaEnabled);
+ LOGGER.debug("Multiple snapshots " + cArgs.isMultipleSnapshot);
+ LOGGER.debug("Is Partial Graph " + cArgs.isPartialGraph);
+
+ if (cArgs.config.isEmpty())
+ cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties";
+
+ LOGGER.debug("config file " + cArgs.config);
+ if (cArgs.nodePropertyFile.isEmpty())
+ cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json";
+ LOGGER.debug("nodePropertyFile file " + cArgs.nodePropertyFile);
+
+ if (cArgs.inputFilterPropertyFile.isEmpty())
+ cArgs.inputFilterPropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/inputFilters.json";
+ LOGGER.debug("inputFilterPropertyFile file " + cArgs.inputFilterPropertyFile);
+
+ if (cArgs.isPartialGraph)
+ cArgs.dataSnapshot = cArgs.dataSnapshot+".partial";
+
+ if (!cArgs.isMultipleSnapshot) {
+ validateFile(cArgs.dataSnapshot);
+ } else {
+ // for multiple snapshots dataSnapshot + ".P" is the prefix of the
+ // files
+ sequenceInputStreams = validateMultipleSnapshots(cArgs.dataSnapshot);
+ }
+
+ LOGGER.debug("Datasnapshot file " + cArgs.dataSnapshot);
+ AAIConfig.init();
+
+ urlBase = AAIConfig.get("aai.server.url.base", "");
+
+ }
+
+ public void generatePayloads() throws AAIException, IOException {
+
+ List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile);
+ /*
+ * Read the inputFilters which will include for each node-type the regex that needs to be
+ * applied and the filtered-node-type
+ * For eg: complex --> apply regex on cloud-region and then traverse to complex
+ * complex --> filtered-node-type: cloud-region, filters: include regex on cloud-region
+ */
+ /*
+ * Example:
+ * { "cloud-region" :
+ * {"filtered-node-type":"cloud-region",
+ * "filters": [ { "property": "cloud-owner", "regex": "att-aic" },
+ * { "property": "cloud-region-id", "regex": "M*" },
+ * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" }
+ * ] },
+ * "complex" : {
+ * "filtered-node-type":"cloud-region",
+ * "filters": [ { "property": "cloud-owner", "regex": "att-aic" },
+ * { "property": "cloud-region-id", "regex": "M*" },
+ * { "property":"cloud-region-version", "regex": "aic2.5|aic3.0" }
+ * ] },
+ *
+ * } }
+ */
+ Map<String, Map<String, String>> inputFilters = readInputFilterPropertyFile(cArgs.inputFilterPropertyFile);
+ Map<String, String> filteredNodeTypes = findFilteredNodeTypes(cArgs.inputFilterPropertyFile);
+ // Read the input filter criteria
+ LOGGER.debug("Load the Graph");
+
+ this.loadGraph();
+ LOGGER.debug("Generate payload");
+ this.generatePayload(nodeFilters, inputFilters, filteredNodeTypes);
+ LOGGER.debug("Close graph");
+ this.closeGraph();
+
+ }
+
+ private List<Map<String, List<String>>> readFile(String inputFile) throws IOException {
+
+ // validate that we can read the inputFile
+ validateFile(inputFile);
+
+ InputStream is = new FileInputStream(inputFile);
+ Scanner scanner = new Scanner(is);
+ String jsonFile = scanner.useDelimiter("\\Z").next();
+ scanner.close();
+
+ List<Map<String, List<String>>> allNodes = new ArrayList<>();
+ Map<String, List<String>> filterCousins = new HashMap<>();
+ Map<String, List<String>> filterParents = new HashMap<>();
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ JsonNode rootNode = mapper.readTree(jsonFile);
+
+ Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
+
+ while (nodeFields.hasNext()) {
+ Entry<String, JsonNode> entry = nodeFields.next();
+ String nodeType = entry.getKey();
+ JsonNode nodeProperty = entry.getValue();
+
+ JsonNode cousinFilter = nodeProperty.path("cousins");
+ JsonNode parentFilter = nodeProperty.path("parents");
+ List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(),
+ new TypeReference<ArrayList<String>>() {
+ });
+
+ List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(),
+ new TypeReference<ArrayList<String>>() {
+ });
+ for (String cousin : cousins) {
+ LOGGER.debug("Cousins-Filtered " + cousin);
+ }
+ for (String parent : parents) {
+ LOGGER.debug("Parents-Filtered " + parent);
+ }
+ filterCousins.put(nodeType, cousins);
+ filterParents.put(nodeType, parents);
+
+ }
+
+ allNodes.add(filterCousins);
+ allNodes.add(filterParents);
+ return allNodes;
+
+ }
+
+ /* Example:
+{
+ "cloud-region" : {
+ "filtered-node-type" :"cloud-region",
+ "filters": [
+ {
+ "property": "cloud-owner",
+ "regex": "att-aic"
+ },
+ {
+ "property": "cloud-region-id",
+ "regex": "M*"
+ },
+ {
+ "property": "cloud-region-version",
+ "regex": "aic2.5|aic3.0"
+ }
+ ]
+ },
+ "complex" : {
+ "filters":[
+ ]
+
+ }
+}
+*/
+ private Map<String, Map<String, String>> readInputFilterPropertyFile(String inputFile) throws IOException {
+
+ validateFile(inputFile);
+
+ InputStream is = new FileInputStream(inputFile);
+ Scanner scanner = new Scanner(is);
+ String jsonFile = scanner.useDelimiter("\\Z").next();
+ scanner.close();
+
+ Map<String, Map<String, String>> propToRegex = new HashMap<String, Map<String, String>>();
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ JsonNode rootNode = mapper.readTree(jsonFile);
+
+ Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
+
+ while (nodeFields.hasNext()) {
+ Entry<String, JsonNode> entry = nodeFields.next();
+ String nodeType = entry.getKey();
+ JsonNode nodeProperty = entry.getValue();
+
+ JsonNode filter = nodeProperty.path("filters");
+ List<JsonNode> filterMap = new ObjectMapper().readValue(filter.traverse(),
+ new TypeReference<ArrayList<JsonNode>>() {
+ });
+ HashMap<String, String> filterMaps = new HashMap<String, String>();
+ for (JsonNode n : filterMap) {
+ filterMaps.put(n.get("property").asText(), n.get("regex").asText());
+ }
+
+ propToRegex.put(nodeType, filterMaps);
+ }
+ return (propToRegex);
+ }
+
+ private Map<String, String> findFilteredNodeTypes(String inputFile) throws IOException {
+
+ validateFile(inputFile);
+
+ InputStream is = new FileInputStream(inputFile);
+ Scanner scanner = new Scanner(is);
+ String jsonFile = scanner.useDelimiter("\\Z").next();
+ scanner.close();
+
+ Map<String, String> filteredNodeTypes = new HashMap<String, String>();
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ JsonNode rootNode = mapper.readTree(jsonFile);
+
+ Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
+
+ while (nodeFields.hasNext()) {
+ Entry<String, JsonNode> entry = nodeFields.next();
+ String nodeType = entry.getKey();
+ JsonNode nodeProperty = entry.getValue();
+
+ JsonNode filter = nodeProperty.path("filtered-node-type");
+
+ filteredNodeTypes.put(nodeType, filter.asText());
+ }
+ return (filteredNodeTypes);
+ }
+
+ public void loadGraph() throws IOException {
+
+ loadGraphIntoMemory();
+ buildDbEngine();
+
+ }
+
+ private void loadGraphIntoMemory() throws IOException {
+ if (!(cArgs.isMultipleSnapshot)) {
+ inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled,
+ cArgs.isPartialGraph);
+ } else {
+ inMemGraph = new InMemoryGraph.Builder().build(sequenceInputStreams, cArgs.config, cArgs.schemaEnabled,
+ cArgs.isPartialGraph);
+ }
+ }
+
+ private void buildDbEngine() {
+ // TODO : parametrise version
+ loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+
+ dbEngine = new InMemoryDBEngine(queryStyle, loader, inMemGraph.getGraph());
+ dbEngine.startTransaction();
+ }
+
+ private void generatePayload(List<Map<String, List<String>>> nodeFilters,
+ Map<String, Map<String, String>> inputFilters, Map<String, String> filteredNodeTypes)
+ throws AAIException, IOException {
+
+ Map<String, List<String>> filterCousinsMap = nodeFilters.get(0);
+ Map<String, List<String>> filterParentsMap = nodeFilters.get(1);
+ Set<String> nodeTypes = filterCousinsMap.keySet();
+
+ for (String nodeType : nodeTypes) {
+ if ("DMAAP-MR".equals(cArgs.format)) {
+ bw = createFile(nodeType + ".json");
+ }
+ List<String> filterCousins = filterCousinsMap.get(nodeType);
+ List<String> filterParents = filterParentsMap.get(nodeType);
+ Map<String, String> nodeInputFilterMap = inputFilters.get(nodeType);
+ String filteredNodeType = nodeType;
+ if(filteredNodeTypes.get(nodeType) != null && !filteredNodeTypes.get(nodeType).isEmpty())
+ filteredNodeType = filteredNodeTypes.get(nodeType);
+ readVertices(nodeType, filterCousins, filterParents, nodeInputFilterMap, filteredNodeType);
+ if(bw != null)
+ bw.close();
+ LOGGER.debug("All Done-" + nodeType);
+ }
+
+ }
+
+ private BufferedWriter createFile(String outfileName) throws IOException {
+ // FileLocation
+ String fileName = outfileName;
+ File outFile = new File(fileName);
+ FileWriter fw = null;
+ LOGGER.debug(" Will write to " + fileName);
+ try {
+ fw = new FileWriter(outFile.getAbsoluteFile());
+ } catch (IOException i) {
+ String emsg = "Unable to write to " + fileName + " Exception = " + i.getMessage();
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ throw i;
+ }
+ return new BufferedWriter(fw);
+ }
+
+ private void createDirectory(String dirName) throws IOException {
+ // FileLocation
+ Path pathDir = null;
+ try {
+ pathDir = Paths.get(dirName);
+ } catch (InvalidPathException i) {
+ String emsg = "Directory " + dirName + " could not be found.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ try {
+ Files.createDirectories(pathDir);
+ } catch (Exception e) {
+ String emsg = "Directory " + dirName + " could not be created. Exception = " + e.getMessage();
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ }
+
+ public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents,
+ Map<String, String> nodeInputFilters, String filteredNodeType) throws AAIException, IOException {
+
+ DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
+
+ /*
+ * Start with nodeType you need to filter and then traverse to the actual nodeType
+ */
+ GraphTraversal<Vertex, Vertex> gtraversal = inMemGraph.getGraph().traversal().V().has("aai-node-type",
+ filteredNodeType);
+
+
+ // input regex
+ if (nodeInputFilters != null && (!nodeInputFilters.isEmpty())) {
+ for (Map.Entry<String, String> entry : nodeInputFilters.entrySet()) {
+ String property = entry.getKey();
+ String regex = entry.getValue();
+ Pattern pa = Pattern.compile(regex);
+
+ gtraversal = gtraversal.has(property, P.test((t, p) -> {
+ Matcher m = ((Pattern) p).matcher((CharSequence) t);
+ boolean b = m.matches();
+ return b;
+ }, pa));
+ }
+ }
+
+ /*
+ * Tenant, AZ, Complex, Zone, pserver come here
+ */
+ if (!filteredNodeType.equals(nodeType)) {
+
+ EdgeRuleQuery treeEdgeRuleQuery = new EdgeRuleQuery
+ .Builder(filteredNodeType, nodeType)
+ .edgeType(EdgeType.TREE)
+ .build();
+
+ EdgeRuleQuery cousinEdgeQuery = new EdgeRuleQuery
+ .Builder(filteredNodeType, nodeType)
+ .edgeType(EdgeType.COUSIN)
+ .build();
+
+ EdgeRule rule = null;
+ boolean hasTreeEdgeRule = true;
+
+ try {
+ rule = edgeRules.getRule(treeEdgeRuleQuery);
+ } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
+ hasTreeEdgeRule = false;
+ }
+
+ if(!hasTreeEdgeRule) {
+ try {
+ rule = edgeRules.getRule(cousinEdgeQuery);
+ } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
+ LOGGER.error("Unable to get a tree or cousin edge between {} and {}", filteredNodeType, nodeType);
+ return;
+ }
+ }
+
+ if (rule.getDirection().toString().equals(AAIDirection.OUT.toString())) {
+ gtraversal.out(rule.getLabel()).has("aai-node-type", nodeType);
+ } else {
+ gtraversal.in(rule.getLabel()).has("aai-node-type", nodeType);
+ }
+
+ }
+
+ String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP;
+ createDirectory(dirName);
+ // TODO: Formatter
+
+ if ("DMAAP-MR".equals(cArgs.format)) {
+ while (gtraversal.hasNext()) {
+ if (bw != null)
+ bw = createFile(nodeType + ".json");
+ Vertex node = gtraversal.next();
+ Introspector nodeObj = serializer.getLatestVersionView(node);
+ createPayloadForDmaap(node, nodeObj);
+ }
+ } else {
+ if ("PAYLOAD".equals(cArgs.format)) {
+ int counter = 0;
+ while (gtraversal.hasNext()) {
+ Vertex node = gtraversal.next();
+ try {
+ counter++;
+ String filename = dirName + counter + "-" + nodeType + ".json";
+ bw = createFile(filename);
+ Introspector obj = loader.introspectorFromName(nodeType);
+ Set<Vertex> seen = new HashSet<>();
+ int depth = AAIProperties.MAXIMUM_DEPTH;
+ boolean nodeOnly = false;
+
+ Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly);
+ TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree);
+ serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins,
+ filterParents);
+ createPayloadForPut(obj);
+ if(bw != null)
+ bw.close();
+
+ URI uri = serializer.getURIForVertex(node);
+ String filenameWithUri = dirName + counter + "-" + nodeType + ".txt";
+ bw = createFile(filenameWithUri);
+ bw.write(uri.toString());
+ bw.newLine();
+ bw.close();
+ } catch (Exception e) {
+ String emsg = "Caught exception while processing [" + counter + "-" + nodeType + "] continuing";
+ System.out.println(emsg);
+ LOGGER.error(emsg);
+
+ }
+ }
+ }
+ }
+
+ }
+
+ public void createPayloadForPut(Introspector nodeObj) throws IOException {
+
+ String entityJson = nodeObj.marshal(false);
+ ObjectMapper mapper = new ObjectMapper();
+
+ ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson);
+ rootNode.remove("resource-version");
+
+ bw.newLine();
+ bw.write(rootNode.toString());
+ bw.newLine();
+ }
+
+ public void createPayloadForDmaap(Vertex node, Introspector nodeObj)
+ throws AAIException, UnsupportedEncodingException {
+
+ DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
+
+ URI uri = serializer.getURIForVertex(node);
+
+ String sourceOfTruth = "";
+ HashMap<String, Introspector> relatedVertices = new HashMap<>();
+ List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node);
+
+ for (Vertex vertex : vertexChain) {
+ try {
+
+ Introspector vertexObj = serializer.getVertexProperties(vertex);
+
+ relatedVertices.put(vertexObj.getObjectId(), vertexObj);
+ } catch (AAIUnknownObjectException e) {
+ LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
+ }
+
+ }
+
+ String transactionId = "TXID";
+ createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices);
+
+ }
+
+ public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj,
+ Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException {
+
+ String action = "CREATE";
+ final Introspector notificationEvent = loader.introspectorFromName("notification-event");
+
+ try {
+ Introspector eventHeader = loader.introspectorFromName("notification-event-header");
+ URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects);
+
+ String entityLink = urlBase + version + uri;
+
+ notificationEvent.setValue("cambria-partition", "AAI");
+
+ eventHeader.setValue("entity-link", entityLink);
+ eventHeader.setValue("action", action);
+ eventHeader.setValue("entity-type", obj.getDbName());
+ eventHeader.setValue("top-entity-type", parser.getTopEntityName());
+ eventHeader.setValue("source-name", sourceOfTruth);
+ eventHeader.setValue("version", version.toString());
+ eventHeader.setValue("id", transactionId);
+ eventHeader.setValue("event-type", "AAI-BASELINE");
+ if (eventHeader.getValue("domain") == null) {
+ eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK"));
+ }
+
+ if (eventHeader.getValue("sequence-number") == null) {
+ eventHeader.setValue("sequence-number",
+ AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK"));
+ }
+
+ if (eventHeader.getValue("severity") == null) {
+ eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK"));
+ }
+
+ if (eventHeader.getValue("id") == null) {
+ eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString());
+
+ }
+
+ if (eventHeader.getValue("timestamp") == null) {
+ eventHeader.setValue("timestamp", genDate());
+ }
+
+ List<Object> parentList = parser.getParentList();
+ parentList.clear();
+
+ if (!parser.getTopEntity().equals(parser.getEntity())) {
+ Introspector child;
+ String json = obj.marshal(false);
+ child = parser.getLoader().unmarshal(parser.getEntity().getName(), json);
+ parentList.add(child.getUnderlyingObject());
+ }
+
+ final Introspector eventObject;
+
+ String json = "";
+ if (parser.getTopEntity().equals(parser.getEntity())) {
+ json = obj.marshal(false);
+ eventObject = loader.unmarshal(obj.getName(), json);
+ } else {
+ json = parser.getTopEntity().marshal(false);
+
+ eventObject = loader.unmarshal(parser.getTopEntity().getName(), json);
+ }
+ notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject());
+ notificationEvent.setValue("entity", eventObject.getUnderlyingObject());
+
+ String entityJson = notificationEvent.marshal(false);
+
+ bw.newLine();
+ bw.write(entityJson);
+
+ } catch (AAIUnknownObjectException e) {
+ LOGGER.error("Fatal error - notification-event-header object not found!");
+ } catch (Exception e) {
+ LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e));
+ }
+ }
+
+ private void closeGraph() {
+ inMemGraph.getGraph().tx().rollback();
+ inMemGraph.getGraph().close();
+ }
+
+ public static String genDate() {
+ Date date = new Date();
+ DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS");
+ return formatter.format(date);
+ }
+
+ public static String genDate2() {
+ Date date = new Date();
+ DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
+ return formatter.format(date);
+ }
+
+ private void validateFile(String filename) {
+ File f = new File(filename);
+ if (!f.exists()) {
+ String emsg = "File " + filename + " could not be found.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ } else if (!f.canRead()) {
+ String emsg = "File " + filename + " could not be read.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ } else if (f.length() == 0) {
+ String emsg = "File " + filename + " had no data.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ }
+
+ private InputStream validateMultipleSnapshots(String filenamePrefix) {
+ if (filenamePrefix == null || filenamePrefix.length() == 0) {
+ String emsg = "No snapshot path was provided.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ String targetDir = ".";
+ int lastSeparator = filenamePrefix.lastIndexOf(File.separator);
+
+ LOGGER.debug("File separator=[" + File.separator + "] lastSeparator=" + lastSeparator + " filenamePrefix="
+ + filenamePrefix);
+ if (lastSeparator >= 0) {
+ targetDir = filenamePrefix.substring(0, lastSeparator);
+ LOGGER.debug("targetDir=" + targetDir);
+ }
+ if (targetDir.length() == 0) {
+ String emsg = "No snapshot directory was found in path:" + filenamePrefix;
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ String prefix = filenamePrefix.substring(lastSeparator + 1);
+ if (prefix == null || prefix.length() == 0) {
+ String emsg = "No snapshot file prefix was provided.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ long timeA = System.nanoTime();
+
+ ArrayList<File> snapFilesArr = new ArrayList<File>();
+ String thisSnapPrefix = prefix + ".P";
+ File fDir = new File(targetDir); // Snapshot directory
+ File[] allFilesArr = fDir.listFiles();
+ for (File snapFile : allFilesArr) {
+ String snapFName = snapFile.getName();
+ if (snapFName.startsWith(thisSnapPrefix)) {
+ snapFilesArr.add(snapFile);
+ }
+ }
+
+ if (snapFilesArr.isEmpty()) {
+ String fullFName = targetDir + AAIConstants.AAI_FILESEP + thisSnapPrefix;
+ String emsg = "Snapshot files " + fullFName + "* could not be found.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+
+ int fCount = snapFilesArr.size();
+ Iterator<File> fItr = snapFilesArr.iterator();
+ Vector<InputStream> inputStreamsV = new Vector<>();
+ for (int i = 0; i < fCount; i++) {
+ File f = snapFilesArr.get(i);
+ String fname = f.getName();
+ if (!f.canRead()) {
+ String emsg = "Snapshot file " + fname + " could not be read.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ } else if (f.length() == 0) {
+ String emsg = "Snapshot file " + fname + " had no data.";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ String fullFName = targetDir + AAIConstants.AAI_FILESEP + fname;
+ InputStream fis = null;
+ try {
+ fis = new FileInputStream(fullFName);
+ } catch (FileNotFoundException e) {
+ // should not happen at this point
+ String emsg = "Snapshot file " + fullFName + " could not be found";
+ LOGGER.error(emsg);
+ System.out.println(emsg);
+ taskExit();
+ }
+ inputStreamsV.add(fis);
+ }
+ // Now add inputStreams.elements() to the Vector,
+ InputStream sis = new SequenceInputStream(inputStreamsV.elements());
+ return (sis);
+ }
+
+ public InMemoryGraph getInMemGraph() {
+ return inMemGraph;
+ }
+
+ public void setInMemGraph(InMemoryGraph inMemGraph) {
+ this.inMemGraph = inMemGraph;
+ }
+}
+
+class CommandLineArgs {
+
+ @Parameter(names = "--help", help = true)
+ public boolean help;
+
+ @Parameter(names = "-d", description = "snapshot file to be loaded", required = true)
+ public String dataSnapshot;
+
+ @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1)
+ public boolean schemaEnabled = true;
+
+ @Parameter(names = "-c", description = "location of configuration file")
+ public String config = "";
+
+ @Parameter(names = "-o", description = "output location")
+ public String output = "";
+
+ @Parameter(names = "-f", description = "format of output")
+ public String format = "PAYLOAD";
+
+ @Parameter(names = "-n", description = "Node input file")
+ public String nodePropertyFile = "";
+
+ @Parameter(names = "-m", description = "multipe snapshots or not", arity = 1)
+ public boolean isMultipleSnapshot = false;
+
+ @Parameter(names = "-i", description = "input filter configuration file")
+ public String inputFilterPropertyFile = "";
+
+ @Parameter(names = "-p", description = "Use the partial graph", arity = 1)
+ public boolean isPartialGraph = true;
+
+}
diff --git a/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java b/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java
index 790bfa1..c12f3f3 100644
--- a/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java
+++ b/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java
@@ -38,8 +38,6 @@ import org.onap.aai.dbmap.AAIGraphConfig;
import org.onap.aai.dbmap.AAIGraph;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.logging.LogFormatTools;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.logging.LoggingContext.StatusCode;
import org.onap.aai.edges.enums.AAIDirection;
import org.onap.aai.edges.enums.EdgeProperty;
import org.onap.aai.util.AAIConfig;
@@ -47,8 +45,8 @@ import org.onap.aai.util.AAIConstants;
import org.slf4j.MDC;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.janusgraph.core.JanusGraphFactory;
import org.janusgraph.core.JanusGraph;
@@ -85,19 +83,9 @@ public class ForceDeleteTool {
Properties props = System.getProperties();
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_FORCE_DELETE_LOGBACK_PROPS);
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
- EELFLogger logger = EELFManager.getInstance().getLogger(ForceDeleteTool.class.getSimpleName());
+ Logger logger = LoggerFactory.getLogger(ForceDeleteTool.class.getSimpleName());
MDC.put("logFilenameAppender", ForceDeleteTool.class.getSimpleName());
- LoggingContext.init();
- LoggingContext.partnerName(FROMAPPID);
- LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.component("forceDeleteTool");
- LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.targetServiceName("main");
- LoggingContext.requestId(TRANSID);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
-
String actionVal = "";
String userIdVal = "";
String dataString = "";
@@ -116,8 +104,6 @@ public class ForceDeleteTool {
if (thisArg.equals("-action")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" No value passed with -action option. ");
exit(0);
}
@@ -127,8 +113,6 @@ public class ForceDeleteTool {
else if (thisArg.equals("-userId")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" No value passed with -userId option. ");
exit(0);
}
@@ -144,8 +128,6 @@ public class ForceDeleteTool {
else if (thisArg.equals("-vertexId")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" No value passed with -vertexId option. ");
exit(0);
}
@@ -154,8 +136,6 @@ public class ForceDeleteTool {
try {
vertexIdLong = Long.parseLong(nextArg);
} catch (Exception e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error("Bad value passed with -vertexId option: ["
+ nextArg + "]");
exit(0);
@@ -164,8 +144,6 @@ public class ForceDeleteTool {
else if (thisArg.equals("-params4Collect")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" No value passed with -params4Collect option. ");
exit(0);
}
@@ -175,8 +153,6 @@ public class ForceDeleteTool {
else if (thisArg.equals("-edgeId")) {
i++;
if (i >= args.length) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" No value passed with -edgeId option. ");
exit(0);
}
@@ -185,8 +161,6 @@ public class ForceDeleteTool {
edgeIdStr = nextArg;
}
else {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(" Unrecognized argument passed to ForceDeleteTool: ["
+ thisArg + "]. ");
logger.error(" Valid values are: -action -userId -vertexId -edgeId -overRideProtection -params4Collect -DISPLAY_ALL_VIDS");
@@ -198,8 +172,6 @@ public class ForceDeleteTool {
if( !actionVal.equals("COLLECT_DATA") && !actionVal.equals("DELETE_NODE") && !actionVal.equals("DELETE_EDGE")){
String emsg = "Bad action parameter [" + actionVal + "] passed to ForceDeleteTool(). Valid values = COLLECT_DATA or DELETE_NODE or DELETE_EDGE\n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(emsg);
exit(0);
}
@@ -207,16 +179,12 @@ public class ForceDeleteTool {
if( actionVal.equals("DELETE_NODE") && vertexIdLong == 0 ){
String emsg = "ERROR: No vertex ID passed on DELETE_NODE request. \n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(emsg);
exit(0);
}
else if( actionVal.equals("DELETE_EDGE") && edgeIdStr.equals("")){
String emsg = "ERROR: No edge ID passed on DELETE_EDGE request. \n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(emsg);
exit(0);
}
@@ -226,8 +194,6 @@ public class ForceDeleteTool {
if( (userIdVal.length() < 6) || userIdVal.toUpperCase().equals("AAIADMIN") ){
String emsg = "Bad userId parameter [" + userIdVal + "] passed to ForceDeleteTool(). must be not empty and not aaiadmin \n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(emsg);
exit(0);
}
@@ -241,8 +207,6 @@ public class ForceDeleteTool {
if( graph == null ){
String emsg = "could not get graph object in ForceDeleteTool() \n";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
logger.error(emsg);
exit(0);
}
@@ -250,23 +214,19 @@ public class ForceDeleteTool {
catch (AAIException e1) {
msg = e1.getErrorObject().toString();
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
logger.error(msg);
exit(0);
}
catch (Exception e2) {
msg = e2.toString();
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
logger.error(msg);
exit(0);
}
msg = "ForceDelete called by: userId [" + userIdVal + "] with these params: [" + argStr4Msg + "]";
System.out.println(msg);
- logger.info(msg);
+ logger.debug(msg);
ForceDelete fd = new ForceDelete(graph);
if( actionVal.equals("COLLECT_DATA") ){
@@ -293,8 +253,6 @@ public class ForceDeleteTool {
if( firstPipeLoc <= 0 ){
msg = "Must use the -params4Collect option when collecting data with data string in a format like: 'propName1|propVal1,propName2|propVal2'";
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(msg);
exit(0);
}
@@ -307,8 +265,6 @@ public class ForceDeleteTool {
if( pipeLoc <= 0 ){
msg = "Must use the -params4Collect option when collecting data with data string in a format like: 'propName1|propVal1,propName2|propVal2'";
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(msg);
exit(0);
}
@@ -329,14 +285,12 @@ public class ForceDeleteTool {
int descendantCount = fd.countDescendants( logger, v, 0 );
String infMsg = " Found " + descendantCount + " descendant nodes \n";
System.out.println( infMsg );
- logger.info( infMsg );
+ logger.debug( infMsg );
}
}
else {
msg = "Bad JanusGraphQuery object. ";
System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.error(msg);
exit(0);
}
@@ -344,7 +298,7 @@ public class ForceDeleteTool {
String infMsg = "\n\n Found: " + resCount + " nodes for this query: [" + qStringForMsg + "]\n";
System.out.println( infMsg );
- logger.info( infMsg );
+ logger.debug( infMsg );
}
else if( actionVal.equals("DELETE_NODE") ){
Iterator <Vertex> vtxItr = graph.vertices( vertexIdLong );
@@ -355,31 +309,31 @@ public class ForceDeleteTool {
String infMsg = " Found " + descendantCount + " descendant nodes. Note - forceDelete does not cascade to " +
" child nodes, but they may become unreachable after the delete. \n";
System.out.println( infMsg );
- logger.info( infMsg );
+ logger.debug( infMsg );
int edgeCount = fd.countEdges( logger, vtx );
infMsg = " Found total of " + edgeCount + " edges incident on this node. \n";
System.out.println( infMsg );
- logger.info( infMsg );
+ logger.debug( infMsg );
if( fd.getNodeDelConfirmation(logger, userIdVal, vtx, descendantCount, edgeCount, overRideProtection) ){
vtx.remove();
graph.tx().commit();
infMsg = ">>>>>>>>>> Removed node with vertexId = " + vertexIdLong;
- logger.info( infMsg );
+ logger.debug( infMsg );
System.out.println(infMsg);
}
else {
infMsg = " Delete Cancelled. ";
System.out.println(infMsg);
- logger.info( infMsg );
+ logger.debug( infMsg );
}
}
else {
String infMsg = ">>>>>>>>>> Vertex with vertexId = " + vertexIdLong + " not found.";
System.out.println( infMsg );
- logger.info( infMsg );
+ logger.debug( infMsg );
}
}
else if( actionVal.equals("DELETE_EDGE") ){
@@ -391,7 +345,7 @@ public class ForceDeleteTool {
if( thisEdge == null ){
String infMsg = ">>>>>>>>>> Edge with edgeId = " + edgeIdStr + " not found.";
- logger.info( infMsg );
+ logger.debug( infMsg );
System.out.println(infMsg);
exit(0);
}
@@ -400,20 +354,20 @@ public class ForceDeleteTool {
thisEdge.remove();
graph.tx().commit();
String infMsg = ">>>>>>>>>> Removed edge with edgeId = " + edgeIdStr;
- logger.info( infMsg );
+ logger.debug( infMsg );
System.out.println(infMsg);
}
else {
String infMsg = " Delete Cancelled. ";
System.out.println(infMsg);
- logger.info( infMsg );
+ logger.debug( infMsg );
}
exit(0);
}
else {
String emsg = "Unknown action parameter [" + actionVal + "] passed to ForceDeleteTool(). Valid values = COLLECT_DATA, DELETE_NODE or DELETE_EDGE \n";
System.out.println(emsg);
- logger.info( emsg );
+ logger.debug( emsg );
exit(0);
}
@@ -429,44 +383,41 @@ public class ForceDeleteTool {
public ForceDelete(JanusGraph graph) {
this.graph = graph;
}
- public void showNodeInfo(EELFLogger logger, Vertex tVert, Boolean displayAllVidsFlag ){
+ public void showNodeInfo(Logger logger, Vertex tVert, Boolean displayAllVidsFlag ){
try {
Iterator<VertexProperty<Object>> pI = tVert.properties();
String infStr = ">>> Found Vertex with VertexId = " + tVert.id() + ", properties: ";
System.out.println( infStr );
- logger.info(infStr);
+ logger.debug(infStr);
while( pI.hasNext() ){
VertexProperty<Object> tp = pI.next();
infStr = " [" + tp.key() + "|" + tp.value() + "] ";
System.out.println( infStr );
- logger.info(infStr);
+ logger.debug(infStr);
}
ArrayList <String> retArr = collectEdgeInfoForNode( logger, tVert, displayAllVidsFlag );
for( String infoStr : retArr ){
System.out.println( infoStr );
- logger.info(infoStr);
+ logger.debug(infoStr);
}
}
catch (Exception e){
String warnMsg = " -- Error -- trying to display edge info. [" + e.getMessage() + "]";
System.out.println( warnMsg );
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.warn(warnMsg);
- LoggingContext.successStatusFields();
}
}// End of showNodeInfo()
- public void showPropertiesForEdge( EELFLogger logger, Edge tEd ){
+ public void showPropertiesForEdge( Logger logger, Edge tEd ){
String infMsg = "";
if( tEd == null ){
infMsg = "null Edge object passed to showPropertiesForEdge()";
System.out.print(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
return;
}
@@ -474,82 +425,82 @@ public class ForceDeleteTool {
try {
infMsg =" Label for this Edge = [" + tEd.label() + "] ";
System.out.print(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
infMsg =" EDGE Properties for edgeId = " + tEd.id() + ": ";
System.out.print(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
Iterator <String> pI = tEd.keys().iterator();
while( pI.hasNext() ){
String propKey = pI.next();
infMsg = "Prop: [" + propKey + "], val = ["
+ tEd.property(propKey) + "] ";
System.out.print(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
}
}
catch( Exception ex ){
infMsg = " Could not retrieve properties for this edge. exMsg = ["
+ ex.getMessage() + "] ";
System.out.println( infMsg );
- logger.info(infMsg);
+ logger.debug(infMsg);
}
// Try to show what's connected to the IN side of this Edge
try {
infMsg = " Looking for the Vertex on the IN side of the edge: ";
System.out.print(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
Vertex inVtx = tEd.inVertex();
Iterator<VertexProperty<Object>> pI = inVtx.properties();
String infStr = ">>> Found Vertex with VertexId = " + inVtx.id()
+ ", properties: ";
System.out.println( infStr );
- logger.info(infStr);
+ logger.debug(infStr);
while( pI.hasNext() ){
VertexProperty<Object> tp = pI.next();
infStr = " [" + tp.key() + "|" + tp.value() + "] ";
System.out.println( infStr );
- logger.info(infStr);
+ logger.debug(infStr);
}
}
catch( Exception ex ){
infMsg = " Could not retrieve vertex data for the IN side of "
+ "the edge. exMsg = [" + ex.getMessage() + "] ";
System.out.println( infMsg );
- logger.info(infMsg);
+ logger.debug(infMsg);
}
// Try to show what's connected to the OUT side of this Edge
try {
infMsg = " Looking for the Vertex on the OUT side of the edge: ";
System.out.print(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
Vertex outVtx = tEd.outVertex();
Iterator<VertexProperty<Object>> pI = outVtx.properties();
String infStr = ">>> Found Vertex with VertexId = " + outVtx.id()
+ ", properties: ";
System.out.println( infStr );
- logger.info(infStr);
+ logger.debug(infStr);
while( pI.hasNext() ){
VertexProperty<Object> tp = pI.next();
infStr = " [" + tp.key() + "|" + tp.value() + "] ";
System.out.println( infStr );
- logger.info(infStr);
+ logger.debug(infStr);
}
}
catch( Exception ex ){
infMsg = " Could not retrieve vertex data for the OUT side of "
+ "the edge. exMsg = [" + ex.getMessage() + "] ";
System.out.println( infMsg );
- logger.info(infMsg);
+ logger.debug(infMsg);
}
}// end showPropertiesForEdge()
- public ArrayList <String> collectEdgeInfoForNode( EELFLogger logger, Vertex tVert, boolean displayAllVidsFlag ){
+ public ArrayList <String> collectEdgeInfoForNode( Logger logger, Vertex tVert, boolean displayAllVidsFlag ){
ArrayList <String> retArr = new ArrayList <String> ();
Direction dir = Direction.OUT;
for ( int i = 0; i <= 1; i++ ){
@@ -597,7 +548,7 @@ public class ForceDeleteTool {
}// end of collectEdgeInfoForNode()
- public int countEdges( EELFLogger logger, Vertex vtx ){
+ public int countEdges( Logger logger, Vertex vtx ){
int edgeCount = 0;
try {
Iterator<Edge> edgesItr = vtx.edges(Direction.BOTH);
@@ -609,25 +560,20 @@ public class ForceDeleteTool {
catch (Exception e) {
String wMsg = "-- ERROR -- Stopping the counting of edges because of Exception [" + e.getMessage() + "]";
System.out.println( wMsg );
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.warn( wMsg );
- LoggingContext.successStatusFields();
}
return edgeCount;
}// end of countEdges()
- public int countDescendants(EELFLogger logger, Vertex vtx, int levelVal ){
+ public int countDescendants(Logger logger, Vertex vtx, int levelVal ){
int totalCount = 0;
int thisLevel = levelVal + 1;
if( thisLevel > MAXDESCENDENTDEPTH ){
String wMsg = "Warning -- Stopping the counting of descendents because we reached the max depth of " + MAXDESCENDENTDEPTH;
System.out.println( wMsg );
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.warn( wMsg );
return totalCount;
}
@@ -643,10 +589,7 @@ public class ForceDeleteTool {
catch (Exception e) {
String wMsg = "Error -- Stopping the counting of descendents because of Exception [" + e.getMessage() + "]";
System.out.println( wMsg );
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.warn( wMsg );
- LoggingContext.successStatusFields();
}
@@ -654,7 +597,7 @@ public class ForceDeleteTool {
}// end of countDescendants()
- public boolean getEdgeDelConfirmation( EELFLogger logger, String uid, Edge ed,
+ public boolean getEdgeDelConfirmation( Logger logger, String uid, Edge ed,
Boolean overRideProtection ) {
showPropertiesForEdge( logger, ed );
@@ -667,20 +610,20 @@ public class ForceDeleteTool {
if (!confirm.equalsIgnoreCase("y")) {
String infMsg = " User [" + uid + "] has chosen to abandon this delete request. ";
System.out.println("\n" + infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
return false;
}
else {
String infMsg = " User [" + uid + "] has confirmed this delete request. ";
System.out.println("\n" + infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
return true;
}
} // End of getEdgeDelConfirmation()
- public boolean getNodeDelConfirmation( EELFLogger logger, String uid, Vertex vtx, int edgeCount,
+ public boolean getNodeDelConfirmation( Logger logger, String uid, Vertex vtx, int edgeCount,
int descendantCount, Boolean overRideProtection ) {
String thisNodeType = "";
try {
@@ -690,10 +633,7 @@ public class ForceDeleteTool {
// Let the user know something is going on - but they can confirm the delete if they want to.
String infMsg = " -- WARNING -- could not get an aai-node-type for this vertex. -- WARNING -- ";
System.out.println( infMsg );
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.warn( infMsg );
- LoggingContext.successStatusFields();
}
String ntListString = "";
@@ -714,10 +654,7 @@ public class ForceDeleteTool {
// Don't worry, we will use default values
String infMsg = "-- WARNING -- could not get aai.forceDel.protected values from aaiconfig.properties -- will use default values. ";
System.out.println( infMsg );
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logger.warn( infMsg );
- LoggingContext.successStatusFields();
}
if( maxDescString != null && !maxDescString.equals("") ){
@@ -752,7 +689,7 @@ public class ForceDeleteTool {
String infMsg = " >> WARNING >> This node has more descendant edges than the max ProtectedDescendantCount: " + edgeCount + ". Max = " +
maxEdgeCount + ". It can be DANGEROUS to delete one of these. << WARNING << ";
System.out.println(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
if( ! overRideProtection ){
// They cannot delete this kind of node without using the override option
giveProtErrorMsg = true;
@@ -767,7 +704,7 @@ public class ForceDeleteTool {
String infMsg = " >> WARNING >> This node has more edges than the max ProtectedEdgeCount: " + edgeCount + ". Max = " +
maxEdgeCount + ". It can be DANGEROUS to delete one of these. << WARNING << ";
System.out.println(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
if( ! overRideProtection ){
// They cannot delete this kind of node without using the override option
giveProtErrorMsg = true;
@@ -782,7 +719,7 @@ public class ForceDeleteTool {
String infMsg = " >> WARNING >> This node is a PROTECTED NODE-TYPE (" + thisNodeType + "). " +
" It can be DANGEROUS to delete one of these. << WARNING << ";
System.out.println(infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
if( ! overRideProtection ){
// They cannot delete this kind of node without using the override option
giveProtErrorMsg = true;
@@ -795,15 +732,12 @@ public class ForceDeleteTool {
if( giveProtOverRideMsg ){
String infMsg = " !!>> WARNING >>!! you are using the overRideProtection parameter which will let you do this potentially dangerous delete.";
System.out.println("\n" + infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
}
else if( giveProtErrorMsg ) {
String errMsg = " ERROR >> this kind of node can only be deleted if you pass the overRideProtection parameter.";
System.out.println("\n" + errMsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logger.error(errMsg);
- LoggingContext.successStatusFields();
return false;
}
@@ -816,20 +750,20 @@ public class ForceDeleteTool {
if (!confirm.equalsIgnoreCase("y")) {
String infMsg = " User [" + uid + "] has chosen to abandon this delete request. ";
System.out.println("\n" + infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
return false;
}
else {
String infMsg = " User [" + uid + "] has confirmed this delete request. ";
System.out.println("\n" + infMsg);
- logger.info(infMsg);
+ logger.debug(infMsg);
return true;
}
} // End of getNodeDelConfirmation()
}
- public static JanusGraph setupGraph(EELFLogger logger){
+ public static JanusGraph setupGraph(Logger logger){
JanusGraph janusGraph = null;
@@ -856,7 +790,7 @@ public class ForceDeleteTool {
return janusGraph;
}
- public static void closeGraph(JanusGraph graph, EELFLogger logger){
+ public static void closeGraph(JanusGraph graph, Logger logger){
try {
if("inmemory".equals(graphType)) {
diff --git a/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java b/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java
index ebe2180..a170453 100644
--- a/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java
+++ b/src/main/java/org/onap/aai/dbgen/GraphSONPartialReader.java
@@ -45,8 +45,8 @@ import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper;
import org.apache.tinkerpop.shaded.jackson.databind.node.JsonNodeType;
import org.onap.aai.dbmap.InMemoryGraph;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@@ -76,7 +76,7 @@ public final class GraphSONPartialReader implements GraphReader {
private boolean unwrapAdjacencyList = false;
private final GraphSONReader reader;
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(InMemoryGraph.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(InMemoryGraph.class);
final TypeReference<Map<String, Object>> mapTypeReference = new TypeReference<Map<String, Object>>() {
};
diff --git a/src/main/java/org/onap/aai/dbgen/UpdatePropertyTool.java b/src/main/java/org/onap/aai/dbgen/UpdatePropertyTool.java
new file mode 100644
index 0000000..032fc9e
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/UpdatePropertyTool.java
@@ -0,0 +1,79 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.aai.dbgen;
+
+import org.janusgraph.core.JanusGraph;
+import org.onap.aai.util.AAIConstants;
+import com.att.eelf.configuration.Configuration;
+import org.slf4j.MDC;
+
+import java.util.Properties;
+import java.util.UUID;
+
+public class UpdatePropertyTool {
+
+ private static final String FROMAPPID = "AAI-DB";
+ private static final String TRANSID = UUID.randomUUID().toString();
+
+ public static boolean SHOULD_EXIT_VM = true;
+
+ public static int EXIT_VM_STATUS_CODE = -1;
+ public static int EXIT_VM_STATUS_CODE_SUCCESS = 0;
+ public static int EXIT_VM_STATUS_CODE_FAILURE = 1;
+ public static final String PROPERTY_LOGGING_FILE_NAME = "updatePropertyTool-logback.xml";
+
+ public static void exit(int statusCode){
+ if(SHOULD_EXIT_VM){
+ System.exit(statusCode);
+ }
+ EXIT_VM_STATUS_CODE = statusCode;
+ }
+
+ /**
+ * The main method.
+ *
+ * @param args the arguments
+ */
+ public static void main(String[] args)
+ {
+ System.setProperty("aai.service.name", UpdatePropertyTool.class.getSimpleName());
+ // Set the logging file properties to be used by EELFManager
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, PROPERTY_LOGGING_FILE_NAME);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+ MDC.put("logFilenameAppender", UpdatePropertyTool.class.getSimpleName());
+
+
+ UpdatePropertyToolInternal updatePropertyToolInternal = new UpdatePropertyToolInternal();
+ JanusGraph graph = updatePropertyToolInternal.openGraph(AAIConstants.REALTIME_DB_CONFIG);
+
+ try {
+ EXIT_VM_STATUS_CODE = updatePropertyToolInternal.run(graph, args) ? EXIT_VM_STATUS_CODE_SUCCESS : EXIT_VM_STATUS_CODE_FAILURE;
+ } catch (Exception e) {
+ e.printStackTrace();
+ EXIT_VM_STATUS_CODE = EXIT_VM_STATUS_CODE_FAILURE;
+ } finally {
+ updatePropertyToolInternal.closeGraph(graph);
+ }
+
+ exit(EXIT_VM_STATUS_CODE);
+ }
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/dbgen/UpdatePropertyToolInternal.java b/src/main/java/org/onap/aai/dbgen/UpdatePropertyToolInternal.java
new file mode 100644
index 0000000..79b988d
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/UpdatePropertyToolInternal.java
@@ -0,0 +1,383 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.aai.dbgen;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.JanusGraphTransaction;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.util.AAIConfig;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class UpdatePropertyToolInternal {
+ private final Logger LOGGER;
+
+ @Parameter(names = {"--filename", "-f"}, description = "Enter path to the file name containing a list of your vertex IDs separated by new lines")
+ String filename;
+ @Parameter(names = {"--vertexId", "-v"}, description = "Vertex id(s) used to update the node(s). Flag can be used multiple times to enter a list of vertex ids")
+ private List<String> vertexIds = new ArrayList<>();
+ @Parameter(names = {"--property", "-p"}, description = "Property to be updated within the given node(s)", required = true)
+ String property;
+ @Parameter(names = {"-h", "--help"},
+ description = "Help/Usage",
+ help = true)
+ boolean help;
+
+ public UpdatePropertyToolInternal(){
+ LOGGER = LoggerFactory.getLogger(UpdatePropertyToolInternal.class.getSimpleName());
+ }
+
+ /**
+ * Run method for the main method of the update property tool. Returns boolean to indicate success or failure
+ * @param graph
+ * @param args
+ * @return
+ */
+ public boolean run(JanusGraph graph, String[] args) {
+ logAndPrint("===============Start UpdatePropertyTool===============");
+ boolean isSuccessful = true;
+ try {
+ // Error check args
+ if (!processCommandLineArguments(args)) {
+ isSuccessful = false;
+ }
+
+ // Aggregate all ids into one list
+ List<String> vertexIdList = aggregateVertexIdList(filename, vertexIds);
+
+ // Attempt update on given vertex ids
+ if (!processUpdateTransaction(graph, vertexIdList, property)) {
+ isSuccessful = false;
+ }
+
+ } catch (Exception e) {
+ isSuccessful = false;
+ logErrorAndPrint("ERROR exception thrown in run() method of UpdatePropertyTool", e);
+ }
+ logAndPrint("===============End UpdatePropertyTool===============");
+ return isSuccessful;
+ }
+
+ /**
+ * Use JCommander to process the provided command-line arguments.
+ * This method is an instance method (not static) to allow JCommander
+ * to use this instance's JCommander-annotated fields.
+ *
+ * @param args Command-line arguments.
+ */
+ private boolean processCommandLineArguments(final String[] args) {
+ logAndPrint("Start of processCommandLineArguments()");
+ final JCommander commander = new JCommander();
+ commander.addObject(this);
+ commander.setVerbose(1);
+ commander.parse(args);
+ commander.setProgramName(UpdatePropertyTool.class.getSimpleName());
+ boolean filenameExists = false;
+ boolean vertexIdExists = false;
+ boolean isValidArgs = true;
+
+ // check for help flag
+ if (help) {
+ commander.usage();
+ }
+
+ // Check for property field
+ if (property != null && !property.isEmpty()) {
+ logAndPrint("The property provided is: " + property);
+ } else {
+ logAndPrint("ERROR: No property argument was entered. Property argument is required.");
+ isValidArgs = false;
+ }
+
+ // Check for file name
+ if (filename != null && !filename.isEmpty()) {
+ logAndPrint("The filename provided is: " + filename);
+ filenameExists = true;
+ }
+
+ // Check for vertex Ids
+ if (vertexIds != null && !vertexIds.isEmpty()) {
+ logAndPrint("The vertex id(s) provided: ".concat(String.join("|", vertexIds)));
+ vertexIdExists = true;
+ }
+
+ // Fail and exit if there are no vertexIds to work with
+ if (!filenameExists && !vertexIdExists) {
+ isValidArgs = false;
+ logAndPrint("ERROR: Cannot execute UpdatePropertyTools without any given vertex Ids.");
+ }
+ logAndPrint("End of processCommandLineArguments()");
+
+ return isValidArgs;
+ }
+
+ /**
+ * Executes Gremlin queries to obtain vertices by their ids and updates the property defined by the given property parameter
+ * Reads a list of vertex IDs, and uses the prop parameter to indicate which property has corrupt index
+ * Uses gremlin console to get the property value and use that value returned to set the property and update. 
+ *
+ * @param vIdList
+ * @param propertyKey property value to be updated
+ */
+ private boolean processUpdateTransaction(JanusGraph graph, List<String> vIdList, String propertyKey) {
+ logAndPrint("Start of processUpdateTransaction()");
+ boolean isValidTransaction = true;
+
+ if (graph == null) {
+ logAndPrint("JanusGraph graph object is null. Stopping processUpdateTransaction()");
+ return false;
+ }
+
+ if (vIdList == null || vIdList.isEmpty()) {
+ logAndPrint("Vertex Id list is null or empty. Stopping processUpdateTransaction()");
+ return false;
+ }
+
+ if (propertyKey == null || propertyKey.isEmpty()) {
+ logAndPrint("propertyKey is null or empty. Stopping processUpdateTransaction()");
+ return false;
+ }
+
+ // if AAIConfig.init() fails, exit application
+ if (!setUpAAIConfig(graph)) {
+ isValidTransaction = false;
+ }
+
+ // Obtain the vertex objects using the given vertex ids
+ JanusGraphTransaction transaction = graph.newTransaction();
+
+ try {
+ GraphTraversalSource g = transaction.traversal();
+ boolean isCommitUpdateValid = false;
+ for (String vertexId: vIdList) {
+ /*
+ * Query the vertex using the vertex id from the graph
+ * Check if the query obtained a vertex
+ * Get the property value from the vertex itself
+ * Update the property using the value obtained from the query
+ */
+ GraphTraversal<Vertex, Vertex> query = g.V(vertexId);
+ if (query.hasNext()) {
+ Vertex vertex = query.next();
+ Object propertyValue = vertex.property(propertyKey).orElse(null);
+ if (propertyValue != null) {
+ vertex.property(propertyKey, propertyValue);
+ isCommitUpdateValid = true;
+ logAndPrint("Updated vertex with property: '" + propertyKey + "'. With value of: '" + propertyValue.toString() + "'");
+ } else {
+ logAndPrint("Could not update the value for property '" + propertyKey + "'. Value was empty.");
+ }
+ } else {
+ logAndPrint("Vertex not found for id: " + vertexId);
+ }
+ }
+
+ // If a transaction to update a property has occurred, commit the transaction(s)
+ if (isCommitUpdateValid) {
+ transaction.commit();
+ logAndPrint("Successful update transaction has occurred. Committing update to graph.");
+ } else {
+ transaction.rollback();
+ logAndPrint("Unsuccessful update transaction. Rolling back graph");
+ }
+ } catch (Exception e) {
+ if (transaction != null) {
+ transaction.rollback();
+ } else {
+ logAndPrint("ERROR: JanusGraphTransaction object is null");
+ }
+ logErrorAndPrint("ERROR: Could not properly query and update vertex.", e);
+ isValidTransaction = false;
+ } finally {
+ // close the transaction -- note: JanusGraph graph object will be closed in the main method.
+ if (transaction != null) {
+ transaction.close();
+ } else {
+ logAndPrint("ERROR: JanusGraphTransaction object is null. Cannot close the transaction.");
+ }
+ }
+
+ logAndPrint("End of processUpdateTransaction()");
+ return isValidTransaction;
+ }
+
+ /**
+ * Combine the vertex ids from the file and list of ids given
+ * @param filePath
+ * @param vertexIds
+ * @return
+ */
+ private List<String> aggregateVertexIdList(String filePath, List<String> vertexIds) {
+ List<String> allVertexIds = new ArrayList<>();
+
+ if (filePath != null && !filePath.isEmpty()) {
+ // Add vertex Ids listed from the given file name
+ try {
+ logAndPrint("Loading file at: " + filePath);
+ BufferedReader br = new BufferedReader(new FileReader(filePath));
+ StringBuilder sb = new StringBuilder();
+ String nextLine = br.readLine();
+
+ while (nextLine != null) {
+ if (!nextLine.matches("[0-9]+")) {
+ logAndPrint("Invalid vertex id: " + nextLine);
+ continue;
+ }
+ allVertexIds.add(nextLine);
+ sb.append(nextLine);
+ sb.append(System.lineSeparator());
+ nextLine = br.readLine();
+ }
+ String allVertexIdsString = sb.toString();
+ logAndPrint("All vertex IDs from file " + filePath + ":\n" + allVertexIdsString);
+ br.close();
+ } catch (IOException ioe) {
+ logErrorAndPrint("ERROR reading in text file failed.", ioe);
+ }
+ }
+
+ // Add all vertex Ids input one at a time from args parameter
+ StringBuilder sb = new StringBuilder();
+ for (String vId : vertexIds) {
+ if (!vId.matches("[0-9]+")) {
+ logAndPrint("Invalid vertex id: " + vId);
+ continue;
+ }
+ allVertexIds.add(vId);
+ sb.append(vId);
+ sb.append(System.lineSeparator());
+ }
+ logAndPrint("Vertex IDs from --vertexId args:\n" + sb.toString());
+
+ return allVertexIds;
+ }
+
+ /**
+ * Set up AAIConfig object
+ *
+ * @return
+ */
+ private boolean setUpAAIConfig(JanusGraph graph) {
+ String msg = "";
+ try {
+ AAIConfig.init();
+ if (graph == null) {
+ String emsg = "Graph is null. Could not get graph object. \n";
+ logAndPrint(emsg);
+ return false;
+ }
+ } catch (AAIException e1) {
+ msg = e1.getErrorObject().toString();
+ logErrorAndPrint("ERROR: AAIConfig set up failed. ", e1);
+ return false;
+ } catch (Exception e2) {
+ msg = e2.toString();
+ logErrorAndPrint("ERROR: AAIConfig set up failed. ", e2);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Set up and return and open JanusGraph Object
+ *
+ * @return
+ */
+ public JanusGraph openGraph(String configPath) {
+ logAndPrint("Setting up Janus Graph...");
+ JanusGraph janusGraph = null;
+
+ try {
+ janusGraph = JanusGraphFactory.open(
+ new AAIGraphConfig.Builder(configPath)
+ .forService(UpdatePropertyTool.class.getSimpleName())
+ .withGraphType("AAITools-" + UpdatePropertyTool.class.getSimpleName())
+ .buildConfiguration()
+ );
+ } catch (Exception e) {
+ logErrorAndPrint("Unable to open the graph. ", e);
+ }
+
+ return janusGraph;
+ }
+
+ /**
+ * Closes the given JanusGraph object
+ *
+ * @param graph
+ */
+ public void closeGraph(JanusGraph graph) {
+
+ try {
+ if (graph != null && graph.isOpen()) {
+ graph.tx().close();
+ graph.close();
+ }
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed
+ logErrorAndPrint("WARNING from final graph.shutdown(): ", ex);
+ }
+ }
+
+ /**
+ * Log and print.
+ *
+ * @param msg the msg
+ */
+ protected void logAndPrint(String msg) {
+ System.out.println(msg);
+ LOGGER.error(msg);
+ }
+
+ /**
+ * Log error and print.
+ *
+ * @param msg the msg
+ */
+ protected void logErrorAndPrint(String msg, Exception e) {
+ System.out.println(msg);
+ System.out.println(e.getCause() + " - " + e.getMessage());
+ LOGGER.error(msg, e);
+ }
+
+ /**
+ * Log error and print.
+ *
+ * @param msg the msg
+ */
+ protected void logErrorAndPrint(String msg) {
+ System.out.println(msg);
+ LOGGER.error(msg);
+ }
+}
diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java
index d1c283f..15b4c81 100644
--- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java
+++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java
@@ -20,16 +20,12 @@
package org.onap.aai.dbgen.schemamod;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
import org.onap.aai.config.PropertyPasswordConfiguration;
-import org.onap.aai.dbmap.DBConnectionType;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.Loader;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.introspection.ModelType;
import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.logging.LoggingContext;
import org.onap.aai.serialization.engines.JanusGraphDBEngine;
import org.onap.aai.serialization.engines.QueryStyle;
import org.onap.aai.serialization.engines.TransactionalGraphEngine;
@@ -38,7 +34,9 @@ import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
import org.onap.aai.util.ExceptionTranslator;
-import org.onap.aai.util.UniquePropertyCheck;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
@@ -49,6 +47,11 @@ public class SchemaMod {
private final LoaderFactory loaderFactory;
private final SchemaVersions schemaVersions;
+
+ private static boolean historyEnabled;
+
+ private Logger logger = LoggerFactory.getLogger(SchemaMod.class.getSimpleName());
+
public SchemaMod(LoaderFactory loaderFactory, SchemaVersions schemaVersions){
this.loaderFactory = loaderFactory;
@@ -57,33 +60,37 @@ public class SchemaMod {
public void execute(String[] args) {
- // Set the logging file properties to be used by EELFManager
- Properties props = System.getProperties();
- props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_SCHEMA_MOD_LOGBACK_PROPS);
- props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
-
- EELFLogger logger = EELFManager.getInstance().getLogger(UniquePropertyCheck.class.getSimpleName());
- MDC.put("logFilenameAppender", SchemaMod.class.getSimpleName());
-
+ logger = LoggerFactory.getLogger(SchemaMod.class.getSimpleName());
+
// NOTE -- We're just working with properties that are used for NODES
// for now.
String propName = "";
String targetDataType = "";
String targetIndexInfo = "";
String preserveDataFlag = "";
+ String commitBlockSizeStr = "";
+ long commitBlockSize = 120000;
- String usageString = "Usage: SchemaMod propertyName targetDataType targetIndexInfo preserveDataFlag \n";
- if (args.length != 4) {
- String emsg = "Four Parameters are required. \n" + usageString;
- logAndPrint(logger, emsg);
- System.exit(1);
- } else {
+ String usageString = "Usage: SchemaMod propertyName targetDataType targetIndexInfo preserveDataFlag [blockSize] \n";
+
+ if (args.length == 4) {
propName = args[0];
targetDataType = args[1];
targetIndexInfo = args[2];
preserveDataFlag = args[3];
}
-
+ else if (args.length == 5) {
+ propName = args[0];
+ targetDataType = args[1];
+ targetIndexInfo = args[2];
+ preserveDataFlag = args[3];
+ commitBlockSizeStr = args[4];
+ }
+ else {
+ String emsg = "Incorrect number of Parameters passed. \n" + usageString;
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
if (propName.equals("")) {
String emsg = "Bad parameter - propertyName cannot be empty. \n" + usageString;
logAndPrint(logger, emsg);
@@ -102,6 +109,17 @@ public class SchemaMod {
logAndPrint(logger, emsg);
System.exit(1);
}
+
+ try {
+ if( !commitBlockSizeStr.equals("")) {
+ // They're over-riding the commitBlockSize
+ commitBlockSize = Long.parseLong(commitBlockSizeStr);
+ }
+ } catch (NumberFormatException nfe) {
+ String emsg = "NumberFormatException - Bad block size passed in: [" + commitBlockSizeStr + "]. ";
+ logAndPrint(logger, emsg );
+ System.exit(1);
+ }
try {
AAIConfig.init();
@@ -112,15 +130,6 @@ public class SchemaMod {
System.exit(1);
}
- // Give a big warning if the DbMaps.PropertyDataTypeMap value does not
- // agree with what we're doing
- String warningMsg = "";
-
- if (!warningMsg.equals("")) {
- logAndPrint(logger, "\n>>> WARNING <<<< ");
- logAndPrint(logger, ">>> " + warningMsg + " <<<");
- }
-
logAndPrint(logger, ">>> Processing will begin in 5 seconds (unless interrupted). <<<");
try {
// Give them a chance to back out of this
@@ -138,8 +147,8 @@ public class SchemaMod {
Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
TransactionalGraphEngine engine = null;
try {
- engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
- SchemaModInternal internal = new SchemaModInternal(engine, logger, propName, targetDataType, targetIndexInfo, new Boolean(preserveDataFlag));
+ engine = new JanusGraphDBEngine(queryStyle, loader);
+ SchemaModInternalBatch internal = new SchemaModInternalBatch(engine, logger, propName, targetDataType, targetIndexInfo, Boolean.parseBoolean(preserveDataFlag), commitBlockSize);
internal.execute();
engine.startTransaction();
engine.tx().close();
@@ -157,13 +166,19 @@ public class SchemaMod {
* @param logger the logger
* @param msg the msg
*/
- protected void logAndPrint(EELFLogger logger, String msg) {
+ protected void logAndPrint(Logger logger, String msg) {
System.out.println(msg);
- logger.info(msg);
+ logger.debug(msg);
}
public static void main(String[] args) throws AAIException {
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_SCHEMA_MOD_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+
+ MDC.put("logFilenameAppender", SchemaMod.class.getSimpleName());
+
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
initializer.initialize(ctx);
@@ -176,11 +191,17 @@ public class SchemaMod {
} catch (Exception e) {
AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
System.out.println("Problems running SchemaMod "+aai.getMessage());
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
throw aai;
}
+
+ historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false"));
+ if( historyEnabled ) {
+ String emsg = "Regular SchemaMod may not be used when history.enabled=true. ";
+ System.out.println(emsg);
+ throw new AAIException("AAI-4005",emsg);
+ }
+
LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions");
SchemaMod schemaMod = new SchemaMod(loaderFactory, schemaVersions);
diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java
new file mode 100644
index 0000000..837f8a8
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod4Hist.java
@@ -0,0 +1,200 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen.schemamod;
+
+import com.att.eelf.configuration.Configuration;
+import org.onap.aai.config.PropertyPasswordConfiguration;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.ExceptionTranslator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import java.util.Properties;
+
+public class SchemaMod4Hist {
+
+ private final LoaderFactory loaderFactory;
+
+ private final SchemaVersions schemaVersions;
+
+ private static boolean historyEnabled;
+
+ public SchemaMod4Hist(LoaderFactory loaderFactory, SchemaVersions schemaVersions){
+ this.loaderFactory = loaderFactory;
+ this.schemaVersions = schemaVersions;
+ }
+
+ public void execute(String[] args) {
+
+ // Set the logging file properties to be used by EELFManager
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_SCHEMA_MOD_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+
+ Logger logger = LoggerFactory.getLogger(SchemaMod4Hist.class.getSimpleName());
+ MDC.put("logFilenameAppender", SchemaMod4Hist.class.getSimpleName());
+
+
+ // NOTE -- We're just working with properties that are used for NODES
+ // for now.
+ String propName = "";
+ String targetDataType = "";
+ String targetIndexInfo = "";
+ String preserveDataFlag = "";
+
+ String usageString = "Usage: SchemaMod4Hist propertyName targetDataType targetIndexInfo preserveDataFlag \n";
+ if (args.length != 4) {
+ String emsg = "Four Parameters are required. \n" + usageString;
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ } else {
+ propName = args[0];
+ targetDataType = args[1];
+ targetIndexInfo = args[2];
+ preserveDataFlag = args[3]; // Note - even if they pass in "false", History will preserve the data
+ }
+
+ if (propName.equals("")) {
+ String emsg = "Bad parameter - propertyName cannot be empty. \n" + usageString;
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ } else if (!targetDataType.equals("String") && !targetDataType.equals("Set<String>")
+ && !targetDataType.equals("Integer") && !targetDataType.equals("Long")
+ && !targetDataType.equals("Boolean")) {
+ String emsg = "Unsupported targetDataType. We only support String, Set<String>, Integer, Long or Boolean for now.\n"
+ + usageString;
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ } else if (!targetIndexInfo.equals("index") && !targetIndexInfo.equals("noIndex")) {
+ String emsg = "Unsupported IndexInfo. We only support: 'index' or 'noIndex' for History db.\n"
+ + usageString;
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+
+ try {
+ AAIConfig.init();
+ ErrorLogHelper.loadProperties();
+ } catch (Exception ae) {
+ String emsg = "Problem with either AAIConfig.init() or ErrorLogHelper.LoadProperties(). ";
+ logAndPrint(logger, emsg + "[" + ae.getMessage() + "]");
+ System.exit(1);
+ }
+
+ // Give a big warning if the DbMaps.PropertyDataTypeMap value does not
+ // agree with what we're doing
+ String warningMsg = "";
+
+ if (!warningMsg.equals("")) {
+ logAndPrint(logger, "\n>>> WARNING <<<< ");
+ logAndPrint(logger, ">>> " + warningMsg + " <<<");
+ }
+
+ logAndPrint(logger, ">>> Processing will begin in 5 seconds (unless interrupted). <<<");
+ try {
+ // Give them a chance to back out of this
+ Thread.sleep(5000);
+ } catch (java.lang.InterruptedException ie) {
+ logAndPrint(logger, " DB Schema Update has been aborted. ");
+ System.exit(1);
+ }
+
+ logAndPrint(logger, " ---- NOTE --- about to open graph (takes a little while)\n");
+
+ SchemaVersion version = schemaVersions.getDefaultVersion();
+ QueryStyle queryStyle = QueryStyle.TRAVERSAL;
+ ModelType introspectorFactoryType = ModelType.MOXY;
+ Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+ TransactionalGraphEngine engine = null;
+ try {
+ engine = new JanusGraphDBEngine(queryStyle, loader);
+ //NOTE - no matter what they passed in, we're passing in "true" for the
+ // preserve-data-flag last parameter since this is for HISTORY.
+ SchemaModInternal4Hist internal = new SchemaModInternal4Hist(engine, logger, propName, targetDataType, targetIndexInfo, true);
+ internal.execute();
+ engine.startTransaction();
+ engine.tx().close();
+ logAndPrint(logger, "------ Completed the SchemaMod -------- ");
+ } catch (Exception e) {
+ String emsg = "Not able to complete the requested SchemaMod4Hist \n";
+ logAndPrint(logger, e.getMessage());
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+ }
+ /**
+ * Log and print.
+ *
+ * @param logger the logger
+ * @param msg the msg
+ */
+ protected void logAndPrint(Logger logger, String msg) {
+ System.out.println(msg);
+ logger.debug(msg);
+ }
+
+ public static void main(String[] args) throws AAIException {
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
+ PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
+ initializer.initialize(ctx);
+ try {
+ ctx.scan(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+ ctx.refresh();
+ } catch (Exception e) {
+ AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
+ System.out.println("Problems running SchemaMod4Hist "+aai.getMessage());
+ ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
+ throw aai;
+ }
+
+ historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false"));
+ if( !historyEnabled ) {
+ String emsg = "SchemaMod4Hist may only be used when history.enabled=true. ";
+ System.out.println(emsg);
+ throw new AAIException("AAI-4005",emsg);
+ }
+
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ SchemaVersions schemaVersions = (SchemaVersions) ctx.getBean("schemaVersions");
+ SchemaMod4Hist schemaMod4H = new SchemaMod4Hist(loaderFactory, schemaVersions);
+ schemaMod4H.execute(args);
+
+ System.exit(0);
+ }
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java
deleted file mode 100644
index b5ce16b..0000000
--- a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/**
- * ============LICENSE_START=======================================================
- * org.onap.aai
- * ================================================================================
- * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END=========================================================
- */
-package org.onap.aai.dbgen.schemamod;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.UUID;
-
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.onap.aai.serialization.engines.TransactionalGraphEngine;
-import org.onap.aai.util.FormatDate;
-import org.onap.aai.util.UniquePropertyCheck;
-
-import com.att.eelf.configuration.EELFLogger;
-import org.janusgraph.core.Cardinality;
-import org.janusgraph.core.PropertyKey;
-import org.janusgraph.core.schema.JanusGraphManagement;
-
-public class SchemaModInternal {
- private static final String FROMAPPID = "AAI-UTILS";
- private final String TRANSID = UUID.randomUUID().toString();
- private final TransactionalGraphEngine engine;
- private final String propName;
- private final Class<?> type;
- private final String indexType;
- private final boolean preserveData;
- private final Cardinality cardinality;
- private final EELFLogger logger;
-
- public SchemaModInternal(TransactionalGraphEngine engine, EELFLogger logger, String propName, String type, String indexType, boolean preserveData) {
- this.engine = engine;
- this.propName = propName;
- this.type = determineClass(type);
- this.indexType = indexType;
- this.preserveData = preserveData;
- this.cardinality = determineCardinality(type);
- this.logger = logger;
- }
-
-
- private Class<?> determineClass(String type) {
- final Class<?> result;
- if (type.equals("String")) {
- result = String.class;
- } else if (type.equals("Set<String>")) {
- result = String.class;
- } else if (type.equals("Integer")) {
- result = Integer.class;
- } else if (type.equals("Boolean")) {
- result = Boolean.class;
- } else if (type.equals("Character")) {
- result = Character.class;
- } else if (type.equals("Long")) {
- result = Long.class;
- } else if (type.equals("Float")) {
- result = Float.class;
- } else if (type.equals("Double")) {
- result = Double.class;
- } else {
- String emsg = "Not able translate the targetDataType [" + type + "] to a Class variable.\n";
- logAndPrint(logger, emsg);
- throw new RuntimeException(emsg);
- }
-
- return result;
- }
- private Cardinality determineCardinality(String type) {
- if (type.equals("Set<String>")) {
- return Cardinality.SET;
- } else {
- return Cardinality.SINGLE;
- }
- }
- public void execute() {
- JanusGraphManagement graphMgt = null;
- boolean success = false;
- try {
- // Make sure this property is in the DB.
- graphMgt = engine.asAdmin().getManagementSystem();
- if (graphMgt == null) {
- String emsg = "Not able to get a graph Management object in SchemaMod.java\n";
- logAndPrint(logger, emsg);
- System.exit(1);
- }
- PropertyKey origPropKey = graphMgt.getPropertyKey(propName);
- if (origPropKey == null) {
- String emsg = "The propName = [" + propName + "] is not defined in our graph. ";
- logAndPrint(logger, emsg);
- System.exit(1);
- }
-
- if (indexType.equals("uniqueIndex")) {
- // Make sure the data in the property being changed can have a
- // unique-index put on it.
- // Ie. if there are duplicate values, we will not be able to
- // migrate the data back into the property.
-
-
- Graph grTmp = engine.tx();
- if( grTmp == null ){
- grTmp = engine.startTransaction();
- }
- // This is good to know in the logs
- logAndPrint(logger, "-- Starting UniquePropertyCheck. (this may take a loooong time) --");
-
- Boolean foundDupesFlag = UniquePropertyCheck.runTheCheckForUniqueness(TRANSID, FROMAPPID,
- grTmp, propName, logger);
- if (foundDupesFlag) {
- logAndPrint(logger,
- "\n\n!!!!!! >> Cannot add a uniqueIndex for the property: [" + propName
- + "] because duplicate values were found. See the log for details on which"
- + " nodes have this value. \nThey will need to be resolved (by updating those values to new"
- + " values or deleting unneeded nodes) using the standard REST-API \n");
- System.exit(1);
- }
- logAndPrint(logger, "-- Finished UniquePropertyCheck. "); // This is good to know in the logs
- }
-
-
- // ---- If we made it to here - we must be OK with making this change
-
- // Rename this property to a backup name (old name with "retired_"
- // appended plus a dateStr)
- FormatDate fd = new FormatDate("MMddHHmm", "GMT");
- String dteStr= fd.getDateTime();
-
- String retiredName = propName + "-" + dteStr + "-RETIRED";
- graphMgt.changeName(origPropKey, retiredName);
-
- // Create a new property using the original property name and the
- // targetDataType
- PropertyKey freshPropKey = graphMgt.makePropertyKey(propName).dataType(type)
- .cardinality(cardinality).make();
-
- // Create the appropriate index (if any)
- if (indexType.equals("uniqueIndex")) {
- String freshIndexName = propName + dteStr;
- graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).unique().buildCompositeIndex();
- } else if (indexType.equals("index")) {
- String freshIndexName = propName + dteStr;
- graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).buildCompositeIndex();
- }
-
- logAndPrint(logger, "Committing schema changes with graphMgt.commit()");
- graphMgt.commit();
- engine.commit();
- Graph grTmp2 = engine.startTransaction();
-
-
- // For each node that has this property, update the new from the old
- // and then remove the
- // old property from that node
- Iterator<Vertex> verts = grTmp2.traversal().V().has(retiredName);
- int vtxCount = 0;
- ArrayList<String> alreadySeenVals = new ArrayList<String>();
- while (verts.hasNext()) {
- vtxCount++;
- Vertex tmpVtx = verts.next();
- String tmpVid = tmpVtx.id().toString();
- Object origVal = tmpVtx.<Object> property(retiredName).orElse(null);
- if (preserveData) {
- tmpVtx.property(propName, origVal);
- if (indexType.equals("uniqueIndex")) {
- // We're working on a property that is being used as a
- // unique index
- String origValStr = "";
- if (origVal != null) {
- origValStr = origVal.toString();
- }
- if (alreadySeenVals.contains(origValStr)) {
- // This property is supposed to be unique, but we've
- // already seen this value in this loop
- // This should have been caught up in the first part
- // of SchemaMod, but since it wasn't, we
- // will just log the problem.
- logAndPrint(logger,
- "\n\n ---------- ERROR - could not migrate the old data [" + origValStr
- + "] for propertyName [" + propName
- + "] because this property is having a unique index put on it.");
- showPropertiesAndEdges(TRANSID, FROMAPPID, tmpVtx, logger);
- logAndPrint(logger, "-----------------------------------\n");
- } else {
- // Ok to add this prop in as a unique value
- tmpVtx.property(propName, origVal);
- logAndPrint(logger,
- "INFO -- just did the add of the freshPropertyKey and updated it with the orig value ("
- + origValStr + ")");
- }
- alreadySeenVals.add(origValStr);
- } else {
- // We are not working with a unique index
- tmpVtx.property(propName, origVal);
- logAndPrint(logger,
- "INFO -- just did the add of the freshPropertyKey and updated it with the orig value ("
- + origVal.toString() + ")");
- }
- } else {
- // existing nodes just won't have that property anymore
- // Not sure if we'd ever actually want to do this -- maybe
- // we'd do this if the new
- // data type was not compatible with the old?
- }
- tmpVtx.property(retiredName).remove();
- logAndPrint(logger, "INFO -- just did the remove of the " + retiredName + " from this vertex. (vid="
- + tmpVid + ")");
- }
-
- success = true;
- } catch (Exception ex) {
- logAndPrint(logger, "Threw a regular Exception: ");
- logAndPrint(logger, ex.getMessage());
- } finally {
- if (graphMgt != null && graphMgt.isOpen()) {
- // Any changes that worked correctly should have already done
- // their commits.
- graphMgt.rollback();
- }
- if (engine != null) {
- if (success) {
- engine.commit();
- } else {
- engine.rollback();
- }
- }
- }
- }
-
- /**
- * Show properties and edges.
- *
- * @param transId the trans id
- * @param fromAppId the from app id
- * @param tVert the t vert
- * @param logger the logger
- */
- private static void showPropertiesAndEdges(String transId, String fromAppId, Vertex tVert, EELFLogger logger) {
-
- if (tVert == null) {
- logAndPrint(logger, "Null node passed to showPropertiesAndEdges.");
- } else {
- String nodeType = "";
- Object ob = tVert.<String> property("aai-node-type");
- if (ob == null) {
- nodeType = "null";
- } else {
- nodeType = ob.toString();
- }
-
- logAndPrint(logger, " AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
- logAndPrint(logger, " Property Detail: ");
- Iterator<VertexProperty<Object>> pI = tVert.properties();
- while (pI.hasNext()) {
- VertexProperty<Object> tp = pI.next();
- Object val = tp.value();
- logAndPrint(logger, "Prop: [" + tp.key() + "], val = [" + val + "] ");
- }
-
- Iterator<Edge> eI = tVert.edges(Direction.BOTH);
- if (!eI.hasNext()) {
- logAndPrint(logger, "No edges were found for this vertex. ");
- }
- while (eI.hasNext()) {
- Edge ed = eI.next();
- String lab = ed.label();
- Vertex vtx;
- if (tVert.equals(ed.inVertex())) {
- vtx = ed.outVertex();
- } else {
- vtx = ed.inVertex();
- }
- if (vtx == null) {
- logAndPrint(logger,
- " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
- } else {
- String nType = vtx.<String> property("aai-node-type").orElse(null);
- String vid = vtx.id().toString();
- logAndPrint(logger, "Found an edge (" + lab + ") from this vertex to a [" + nType
- + "] node with VtxId = " + vid);
- }
- }
- }
- } // End of showPropertiesAndEdges()
-
- /**
- * Log and print.
- *
- * @param logger the logger
- * @param msg the msg
- */
- protected static void logAndPrint(EELFLogger logger, String msg) {
- System.out.println(msg);
- logger.info(msg);
- }
-
-}
diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java
new file mode 100644
index 0000000..f656b9f
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal4Hist.java
@@ -0,0 +1,181 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen.schemamod;
+
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.util.FormatDate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+
+public class SchemaModInternal4Hist {
+ private final TransactionalGraphEngine engine;
+ private final String propName;
+ private final Class<?> type;
+ private final String indexType;
+ private final boolean preserveData;
+ private final Cardinality cardinality;
+ private final Logger logger;
+
+ public SchemaModInternal4Hist(TransactionalGraphEngine engine, Logger logger, String propName, String type, String indexType, boolean preserveData) {
+ this.engine = engine;
+ this.propName = propName;
+ this.type = determineClass(type);
+ this.indexType = indexType;
+ this.preserveData = preserveData;
+ this.cardinality = Cardinality.LIST; // Always use this for History
+ this.logger = logger;
+ }
+
+ private Class<?> determineClass(String type) {
+ final Class<?> result;
+ if (type.equals("String")) {
+ result = String.class;
+ } else if (type.equals("Set<String>")) {
+ result = String.class;
+ } else if (type.equals("Integer")) {
+ result = Integer.class;
+ } else if (type.equals("Boolean")) {
+ result = Boolean.class;
+ } else if (type.equals("Character")) {
+ result = Character.class;
+ } else if (type.equals("Long")) {
+ result = Long.class;
+ } else if (type.equals("Float")) {
+ result = Float.class;
+ } else if (type.equals("Double")) {
+ result = Double.class;
+ } else {
+ String emsg = "Not able translate the targetDataType [" + type + "] to a Class variable.\n";
+ logAndPrint(logger, emsg);
+ throw new RuntimeException(emsg);
+ }
+
+ return result;
+ }
+
+ public void execute() {
+ JanusGraphManagement graphMgt = null;
+ boolean success = false;
+ try {
+ // Make sure this property is in the DB.
+ graphMgt = engine.asAdmin().getManagementSystem();
+ if (graphMgt == null) {
+ String emsg = "Not able to get a graph Management object in SchemaModInternal4Hist.java\n";
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+ PropertyKey origPropKey = graphMgt.getPropertyKey(propName);
+ if (origPropKey == null) {
+ String emsg = "The propName = [" + propName + "] is not defined in our graph. ";
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+
+ // Rename this property to a backup name (old name with "retired_"
+ // appended plus a dateStr)
+ FormatDate fd = new FormatDate("MMddHHmm", "GMT");
+ String dteStr= fd.getDateTime();
+
+ String retiredName = propName + "-" + dteStr + "-RETIRED";
+ graphMgt.changeName(origPropKey, retiredName);
+
+ // Create a new property using the original property name and the
+ // targetDataType
+ PropertyKey freshPropKey = graphMgt.makePropertyKey(propName).dataType(type)
+ .cardinality(cardinality).make();
+
+ // Create an index if needed (regular index will be used instead of unique for history)
+ if (indexType.equals("index") || indexType.equals("uniqueIndex")) {
+ String freshIndexName = propName + dteStr;
+ graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).buildCompositeIndex();
+ }
+
+ logAndPrint(logger, "Committing schema changes with graphMgt.commit()");
+ graphMgt.commit();
+ engine.commit();
+ Graph grTmp2 = engine.startTransaction();
+
+
+ // For each node that has this property, update the new from the old
+ // and then remove the
+ // old property from that node
+ Iterator<Vertex> verts = grTmp2.traversal().V().has(retiredName);
+ int vtxCount = 0;
+ while (verts.hasNext()) {
+ vtxCount++;
+ Vertex tmpVtx = verts.next();
+ String tmpVid = tmpVtx.id().toString();
+ Object origVal = tmpVtx.<Object> property(retiredName).orElse(null);
+ if (preserveData) {
+ tmpVtx.property(propName, origVal);
+ logAndPrint(logger,
+ "INFO -- just did the add of the freshPropertyKey and updated it with the orig value ("
+ + origVal.toString() + ")");
+ } else {
+ // existing nodes just won't have that property anymore
+ // Not sure if we'd ever actually want to do this -- maybe
+ // we'd do this if the new
+ // data type was not compatible with the old?
+ }
+ tmpVtx.property(retiredName).remove();
+ logAndPrint(logger, "INFO -- just did the remove of the " + retiredName + " from this vertex. (vid="
+ + tmpVid + ")");
+ }
+
+ success = true;
+ } catch (Exception ex) {
+ logAndPrint(logger, "Threw a regular Exception: ");
+ logAndPrint(logger, ex.getMessage());
+ } finally {
+ if (graphMgt != null && graphMgt.isOpen()) {
+ // Any changes that worked correctly should have already done
+ // their commits.
+ graphMgt.rollback();
+ }
+ if (engine != null) {
+ if (success) {
+ engine.commit();
+ } else {
+ engine.rollback();
+ }
+ }
+ }
+ }
+
+ /**
+ * Log and print.
+ *
+ * @param logger the logger
+ * @param msg the msg
+ */
+ protected static void logAndPrint(Logger logger, String msg) {
+ System.out.println(msg);
+ logger.debug(msg);
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternalBatch.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternalBatch.java
new file mode 100644
index 0000000..e88e2bf
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternalBatch.java
@@ -0,0 +1,451 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen.schemamod;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.util.FormatDate;
+
+import org.slf4j.Logger;
+import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+
+public class SchemaModInternalBatch {
+ private static final String FROMAPPID = "AAI-UTILS";
+ private final String TRANSID = UUID.randomUUID().toString();
+ private final TransactionalGraphEngine engine;
+ private final String propName;
+ private final Class<?> type;
+ private final String indexType;
+ private final boolean preserveData;
+ private final Cardinality cardinality;
+ private final long commitBlockSize;
+ private final Logger logger;
+
+ public SchemaModInternalBatch(TransactionalGraphEngine engine, Logger logger, String propName,
+ String type, String indexType, boolean preserveData, long commitBlockSize) {
+ this.engine = engine;
+ this.propName = propName;
+ this.type = determineClass(type);
+ this.indexType = indexType;
+ this.preserveData = preserveData;
+ this.cardinality = determineCardinality(type);
+ this.commitBlockSize = commitBlockSize;
+ this.logger = logger;
+ }
+
+
+ private Class<?> determineClass(String type) {
+ final Class<?> result;
+ if (type.equals("String")) {
+ result = String.class;
+ } else if (type.equals("Set<String>")) {
+ result = String.class;
+ } else if (type.equals("Integer")) {
+ result = Integer.class;
+ } else if (type.equals("Boolean")) {
+ result = Boolean.class;
+ } else if (type.equals("Character")) {
+ result = Character.class;
+ } else if (type.equals("Long")) {
+ result = Long.class;
+ } else if (type.equals("Float")) {
+ result = Float.class;
+ } else if (type.equals("Double")) {
+ result = Double.class;
+ } else {
+ String emsg = "Not able translate the targetDataType [" + type + "] to a Class variable.\n";
+ logAndPrint(logger, emsg);
+ throw new RuntimeException(emsg);
+ }
+
+ return result;
+ }
+
+ private Cardinality determineCardinality(String type) {
+ if (type.equals("Set<String>")) {
+ return Cardinality.SET;
+ } else {
+ return Cardinality.SINGLE;
+ }
+ }
+
+ public void execute() {
+ JanusGraphManagement graphMgt = null;
+ String retiredName = "";
+ boolean success = false;
+ long timeStart = System.nanoTime();
+ int batchCt = 0;
+ int totalCount = 0;
+
+ ArrayList<HashMap<String,Object>> allVerts = new ArrayList<HashMap<String,Object>>();
+ HashMap<String,Object> batchVHash = new HashMap<String,Object>();
+
+ try {
+ // Make sure this property is in the DB.
+ graphMgt = engine.asAdmin().getManagementSystem();
+ if (graphMgt == null) {
+ String emsg = "Not able to get a graph Management object in SchemaMod.java\n";
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+ PropertyKey origPropKey = graphMgt.getPropertyKey(propName);
+ if (origPropKey == null) {
+ String emsg = "The propName = [" + propName + "] is not defined in our graph. ";
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+
+ // Collect the data that needs to be processed and
+ // store as hashes of vertex-id's and the original property value
+ long timeA = System.nanoTime();
+ int msgEveryXCount = 1000;
+ Graph grTmp1 = engine.startTransaction();
+ Iterator<Vertex> allVtxItr = grTmp1.traversal().V().has(propName);
+ // Will hold these in lists that are no bigger than our
+ // allowed commitBatch size.
+ logAndPrint(logger, "Collecting the data (this takes a little while).. ");
+ int batchKey = 0;
+ int batchVCount = 0;
+ totalCount = 0;
+ int msgCount = 0;
+ logAndPrint(logger, "Collecting the data for batch # " + batchKey );
+ Object origVal = null;
+ while (allVtxItr.hasNext()) {
+ Vertex v = allVtxItr.next();
+ origVal = v.<Object>property(propName).orElse(null);
+ batchVHash.put(v.id().toString(), origVal);
+ batchVCount++;
+ totalCount++;
+ msgCount++;
+ if (batchVCount >= commitBlockSize ) {
+ // This was the last one for this batch
+ allVerts.add(batchKey, batchVHash);
+ batchKey++;
+ logAndPrint(logger, "Collecting the data for batch # " + batchKey );
+ batchVCount = 0;
+ batchVHash = new HashMap<String,Object>();
+ }
+ if( msgCount > msgEveryXCount ) {
+ msgCount = 0;
+ logAndPrint(logger, " Initial processing running... total so far = " + totalCount );
+ }
+ }
+
+ if( batchVCount > 0 ) {
+ // Add the last partial set if there is one.
+ allVerts.add(batchKey, batchVHash);
+ }
+ logAndPrint(logger, "Found " + totalCount + " nodes that will be affected. ");
+
+ batchCt = batchKey +1;
+
+ if( totalCount == 0 ) {
+ logAndPrint(logger, "INFO -- No data found to process. ");
+ System.exit(1);
+ }
+
+ logAndPrint(logger, "INFO -- Total of " + totalCount +
+ " nodes to process. Will use " + batchCt +
+ " batches. " );
+
+ long timeB = System.nanoTime();
+ long diffTime = timeB - timeA;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ logAndPrint(logger, " -- To collect all nodes took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ if (indexType.equals("uniqueIndex")) {
+ // Make sure the data in the property being changed can have a
+ // unique-index put on it.
+ // Ie. if there are duplicate values, we will not be able to
+ // migrate the data back into the property.
+ Boolean foundDupesFlag = false;
+ try {
+ foundDupesFlag = doUniquenessCheck(allVerts, propName);
+ } catch (Exception e) {
+ logAndPrint(logger, "ERROR thrown in doUniquenessCheck(): [" +
+ e.getMessage() + "]");
+ System.exit(1);
+ }
+ if (foundDupesFlag) {
+ logAndPrint(logger,
+ "\n\n!!!!!! >> Cannot add a uniqueIndex for the property: [" + propName
+ + "] because duplicate values were found. See the log for details on which"
+ + " nodes have this value. \nThey will need to be resolved (by updating those values to"
+ + " new values or deleting unneeded nodes) using the standard REST-API \n");
+ System.exit(1);
+ }
+ logAndPrint(logger, "-- Finished/Passed UniquePropertyCheck. ");
+ logAndPrint(logger, "There are " + totalCount + " nodes that have this property. ");
+ }
+
+ // ---- If we made it to here - we must be OK with making this change
+
+ // Rename this property to a backup name (old name with a dateString and
+ // "-RETIRED" appended)
+ long timeE = System.nanoTime();
+ FormatDate fd = new FormatDate("MMddHHmm", "GMT");
+ String dteStr= fd.getDateTime();
+ retiredName = propName + "-" + dteStr + "-RETIRED";
+ graphMgt.changeName(origPropKey, retiredName);
+ logAndPrint(logger, " -- Temporary property name will be: [" + retiredName + "]. ");
+
+ // Create a new property using the original property name and the
+ // targetDataType
+ PropertyKey freshPropKey = graphMgt.makePropertyKey(propName).dataType(type)
+ .cardinality(cardinality).make();
+
+ // Create the appropriate index (if any)
+ if (indexType.equals("uniqueIndex")) {
+ String freshIndexName = propName + dteStr;
+ graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).unique().buildCompositeIndex();
+ } else if (indexType.equals("index")) {
+ String freshIndexName = propName + dteStr;
+ graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).buildCompositeIndex();
+ }
+
+ logAndPrint(logger, "Committing schema changes with graphMgt.commit()");
+ graphMgt.commit();
+ success = true;
+
+ long timeF = System.nanoTime();
+ diffTime = timeF - timeE;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ logAndPrint(logger, " -- Temporary property Name Change took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ } catch (Exception ex) {
+ logAndPrint(logger, "Threw a regular Exception: ");
+ logAndPrint(logger, ex.getMessage());
+ System.exit(1);
+ } finally {
+ if (graphMgt != null && graphMgt.isOpen()) {
+ // Any changes that worked correctly should have already done
+ // their commits.
+ graphMgt.rollback();
+ }
+ if (engine != null) {
+ if (success) {
+ engine.commit();
+ } else {
+ engine.rollback();
+ }
+ }
+ }
+
+
+ // For each node that has this property, update the new from the old
+ // and then remove the old property from that node
+ // Note - do it in batches since there can be a LOT of updates.
+
+ long timeE = System.nanoTime();
+ ArrayList <String> emsgList = new ArrayList <String> ();
+ for( int batNo=0; batNo < batchCt; batNo++ ) {
+ try {
+ logAndPrint(logger, "BEGIN -- Batch # " + batNo );
+ processUpdateForBatch( allVerts.get(batNo), retiredName );
+ logAndPrint(logger, "Completed Batch # " + batNo );
+ } catch (Exception e) {
+ String emsg = "ERROR -- Batch # " + batNo +
+ " failed to process. Please clean up manually. " +
+ " data in [" + retiredName +
+ "] will have to be moved to the original property.";
+ logAndPrint(logger, emsg);
+ emsgList.add(emsg);
+ }
+ }
+ long timeF = System.nanoTime();
+ long diffTime = timeF - timeE;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ logAndPrint(logger, " -- Time to process all batches: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ logAndPrint(logger, "\nINFO -- Total of " + totalCount +
+ " nodes processed using: " + batchCt + " batches. " );
+
+ if( !emsgList.isEmpty() ) {
+ Iterator <String> eItr = emsgList.iterator();
+ logAndPrint(logger, ">>> These will need to be taken care of: ");
+ while( eItr.hasNext() ) {
+ logAndPrint(logger, (String)eItr.next());
+ }
+ }
+
+ long timeEnd = System.nanoTime();
+ diffTime = timeEnd - timeStart;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ logAndPrint(logger, " -- Total Processing time was: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ }// End of Execute()
+
+
+ private void processUpdateForBatch( HashMap<String,Object> vertHash,
+ String retiredName ) throws Exception {
+
+ Iterator<Map.Entry<String, Object>> vertHashItr = vertHash.entrySet().iterator();
+ int vtxCount = 0;
+ Boolean success = false;
+ Graph grTmpBat = engine.startTransaction();
+ try {
+ while( vertHashItr.hasNext() ){
+ Map.Entry<String, Object> entry = vertHashItr.next();
+ String tmpVid = entry.getKey();
+ Vertex tmpVtx = null;
+
+ Iterator<Vertex> oneVItr = grTmpBat.traversal().V(tmpVid);
+ while( oneVItr.hasNext() ) {
+ // should never find more than one...
+ tmpVtx = oneVItr.next();
+ Object origVal = entry.getValue();
+ if (preserveData) {
+ tmpVtx.property(propName, origVal);
+ } else {
+ // existing nodes just won't have that property anymore
+ // Might want to do this if the new
+ // data type was not compatible with the old.
+ }
+ tmpVtx.property(retiredName).remove();
+ logAndPrint(logger, "INFO -- update item: (vid= "
+ + tmpVid + ", val=[" + origVal + "])");
+ vtxCount++;
+ }
+ }
+
+ logAndPrint(logger, "INFO -- finished processing a batch with " + vtxCount + " nodes.");
+ success = true;
+ } catch (Exception ex) {
+ logAndPrint(logger, "Threw a regular Exception: ");
+ logAndPrint(logger, ex.getMessage());
+ } finally {
+ if (engine != null) {
+ if (success) {
+ logAndPrint(logger, "INFO -- committing node updates for this batch.");
+ engine.commit();
+ } else {
+ logAndPrint(logger, "ERROR -- rolling back node updates for this batch.");
+ engine.rollback();
+ }
+ }
+ }
+ if( ! success ) {
+ throw new Exception ("ERROR - could not process this batch -- see the log for details.");
+ }
+
+ }// end of processUpdateForBatch()
+
+
+ private Boolean doUniquenessCheck( ArrayList<HashMap<String,Object>> allVerts,
+ String propertyName ){
+ // Note - property can be found in more than one nodetype
+ // our uniqueness constraints are always across the entire db - so this
+ // tool looks across all nodeTypes that the property is found in.
+ long timeStart = System.nanoTime();
+ int batchCt = allVerts.size();
+ HashMap <String,Object> bigSingleHash = new HashMap <String,Object> ();
+
+ for( int batNo=0; batNo < batchCt; batNo++ ) {
+ bigSingleHash.putAll(allVerts.get(batNo));
+ }
+
+ ArrayList <Object> dupeValues = new ArrayList<Object> ();
+ int dupeCount = 0;
+
+ Iterator bItr = bigSingleHash.entrySet().iterator();
+ while( bItr.hasNext() ) {
+ Map.Entry pair = (Map.Entry)bItr.next();
+ Object thisVal = pair.getValue();
+ bItr.remove();
+ if( bigSingleHash.containsValue(thisVal) ) {
+ // Found a dupe - because the value was still in the bigHash after
+ // we removed this pair from the bigHash
+ logAndPrint(logger, " Found a dupe node with val [" + thisVal + "]");
+ if( dupeCount == 0 ) {
+ dupeValues.add(thisVal);
+ }
+ else if( !dupeValues.contains(thisVal) ){
+ // Only record the first time we see it since we're just tracking
+ // the values, not the vids
+ dupeValues.add(thisVal);
+ }
+ dupeCount++;
+ }
+ }
+
+ long timeEnd = System.nanoTime();
+ long diffTime = timeEnd - timeStart;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ logAndPrint(logger, " -- Total Uniqueness Check took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ if( dupeValues.isEmpty() ){
+ logAndPrint(logger, "\n ------------ No Duplicates Found -------- \n");
+ }
+ else {
+ logAndPrint(logger, "\n -------------- Found " + dupeCount +
+ " cases of duplicate values for property [" + propertyName + "\n\n");
+ logAndPrint(logger, "\n --- These values are in the db twice or more: ");
+ Iterator <?> dupeValItr = dupeValues.iterator();
+ while( dupeValItr.hasNext() ){
+ logAndPrint(logger, " value = [" + dupeValItr.next() + "]");
+ }
+ }
+
+ if( dupeCount > 0 ) {
+ return true;
+ }else {
+ return false;
+ }
+
+ }// end of doUniquenessCheck()
+
+
+
+ /**
+ * Log and print.
+ *
+ * @param logger the logger
+ * @param msg the msg
+ */
+ protected static void logAndPrint(Logger logger, String msg) {
+ System.out.println(msg);
+ logger.debug(msg);
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java b/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java
new file mode 100644
index 0000000..45b5d04
--- /dev/null
+++ b/src/main/java/org/onap/aai/historytruncate/HistoryTruncate.java
@@ -0,0 +1,518 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.historytruncate;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import org.onap.aai.config.PropertyPasswordConfiguration;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LogFormatTools;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.AAISystemExitUtil;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import com.att.eelf.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.janusgraph.core.JanusGraph;
+
+public class HistoryTruncate {
+
+ private static Logger LOGGER = LoggerFactory.getLogger(HistoryTruncate.class);
+
+ /* Using realtime d */
+ private static final String REALTIME_DB = "realtime";
+
+ private static final String LOG_ONLY_MODE = "LOG_ONLY";
+ private static final String DELETE_AND_LOG_MODE = "DELETE_AND_LOG";
+ private static final String SILENT_DELETE_MODE = "SILENT_DELETE";
+ static ArrayList <String> VALIDMODES = new <String> ArrayList ();
+ static {
+ VALIDMODES.add(LOG_ONLY_MODE);
+ VALIDMODES.add(DELETE_AND_LOG_MODE);
+ VALIDMODES.add(SILENT_DELETE_MODE);
+ }
+
+ private static final int batchCommitSize = 500;
+
+ private static boolean historyEnabled;
+ private static String defaultTruncateMode;
+ private static Integer defaultTruncateWindowDays;
+
+
+ /**
+ * The main method.
+ *
+ */
+ public static void main(String[] args) {
+
+ // Set the logging file properties to be used by EELFManager
+ System.setProperty("aai.service.name", HistoryTruncate.class.getSimpleName());
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
+ PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
+ initializer.initialize(ctx);
+ try {
+ ctx.scan(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+ ctx.refresh();
+ } catch (Exception e) {
+ LOGGER.error("Error - Could not initialize context beans for HistoryTruncate. ");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false"));
+ if( !historyEnabled ) {
+ String emsg = "Error - HistoryTruncate may only be used when history.enabled=true. ";
+ System.out.println(emsg);
+ LOGGER.error(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ defaultTruncateWindowDays = Integer.parseInt(ctx.getEnvironment().getProperty("history.truncate.window.days","999"));
+ defaultTruncateMode = ctx.getEnvironment().getProperty("history.truncate.mode",LOG_ONLY_MODE);
+
+ HistoryTruncate histTrunc = new HistoryTruncate();
+ boolean success = histTrunc.executeCommand(args);
+ if(success){
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ } else {
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ }// End of main()
+
+
+ public boolean executeCommand(String[] args) {
+ boolean successStatus = true;
+ // If they passed in args on the command line, then we should
+ // use those in place of the default ones we got from environment variables.
+ // "-truncateMode","LOG_ONLY","-truncateWindow","999"
+ String truncateMode = defaultTruncateMode;
+ int truncateWindowDays = defaultTruncateWindowDays;
+
+ if (args != null && args.length > 0) {
+ // They passed some arguments in that will affect processing
+ for (int i = 0; i < args.length; i++) {
+ String thisArg = args[i];
+ if (thisArg.equals("-truncateMode")) {
+ i++;
+ if (i >= args.length) {
+ LOGGER.error(" No value passed with -truncateMode option. ");
+ return false;
+ }
+ if( !VALIDMODES.contains(args[i]) ) {
+ LOGGER.error(" Unrecognized -truncateMode value passed: [" +
+ args[i] + "]. Valid values = " + VALIDMODES.toString() );
+ return false;
+ }
+ truncateMode = args[i];
+ } else if (thisArg.equals("-truncateWindowDays")) {
+ i++;
+ if (i >= args.length) {
+ LOGGER.error("No value passed with -truncateWindowDays option.");
+ return false;
+ }
+ String nextArg = args[i];
+ try {
+ truncateWindowDays = Integer.parseInt(nextArg);
+ } catch (Exception e) {
+ LOGGER.error("Bad value passed with -truncateWindowDays option: ["
+ + nextArg + "]");
+ return false;
+ }
+ } else {
+ LOGGER.error(" Unrecognized argument passed to HistoryTruncate: ["
+ + thisArg + "]. ");
+ LOGGER.error(" Valid values are: -truncateMode -truncateWindowDays ");
+ return false;
+ }
+ }
+ }
+
+ LOGGER.debug(" Running HistoryTruncate with: truncateMode = " + truncateMode +
+ ", truncateWindowDays = " + truncateWindowDays );
+
+ Long truncateEndTs = calculateTruncWindowEndTimeStamp(truncateWindowDays);
+ JanusGraph jgraph = null;
+ long scriptStartTime = System.currentTimeMillis();
+ Boolean doLogging = doLoggingOrNot( truncateMode );
+ Boolean doDelete = doDeleteOrNot( truncateMode );
+
+ try {
+ AAIConfig.init();
+ ErrorLogHelper.loadProperties();
+
+ LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while) ");
+ verifyGraph(AAIGraph.getInstance().getGraph());
+ jgraph = AAIGraph.getInstance().getGraph();
+ LOGGER.debug(" ---- got the new graph instance. ");
+
+ // Note - process edges first so they get logged as they are deleted since
+ // edges connected to vertices being deleted would get auto-deleted by the db.
+ long timeA = System.nanoTime();
+ processEdges(jgraph, truncateEndTs, doLogging, doDelete);
+ long timeB = System.nanoTime();
+ long diffTime = timeB - timeA;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" Took this long to process the Edges: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ processVerts(jgraph, truncateEndTs, doLogging, doDelete);
+ long timeC = System.nanoTime();
+ diffTime = timeC - timeB;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" Took this long to process the Vertices: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ } catch (AAIException e) {
+ ErrorLogHelper.logError("AAI_6128", e.getMessage());
+ LOGGER.error("Encountered an exception during the historyTruncate: ", e);
+ e.printStackTrace();
+ successStatus = false;
+ } catch (Exception ex) {
+ ErrorLogHelper.logError("AAI_6128", ex.getMessage());
+ LOGGER.error("Encountered an exception during the historyTruncate: ", ex);
+ ex.printStackTrace();
+ successStatus = false;
+ } finally {
+ if (jgraph != null ) {
+ // Any changes that worked correctly should have already done
+ // their commits.
+ if(!"true".equals(System.getProperty("org.onap.aai.graphadmin.started"))) {
+ if (jgraph.isOpen()) {
+ jgraph.tx().rollback();
+ jgraph.close();
+ }
+ }
+ }
+ }
+
+ return successStatus;
+ }
+
+
+ public void processVerts(JanusGraph jgraph, Long truncBeforeTs,
+ Boolean doLogging, Boolean doDelete ) {
+
+ Graph g = jgraph.newTransaction();
+ GraphTraversalSource gts = g.traversal();
+ //Iterator <Vertex> vertItr = gts.V().has(AAIProperties.END_TS, P.lt(truncBeforeTs));
+ Iterator <Vertex> vertItr = gts.V().has("end-ts", P.lt(truncBeforeTs));
+ ArrayList <Long> vidList = new ArrayList <Long> ();
+ while( vertItr.hasNext() ) {
+ Vertex tmpV = vertItr.next();
+ Long tmpVid = Long.valueOf(tmpV.id().toString());
+ vidList.add(tmpVid);
+ }
+
+ int vTotalCount = vidList.size();
+ int batchCount = vTotalCount / batchCommitSize;
+ if((batchCount * batchCommitSize) < vTotalCount){
+ batchCount++;
+ }
+
+ LOGGER.info( " Vertex TotalCount = " + vTotalCount +
+ ", we get batchCount = " + batchCount +
+ ", using commit size = " + batchCommitSize );
+
+ int vIndex = 0;
+ for(int batchNo=1; batchNo<=batchCount; batchNo++){
+ ArrayList <Long> batchVids = new ArrayList <Long> ();
+ int thisBVCount = 0;
+ while( (thisBVCount < batchCommitSize) && (vIndex < vTotalCount) ) {
+ batchVids.add(vidList.get(vIndex));
+ thisBVCount++;
+ vIndex++;
+ }
+ // now process this batch
+ LOGGER.info( "Process vertex batch # " + batchNo +
+ ", which contains " + batchVids.size() + " ids. ");
+ processVertBatch(jgraph, doLogging, doDelete, batchVids);
+ }
+ }
+
+
+ private void processVertBatch(JanusGraph jgraph, Boolean doLogging,
+ Boolean doDelete, ArrayList <Long> vidList ) {
+
+ Graph g = jgraph.newTransaction();
+ GraphTraversalSource gts = g.traversal();
+ int delFailCount = 0;
+ int vCount = 0;
+ int delCount = 0;
+
+ Iterator <Vertex> vertItr = gts.V(vidList);
+ while( vertItr.hasNext() ) {
+ vCount++;
+ Vertex tmpV = vertItr.next();
+ String tmpVid = tmpV.id().toString();
+ String tmpPropsStr = "";
+ if( doLogging ) {
+ Iterator<VertexProperty<Object>> pI = tmpV.properties();
+ while( pI.hasNext() ){
+ VertexProperty<Object> tp = pI.next();
+ Object val = tp.value();
+ tmpPropsStr = tmpPropsStr + "[" + tp.key() + "=" + val + "]";
+ }
+ LOGGER.info(" vid = " + tmpVid + ", props: (" + tmpPropsStr + ") " );
+ }
+
+ if( doDelete ) {
+ LOGGER.info("Removing vid = " + tmpVid );
+ try {
+ tmpV.remove();
+ delCount++;
+ } catch ( Exception e ) {
+ // figure out what to do
+ delFailCount++;
+ LOGGER.error("ERROR trying to delete Candidate VID = " + tmpVid + " " + LogFormatTools.getStackTop(e));
+ }
+ }
+ }
+
+ if( doDelete ) {
+ LOGGER.info("Calling commit on delete of Vertices." );
+ try {
+ g.tx().commit();
+ } catch ( Exception e ) {
+ LOGGER.error("ERROR trying to commit Vertex Deletes for this batch. " +
+ LogFormatTools.getStackTop(e) );
+ LOGGER.info( vCount + " candidate vertices processed. "
+ + " vertex deletes - COMMIT FAILED. ");
+ return;
+ }
+ }
+
+ if( doDelete ) {
+ LOGGER.info( vCount + " candidate vertices processed. " +
+ delFailCount + " delete attempts failed, " +
+ delCount + " deletes successful. ");
+ }
+ else {
+ LOGGER.info( vCount + " candidate vertices processed in this batch. " );
+ }
+ }
+
+
+ public void processEdges(JanusGraph jgraph, Long truncBeforeTs,
+ Boolean doLogging, Boolean doDelete ) {
+
+ Graph g = jgraph.newTransaction();
+ GraphTraversalSource gts = g.traversal();
+ //Iterator <Edge> edgeItr = gts.E().has(AAIProperties.END_TS, P.lt(truncBeforeTs));
+ Iterator <Edge> edgeItr = gts.E().has("end-ts", P.lt(truncBeforeTs));
+ ArrayList <String> eidList = new ArrayList <String> ();
+ while( edgeItr.hasNext() ) {
+ Edge tmpE = edgeItr.next();
+ String tmpEid = tmpE.id().toString();
+ eidList.add(tmpEid);
+ }
+
+ int eTotalCount = eidList.size();
+ int batchCount = eTotalCount / batchCommitSize;
+ if((batchCount * batchCommitSize) < eTotalCount){
+ batchCount++;
+ }
+
+ LOGGER.info( " Edge TotalCount = " + eTotalCount +
+ ", we get batchCount = " + batchCount +
+ ", using commit size = " + batchCommitSize );
+
+ int eIndex = 0;
+ for(int batchNo=1; batchNo<=batchCount; batchNo++){
+ ArrayList <String> batchEids = new ArrayList <String> ();
+ int thisBECount = 0;
+ while( (thisBECount < batchCommitSize) && (eIndex < eTotalCount) ) {
+ batchEids.add(eidList.get(eIndex));
+ thisBECount++;
+ eIndex++;
+ }
+ // now process this batch
+ LOGGER.info( "Process edge batch # " + batchNo +
+ ", which contains " + batchEids.size() + " ids. ");
+ processEdgeBatch(jgraph, doLogging, doDelete, batchEids);
+ }
+ }
+
+
+ private void processEdgeBatch(JanusGraph jgraph, Boolean doLogging,
+ Boolean doDelete, ArrayList <String> eidList ) {
+
+ Graph g = jgraph.newTransaction();
+ GraphTraversalSource gts = g.traversal();
+ int delFailCount = 0;
+ int eCount = 0;
+ int delCount = 0;
+
+ Iterator <Edge> edgeItr = gts.E(eidList);
+ while( edgeItr.hasNext() ) {
+ eCount++;
+ Edge tmpE = edgeItr.next();
+ String tmpEid = tmpE.id().toString();
+ if( doLogging ) {
+ String tmpEProps = "";
+ Iterator<Property<Object>> epI = tmpE.properties();
+ while( epI.hasNext() ){
+ Property<Object> ep = epI.next();
+ Object val = ep.value();
+ tmpEProps = tmpEProps + "[" + ep.key() + "=" + val + "]";
+ }
+ Iterator <Vertex> conVtxs = tmpE.bothVertices();
+ String tmpConVs = "";
+ while( conVtxs.hasNext() ) {
+ Vertex conV = conVtxs.next();
+ tmpConVs = tmpConVs + "[" + conV.id().toString() + "] ";
+ }
+ LOGGER.info(" eid = " + tmpEid
+ + ", Connecting vids = " + tmpConVs
+ + ", props: (" + tmpEProps + "). " );
+ }
+
+ if( doDelete ) {
+ LOGGER.info("Removing Edge eid = " + tmpEid );
+ try {
+ tmpE.remove();
+ delCount++;
+ } catch ( Exception e ) {
+ delFailCount++;
+ LOGGER.error("ERROR trying to delete Candidate Edge with eid = " + tmpEid + " " + LogFormatTools.getStackTop(e));
+ }
+ }
+ }
+
+ if( doDelete ) {
+ LOGGER.info("Calling commit on delete of Edges." );
+ try {
+ g.tx().commit();
+ } catch ( Exception e ) {
+ LOGGER.error("ERROR trying to commit Edge Deletes for this batch. " +
+ LogFormatTools.getStackTop(e) );
+ LOGGER.info( eCount + " candidate edges processed. "
+ + " edge deletes - COMMIT FAILED. ");
+ return;
+ }
+ }
+
+ if( doDelete ) {
+ LOGGER.info( eCount + " candidate edges processed. " +
+ delFailCount + " delete attempts failed, " +
+ delCount + " deletes successful. ");
+ }
+ else {
+ LOGGER.info( eCount + " candidate edges processed in this batch. " );
+ }
+ }
+
+
+ public int getCandidateVertexCount(JanusGraph jgraph, int windowDaysVal) {
+ Graph g = jgraph.newTransaction();
+ GraphTraversalSource gts = g.traversal();
+ Long truncTs = calculateTruncWindowEndTimeStamp(windowDaysVal);
+ //int candVCount = gts.V().has(AAIProperties.END_TS, P.lt(truncTs)).count().next().intValue();
+ int candVCount = gts.V().has("end-ts", P.lt(truncTs)).count().next().intValue();
+ LOGGER.info( " for the timeStamp = " + truncTs
+ + ", which corresponds to the passed truncateWindowDays = "
+ + windowDaysVal
+ + ", found " + candVCount
+ + " candidate vertices. ");
+ return candVCount;
+ }
+
+
+ public int getCandidateEdgeCount(JanusGraph jgraph, int windowDaysVal) {
+ Graph g = jgraph.newTransaction();
+ GraphTraversalSource gts = g.traversal();
+ Long truncTs = calculateTruncWindowEndTimeStamp(windowDaysVal);
+ //int candECount = gts.E().has(AAIProperties.END_TS, P.lt(truncTs)).count().next().intValue();
+ int candECount = gts.E().has("end-ts", P.lt(truncTs)).count().next().intValue();
+ LOGGER.info( " for the timeStamp = " + truncTs
+ + ", which corresponds to the passed truncateWindowDays = "
+ + windowDaysVal
+ + ", found " + candECount
+ + " candidate Edges. ");
+ return candECount;
+ }
+
+
+ public static void verifyGraph(JanusGraph graph) {
+
+ if (graph == null) {
+ String emsg = "Not able to get a graph object in DataSnapshot.java\n";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ }
+
+ public long calculateTruncWindowEndTimeStamp( int timeWindowDays ){
+ // Given a window size in days, calculate the timestamp that
+ // represents the early-edge of that window.
+
+ long unixTimeNow = System.currentTimeMillis();
+ if( timeWindowDays <= 0 ){
+ // This just means that they want to truncate all the way up to the current time
+ return unixTimeNow;
+ }
+
+ long windowInMillis = timeWindowDays * 24 * 60 * 60L * 1000;
+ long windowEdgeTimeStampInMs = unixTimeNow - windowInMillis;
+ return windowEdgeTimeStampInMs;
+
+ }
+
+ private Boolean doLoggingOrNot( String truncMode ){
+ if( truncMode.equals(SILENT_DELETE_MODE) ){
+ return false;
+ }
+ else {
+ return true;
+ }
+ }
+
+ private Boolean doDeleteOrNot( String truncMode ){
+ if( truncMode.equals(LOG_ONLY_MODE) ){
+ return false;
+ }
+ else {
+ return true;
+ }
+ }
+
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/historytruncate/HistoryTruncateTasks.java b/src/main/java/org/onap/aai/historytruncate/HistoryTruncateTasks.java
new file mode 100644
index 0000000..ec6fac3
--- /dev/null
+++ b/src/main/java/org/onap/aai/historytruncate/HistoryTruncateTasks.java
@@ -0,0 +1,108 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.historytruncate;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LogFormatTools;
+import org.onap.aai.util.AAIConfig;
+import org.onap.logging.filter.base.ONAPComponents;
+import org.onap.logging.ref.slf4j.ONAPLogConstants;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+@Component
+@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties")
+public class HistoryTruncateTasks {
+
+ @Autowired
+ private AaiScheduledTaskAuditLog auditLog;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(HistoryTruncateTasks.class);
+ private final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
+
+ @Scheduled(cron = "${historytruncatetasks.cron}" )
+ public void historyTruncateScheduleTask() throws AAIException, Exception {
+
+ if(!"true".equals(AAIConfig.get("aai.disable.check.historytruncate.running", "false"))){
+ if(checkIfHistoryTruncateIsRunning()){
+ LOGGER.debug("History Truncate is already running on the system");
+ return;
+ }
+ }
+
+ auditLog.logBefore("historyTruncateTask", ONAPComponents.AAI.toString() );
+ LOGGER.debug("Started cron job HistoryTruncate @ " + dateFormat.format(new Date()));
+ try {
+ if (AAIConfig.get("aai.cron.enable.historytruncate").equals("true")) {
+ // Until we're comfortable with how it is working, we will keep it in "LOG_ONLY" mode
+ String defaultTruncMode = "LOG_ONLY";
+ String defaultTruncWindowDays = "999";
+ String [] params = {"-truncateMode",defaultTruncMode,"-truncateWindowDays",defaultTruncWindowDays};
+ HistoryTruncate.main(params);
+ }
+ }
+ catch (Exception e) {
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for HistoryTruncate "+LogFormatTools.getStackTop(e));
+ LOGGER.debug("AAI_4000", "Exception running cron job for HistoryTruncate "+LogFormatTools.getStackTop(e));
+ } finally {
+ LOGGER.debug("Ended cron job historyTruncate @ " + dateFormat.format(new Date()));
+ }
+ auditLog.logAfter();
+
+ }
+
+ private boolean checkIfHistoryTruncateIsRunning(){
+
+ Process process = null;
+
+ int count = 0;
+ try {
+ process = new ProcessBuilder().command("bash", "-c", "ps -ef | grep '[H]istoryTruncate'").start();
+ InputStream is = process.getInputStream();
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+ while (br.readLine() != null){
+ count++;
+ }
+
+ int exitVal = process.waitFor();
+ LOGGER.debug("Exit value of the historyTruncate check process: " + exitVal);
+ } catch (Exception e) {
+ LOGGER.debug("Exception in checkIfHistoryTruncateIsRunning" + LogFormatTools.getStackTop(e));
+ }
+
+ return count > 0;
+ }
+}
+
+
diff --git a/src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java b/src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java
deleted file mode 100644
index baf28ad..0000000
--- a/src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * ============LICENSE_START=======================================================
- * org.onap.aai
- * ================================================================================
- * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END=========================================================
- */
-package org.onap.aai.interceptors.post;
-
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-import org.onap.aai.interceptors.AAIContainerFilter;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.logging.LoggingContext.StatusCode;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import javax.annotation.Priority;
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.container.ContainerResponseContext;
-import javax.ws.rs.container.ContainerResponseFilter;
-import javax.ws.rs.core.Response.Status;
-import javax.ws.rs.core.Response.StatusType;
-import java.io.IOException;
-
-@Priority(AAIResponseFilterPriority.RESET_LOGGING_CONTEXT)
-public class ResetLoggingContext extends AAIContainerFilter implements ContainerResponseFilter {
-
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(ResetLoggingContext.class);
-
- @Autowired
- private HttpServletRequest httpServletRequest;
-
- @Override
- public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext)
- throws IOException {
-
- this.cleanLoggingContext(responseContext);
-
- }
-
- private void cleanLoggingContext(ContainerResponseContext responseContext) {
- //String url = httpServletRequest.getRequestURL().toString();
- boolean success = true;
- String uri = httpServletRequest.getRequestURI();
- String queryString = httpServletRequest.getQueryString();
-
- if(queryString != null && !queryString.isEmpty()){
- uri = uri + "?" + queryString;
- }
- // For now, we use the the HTTP status code,
- // This may change, once the requirements for response codes are defined
-
- int httpStatusCode = responseContext.getStatus();
- if ( httpStatusCode < 100 || httpStatusCode > 599 ) {
- httpStatusCode = Status.INTERNAL_SERVER_ERROR.getStatusCode();
- }
- LoggingContext.responseCode(Integer.toString(httpStatusCode));
-
- StatusType sType = responseContext.getStatusInfo();
- if ( sType != null ) {
- Status.Family sFamily = sType.getFamily();
- if ( ! ( Status.Family.SUCCESSFUL.equals(sFamily) ||
- ( Status.NOT_FOUND.equals(Status.fromStatusCode(httpStatusCode)) ) ) ) {
- success = false;
- }
- }
- else {
- if ( (httpStatusCode < 200 || httpStatusCode > 299) && ( ! ( Status.NOT_FOUND.equals(Status.fromStatusCode(httpStatusCode) ) ) ) ) {
- success = false;
- }
- }
- if (success) {
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LOGGER.info(uri + " call succeeded");
- }
- else {
- LoggingContext.statusCode(StatusCode.ERROR);
- LOGGER.error(uri + " call failed with responseCode=" + httpStatusCode);
- }
- LoggingContext.clear();
-
-
- }
-
-}
diff --git a/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java b/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java
index 547a7c8..a544a90 100644
--- a/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java
+++ b/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java
@@ -19,8 +19,8 @@
*/
package org.onap.aai.interceptors.post;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.gson.JsonObject;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.interceptors.AAIContainerFilter;
@@ -41,7 +41,7 @@ import java.util.Optional;
@Priority(AAIResponseFilterPriority.RESPONSE_TRANS_LOGGING)
public class ResponseTransactionLogging extends AAIContainerFilter implements ContainerResponseFilter {
- private static final EELFLogger TRANSACTION_LOGGER = EELFManager.getInstance().getLogger(ResponseTransactionLogging.class);
+ private static final Logger TRANSACTION_LOGGER = LoggerFactory.getLogger(ResponseTransactionLogging.class);
@Autowired
private HttpServletResponse httpServletResponse;
diff --git a/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java b/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java
index afacf66..0182f2c 100644
--- a/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java
+++ b/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java
@@ -23,6 +23,8 @@ import org.onap.aai.exceptions.AAIException;
import org.onap.aai.interceptors.AAIContainerFilter;
import org.onap.aai.interceptors.AAIHeaderProperties;
import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.logging.filter.base.Constants;
+import org.onap.logging.ref.slf4j.ONAPLogConstants;
import javax.annotation.Priority;
import javax.ws.rs.container.ContainerRequestContext;
@@ -36,7 +38,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
-import java.util.UUID;
@Provider
@PreMatching
@@ -49,36 +50,28 @@ public class HeaderValidation extends AAIContainerFilter implements ContainerReq
Optional<Response> oResp;
MultivaluedMap<String, String> headersMap = requestContext.getHeaders();
-
- String transId = headersMap.getFirst(AAIHeaderProperties.TRANSACTION_ID);
- String fromAppId = headersMap.getFirst(AAIHeaderProperties.FROM_APP_ID);
List<MediaType> acceptHeaderValues = requestContext.getAcceptableMediaTypes();
-
+ String fromAppId = getPartnerName(requestContext);
oResp = this.validateHeaderValuePresence(fromAppId, "AAI_4009", acceptHeaderValues);
if (oResp.isPresent()) {
requestContext.abortWith(oResp.get());
return;
}
+ String transId = getRequestId(requestContext);
oResp = this.validateHeaderValuePresence(transId, "AAI_4010", acceptHeaderValues);
if (oResp.isPresent()) {
requestContext.abortWith(oResp.get());
return;
}
- if (!this.isValidUUID(transId)) {
- transId = UUID.randomUUID().toString();
- requestContext.getHeaders().get(AAIHeaderProperties.TRANSACTION_ID).clear();
- requestContext.getHeaders().add(AAIHeaderProperties.TRANSACTION_ID, transId);
- }
-
}
private Optional<Response> validateHeaderValuePresence(String value, String errorCode,
List<MediaType> acceptHeaderValues) {
Response response = null;
AAIException aaie;
- if (value == null) {
+ if (value == null || value.isEmpty()) {
aaie = new AAIException(errorCode);
return Optional.of(Response.status(aaie.getErrorObject().getHTTPResponseCode())
.entity(ErrorLogHelper.getRESTAPIErrorResponse(acceptHeaderValues, aaie, new ArrayList<>()))
@@ -87,5 +80,52 @@ public class HeaderValidation extends AAIContainerFilter implements ContainerReq
return Optional.ofNullable(response);
}
+ public String getRequestId(ContainerRequestContext requestContext) {
+ String requestId = requestContext.getHeaderString(ONAPLogConstants.Headers.REQUEST_ID);
+ if (requestId == null || requestId.isEmpty()) {
+ requestId = requestContext.getHeaderString(Constants.HttpHeaders.HEADER_REQUEST_ID);
+ if (requestId == null || requestId.isEmpty()) {
+ requestId = requestContext.getHeaderString(Constants.HttpHeaders.TRANSACTION_ID);
+ if (requestId == null || requestId.isEmpty()) {
+ requestId = requestContext.getHeaderString(Constants.HttpHeaders.ECOMP_REQUEST_ID);
+ if (requestId == null || requestId.isEmpty()) {
+ return requestId;
+ }
+ }
+ }
+ }
+ if (requestContext.getHeaders().get(ONAPLogConstants.Headers.REQUEST_ID) != null) {
+ requestContext.getHeaders().get(ONAPLogConstants.Headers.REQUEST_ID).clear();
+ }
+ if (requestContext.getHeaders().get(Constants.HttpHeaders.TRANSACTION_ID) != null) {
+ requestContext.getHeaders().get(Constants.HttpHeaders.TRANSACTION_ID).clear();
+ }
+ if (requestContext.getHeaders().get(Constants.HttpHeaders.HEADER_REQUEST_ID) != null) {
+ requestContext.getHeaders().get(Constants.HttpHeaders.HEADER_REQUEST_ID).clear();
+ }
+ if (requestContext.getHeaders().get(Constants.HttpHeaders.ECOMP_REQUEST_ID) != null) {
+ requestContext.getHeaders().get(Constants.HttpHeaders.ECOMP_REQUEST_ID).clear();
+ }
+ requestContext.getHeaders().add(Constants.HttpHeaders.TRANSACTION_ID, requestId);
+
+ return requestId;
+ }
+ public String getPartnerName(ContainerRequestContext requestContext) {
+ String partnerName = requestContext.getHeaderString(ONAPLogConstants.Headers.PARTNER_NAME);
+ if (partnerName == null || (partnerName.isEmpty())) {
+ partnerName = requestContext.getHeaderString(AAIHeaderProperties.FROM_APP_ID);
+ if (partnerName == null || (partnerName.isEmpty())) {
+ return partnerName;
+ }
+ }
+ if (requestContext.getHeaders().get(ONAPLogConstants.Headers.PARTNER_NAME) != null) {
+ requestContext.getHeaders().get(ONAPLogConstants.Headers.PARTNER_NAME).clear();
+ }
+ if (requestContext.getHeaders().get(AAIHeaderProperties.FROM_APP_ID) != null) {
+ requestContext.getHeaders().get(AAIHeaderProperties.FROM_APP_ID).clear();
+ }
+ requestContext.getHeaders().add(AAIHeaderProperties.FROM_APP_ID, partnerName);
+ return partnerName;
+ }
}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java b/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java
index b770296..6aa180b 100644
--- a/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java
+++ b/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java
@@ -19,8 +19,8 @@
*/
package org.onap.aai.interceptors.pre;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.gson.JsonObject;
import org.glassfish.jersey.message.internal.ReaderWriter;
import org.onap.aai.exceptions.AAIException;
@@ -53,7 +53,7 @@ import java.util.UUID;
@Priority(AAIRequestFilterPriority.REQUEST_TRANS_LOGGING)
public class RequestTransactionLogging extends AAIContainerFilter implements ContainerRequestFilter {
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(RequestTransactionLogging.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(RequestTransactionLogging.class);
@Autowired
private HttpServletRequest httpServletRequest;
diff --git a/src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java b/src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java
deleted file mode 100644
index 6c3a7fc..0000000
--- a/src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * ============LICENSE_START=======================================================
- * org.onap.aai
- * ================================================================================
- * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END=========================================================
- */
-package org.onap.aai.interceptors.pre;
-
-import org.onap.aai.interceptors.AAIContainerFilter;
-import org.onap.aai.interceptors.AAIHeaderProperties;
-import org.onap.aai.logging.LoggingContext;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.env.Environment;
-
-import javax.annotation.Priority;
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.container.ContainerRequestFilter;
-import javax.ws.rs.container.PreMatching;
-import javax.ws.rs.core.MultivaluedMap;
-import javax.ws.rs.ext.Provider;
-import java.io.IOException;
-
-@Provider
-@PreMatching
-@Priority(AAIRequestFilterPriority.SET_LOGGING_CONTEXT)
-public class SetLoggingContext extends AAIContainerFilter implements ContainerRequestFilter {
-
- @Autowired
- private Environment environment;
-
- @Autowired
- private HttpServletRequest httpServletRequest;
-
- @Override
- public void filter(ContainerRequestContext requestContext) throws IOException {
-
- String uri = httpServletRequest.getRequestURI();
- String queryString = httpServletRequest.getQueryString();
-
- if(queryString != null && !queryString.isEmpty()){
- uri = uri + "?" + queryString;
- }
-
- String httpMethod = requestContext.getMethod();
-
- MultivaluedMap<String, String> headersMap = requestContext.getHeaders();
-
- String transId = headersMap.getFirst(AAIHeaderProperties.TRANSACTION_ID);
- String fromAppId = headersMap.getFirst(AAIHeaderProperties.FROM_APP_ID);
-
- LoggingContext.init();
- LoggingContext.requestId(transId);
- LoggingContext.partnerName(fromAppId);
- LoggingContext.targetEntity(environment.getProperty("spring.application.name"));
- LoggingContext.component(fromAppId);
- LoggingContext.serviceName(httpMethod + " " + uri);
- LoggingContext.targetServiceName(httpMethod + " " + uri);
- LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
- }
-
-}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java b/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java
index 73b7877..8c973a1 100644
--- a/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java
+++ b/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java
@@ -19,7 +19,7 @@
*/
package org.onap.aai.interceptors.pre;
-import org.onap.aai.auth.AAIAuthCore;
+import org.onap.aai.aaf.auth.AAIAuthCore;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.interceptors.AAIContainerFilter;
import org.onap.aai.interceptors.AAIHeaderProperties;
diff --git a/src/main/java/org/onap/aai/migration/EdgeMigrator.java b/src/main/java/org/onap/aai/migration/EdgeMigrator.java
index bb8acfc..c002afd 100644
--- a/src/main/java/org/onap/aai/migration/EdgeMigrator.java
+++ b/src/main/java/org/onap/aai/migration/EdgeMigrator.java
@@ -124,7 +124,7 @@ public abstract class EdgeMigrator extends Migrator {
e.properties().forEachRemaining(prop -> prop.remove());
edgeSerializer.addProperties(e, rule);
} else {
- logger.info("found vertices connected by unkwown label: out=" + out + " label="
+ logger.debug("found vertices connected by unkwown label: out=" + out + " label="
+ e.label() + " in=" + in);
}
}
diff --git a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java
index b3faec8..3f25119 100644
--- a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java
+++ b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java
@@ -106,12 +106,12 @@ public abstract class EdgeSwingMigrator extends Migrator {
try {
// If the old and new Vertices aren't populated, throw an exception
if( oldNode == null ){
- logger.info ( "null oldNode passed to swingEdges() ");
+ logger.debug ( "null oldNode passed to swingEdges() ");
success = false;
return;
}
else if( newNode == null ){
- logger.info ( "null newNode passed to swingEdges() ");
+ logger.debug ( "null newNode passed to swingEdges() ");
success = false;
return;
}
@@ -120,7 +120,7 @@ public abstract class EdgeSwingMigrator extends Migrator {
&& !edgeDirRestr.equals("IN")
&& !edgeDirRestr.equals("OUT") )
){
- logger.info ( "invalid direction passed to swingEdges(). valid values are BOTH/IN/OUT ");
+ logger.debug ( "invalid direction passed to swingEdges(). valid values are BOTH/IN/OUT ");
success = false;
return;
}
@@ -140,7 +140,7 @@ public abstract class EdgeSwingMigrator extends Migrator {
// If the nodeTypes don't match, throw an error
if( !oldNodeType.equals(newNodeType) ){
- logger.info ( "Can not swing edge from a [" + oldNodeType + "] node to a [" +
+ logger.debug ( "Can not swing edge from a [" + oldNodeType + "] node to a [" +
newNodeType + "] node. ");
success = false;
return;
@@ -182,7 +182,7 @@ public abstract class EdgeSwingMigrator extends Migrator {
}
String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present";
- logger.info ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is ["
+ logger.debug ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is ["
+ otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType
+ "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. ");
// remove the old edge
@@ -204,7 +204,7 @@ public abstract class EdgeSwingMigrator extends Migrator {
newOutE.property(pair.getKey().toString(), pair.getValue().toString() );
}
}else {
- logger.info("\n Edge was not swung due to Multiplicity Rule Violation...");
+ logger.debug("\n Edge was not swung due to Multiplicity Rule Violation...");
}
}
}
@@ -245,7 +245,7 @@ public abstract class EdgeSwingMigrator extends Migrator {
}
String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present";
- logger.info ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is ["
+ logger.debug ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is ["
+ otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType
+ "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. ");
@@ -268,7 +268,7 @@ public abstract class EdgeSwingMigrator extends Migrator {
newInE.property(pair.getKey().toString(), pair.getValue().toString() );
}
} else {
- logger.info("\t Edge was not swung due to Multiplicity Rule Violation...");
+ logger.debug("\t Edge was not swung due to Multiplicity Rule Violation...");
}
}
}
diff --git a/src/main/java/org/onap/aai/migration/MigrationController.java b/src/main/java/org/onap/aai/migration/MigrationController.java
index abd4648..8d758e3 100644
--- a/src/main/java/org/onap/aai/migration/MigrationController.java
+++ b/src/main/java/org/onap/aai/migration/MigrationController.java
@@ -25,8 +25,6 @@ import org.onap.aai.edges.EdgeIngestor;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.logging.LoggingContext.StatusCode;
import org.onap.aai.serialization.db.EdgeSerializer;
import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConstants;
@@ -49,16 +47,6 @@ public class MigrationController {
*/
public static void main(String[] args) throws AAIException {
- LoggingContext.init();
- LoggingContext.partnerName("Migration");
- LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.component("MigrationController");
- LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.targetServiceName("main");
- LoggingContext.requestId(UUID.randomUUID().toString());
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
-
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
initializer.initialize(ctx);
@@ -71,8 +59,6 @@ public class MigrationController {
} catch (Exception e) {
AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
System.out.println("Problems running tool "+aai.getMessage());
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
throw aai;
}
diff --git a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java
index 55b03fb..7cb71ca 100644
--- a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java
+++ b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java
@@ -20,51 +20,44 @@
package org.onap.aai.migration;
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Properties;
-import java.util.Set;
-import java.util.stream.Collectors;
-
+import com.att.eelf.configuration.Configuration;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.onap.aai.datasnapshot.DataSnapshot;
import org.onap.aai.db.props.AAIProperties;
import org.onap.aai.dbmap.AAIGraph;
-import org.onap.aai.dbmap.DBConnectionType;
import org.onap.aai.edges.EdgeIngestor;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.Loader;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.introspection.ModelType;
import org.onap.aai.serialization.db.EdgeSerializer;
-import org.onap.aai.setup.SchemaVersions;
-import org.onap.aai.setup.SchemaVersion;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.logging.LoggingContext.StatusCode;
-import org.onap.aai.serialization.engines.QueryStyle;
import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.QueryStyle;
import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConstants;
import org.onap.aai.util.FormatDate;
import org.reflections.Reflections;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
-import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
/**
* Runs a series of migrations from a defined directory based on the presence of
@@ -74,8 +67,7 @@ import com.beust.jcommander.Parameter;
*/
public class MigrationControllerInternal {
- private EELFLogger logger;
- private final int DANGER_ZONE = 10;
+ private Logger logger;
public static final String VERTEX_TYPE = "migration-list-1707";
private final List<String> resultsSummary = new ArrayList<>();
private final List<NotificationHelper> notifications = new ArrayList<>();
@@ -107,11 +99,9 @@ public class MigrationControllerInternal {
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml");
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
- logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName());
+ logger = LoggerFactory.getLogger(MigrationControllerInternal.class.getSimpleName());
MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
- boolean loadSnapshot = false;
-
CommandLineArgs cArgs = new CommandLineArgs();
JCommander jCommander = new JCommander(cArgs, args);
@@ -123,8 +113,6 @@ public class MigrationControllerInternal {
try {
PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config);
if (config.getString("storage.backend").equals("inmemory")) {
- loadSnapshot = true;
-// System.setProperty("load.snapshot.file", "true");
System.setProperty("snapshot.location", cArgs.dataSnapshot);
String snapshotLocation =cArgs.dataSnapshot;
String snapshotDir;
@@ -136,15 +124,13 @@ public class MigrationControllerInternal {
snapshotFile = snapshotLocation;
} else {
snapshotDir = snapshotLocation.substring(0, index+1);
- snapshotFile = snapshotLocation.substring(index+1, snapshotLocation.length()) ;
+ snapshotFile = snapshotLocation.substring(index+1) ;
}
String [] dataSnapShotArgs = {"-c","MULTITHREAD_RELOAD","-f", snapshotFile, "-oldFileDir",snapshotDir, "-caller","migration"};
DataSnapshot dataSnapshot = new DataSnapshot();
dataSnapshot.executeCommand(dataSnapShotArgs, true, false, null, "MULTITHREAD_RELOAD", snapshotFile);
}
} catch (ConfigurationException e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e));
return;
}
@@ -160,7 +146,7 @@ public class MigrationControllerInternal {
QueryStyle queryStyle = QueryStyle.TRAVERSAL;
ModelType introspectorFactoryType = ModelType.MOXY;
Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
- TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
+ TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, loader);
if (cArgs.help) {
jCommander.usage();
@@ -172,7 +158,7 @@ public class MigrationControllerInternal {
List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections));
//Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names
if (cArgs.list) {
- listMigrationWithStatus(cArgs, migratorClasses, engine);
+ listMigrationWithStatus(migratorClasses, engine);
return;
}
@@ -180,18 +166,15 @@ public class MigrationControllerInternal {
//Excluding any migration class when run migration from script.Pass flag "-e" following by the class names
if (!cArgs.excludeClasses.isEmpty()) {
migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses);
- listMigrationWithStatus(cArgs, migratorClasses, engine);
+ listMigrationWithStatus(migratorClasses, engine);
}
List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses);
sortList(migratorClassesToRun);
if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + ".");
logAndPrint("---------- Done ----------");
- LoggingContext.successStatusFields();
}
logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts.");
@@ -199,7 +182,7 @@ public class MigrationControllerInternal {
if (!cArgs.skipPreMigrationSnapShot) {
- takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun);
+ takePreSnapshotIfRequired(engine);
}
for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
@@ -222,10 +205,7 @@ public class MigrationControllerInternal {
SchemaVersions.class
).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions);
} catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e));
- LoggingContext.successStatusFields();
engine.rollback();
continue;
}
@@ -245,11 +225,8 @@ public class MigrationControllerInternal {
try {
notificationHelper.triggerEvents();
} catch (AAIException e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
logAndPrint("\tcould not event");
logger.error("could not event", e);
- LoggingContext.successStatusFields();
}
}
logAndPrint("---------- Done ----------");
@@ -276,16 +253,13 @@ public class MigrationControllerInternal {
List<String> excludeClasses,
List<Class<? extends Migrator>> migratorClasses) {
- List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses
+ return migratorClasses
.stream()
.filter(migratorClass -> !excludeClasses.contains(migratorClass
.getSimpleName())).collect(Collectors.toList());
-
- return filteredMigratorClasses;
}
- private void listMigrationWithStatus(CommandLineArgs cArgs,
- List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) {
+ private void listMigrationWithStatus(List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) {
sortList(migratorClasses);
engine.startTransaction();
System.out.println("---------- List of all migrations ----------");
@@ -317,9 +291,9 @@ public class MigrationControllerInternal {
return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext();
}
private Set<Class<? extends Migrator>> findClasses(Reflections reflections) {
- Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class).stream()
- .filter(clazz -> clazz.isAnnotationPresent(MigrationPriority.class))
- .collect(Collectors.toSet());
+ Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class).stream()
+ .filter(clazz -> clazz.isAnnotationPresent(MigrationPriority.class))
+ .collect(Collectors.toSet());
/*
* TODO- Change this to make sure only classes in the specific $release are added in the runList
* Or add a annotation like exclude which folks again need to remember to add ??
@@ -331,22 +305,7 @@ public class MigrationControllerInternal {
}
- private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) {
-
- /*int sum = 0;
- for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
- if (migratorClass.isAnnotationPresent(Enabled.class)) {
- sum += migratorClass.getAnnotation(MigrationPriority.class).value();
- }
- }
-
- if (sum >= DANGER_ZONE) {
-
- logAndPrint("Entered Danger Zone. Taking snapshot.");
- }*/
-
- //always take snapshot for now
-
+ private void takePreSnapshotIfRequired(TransactionalGraphEngine engine) {
generateSnapshot(engine, "pre");
}
@@ -357,15 +316,13 @@ public class MigrationControllerInternal {
List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>();
if (cArgs.scripts.isEmpty()) {
return migratorClasses;
+
}
-
for (Class<? extends Migrator> migratorClass : migratorClasses) {
- if (migratorExplicitlySpecified(cArgs, migratorClass.getSimpleName())
- || migratorToRunWhenDisabled(cArgs, migratorClass.getSimpleName())) {
+ if (migratorExplicitlySpecified(cArgs, migratorClass.getSimpleName()) || migratorToRunWhenDisabled(cArgs, migratorClass.getSimpleName())) {
migratorClassesToRun.add(migratorClass);
}
}
-
return migratorClassesToRun;
}
@@ -377,7 +334,7 @@ public class MigrationControllerInternal {
}
private void sortList(List<Class<? extends Migrator>> migratorClasses) {
- Collections.sort(migratorClasses, (m1, m2) -> {
+ migratorClasses.sort((m1, m2) -> {
try {
if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) {
return 1;
@@ -399,7 +356,6 @@ public class MigrationControllerInternal {
String dateStr= fd.getDateTime();
String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson";
logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName);
- Graph transaction = null;
try {
Path pathToFile = Paths.get(fileName);
@@ -409,14 +365,8 @@ public class MigrationControllerInternal {
String [] dataSnapshotArgs = {"-c","THREADED_SNAPSHOT", "-fileName",fileName, "-caller","migration"};
DataSnapshot dataSnapshot = new DataSnapshot();
dataSnapshot.executeCommand(dataSnapshotArgs, true, false, null, "THREADED_SNAPSHOT", null);
-// transaction = engine.startTransaction();
-// transaction.io(IoCore.graphson()).writeGraph(fileName);
-// engine.rollback();
} catch (IOException e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e));
- LoggingContext.successStatusFields();
engine.rollback();
}
@@ -430,7 +380,7 @@ public class MigrationControllerInternal {
*/
protected void logAndPrint(String msg) {
System.out.println(msg);
- logger.info(msg);
+ logger.debug(msg);
}
/**
@@ -448,24 +398,18 @@ public class MigrationControllerInternal {
String message;
if (migrator.getStatus().equals(Status.FAILURE)) {
message = "Migration " + simpleName + " Failed. Rolling back.";
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logAndPrint("\t" + message);
- LoggingContext.successStatusFields();
migrator.rollback();
} else if (migrator.getStatus().equals(Status.CHECK_LOGS)) {
message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back.";
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
logAndPrint("\t" + message);
- LoggingContext.successStatusFields();
migrator.rollback();
} else {
MDC.put("logFilenameAppender", simpleName + "/" + simpleName);
if (cArgs.commit) {
if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) {
- engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate();
+ engine.asAdmin().getTraversalSource().addV(VERTEX_TYPE).property(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate();
}
engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE)
.property(simpleName, true).iterate();
diff --git a/src/main/java/org/onap/aai/migration/Migrator.java b/src/main/java/org/onap/aai/migration/Migrator.java
index 791fec0..7d6a7c1 100644
--- a/src/main/java/org/onap/aai/migration/Migrator.java
+++ b/src/main/java/org/onap/aai/migration/Migrator.java
@@ -55,8 +55,8 @@ import org.onap.aai.serialization.engines.TransactionalGraphEngine;
import org.onap.aai.setup.SchemaVersion;
import org.onap.aai.setup.SchemaVersions;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class defines an A&AI Migration
@@ -65,7 +65,7 @@ import com.att.eelf.configuration.EELFManager;
@MigrationDangerRating(0)
public abstract class Migrator implements Runnable {
- protected EELFLogger logger = null;
+ protected Logger logger = null;
protected DBSerializer serializer = null;
protected Loader loader = null;
@@ -96,8 +96,8 @@ public abstract class Migrator implements Runnable {
this.schemaVersions = schemaVersions;
initDBSerializer();
this.notificationHelper = new NotificationHelper(loader, serializer, loaderFactory, schemaVersions, engine, "AAI-MIGRATION", this.getMigrationName());
- logger = EELFManager.getInstance().getLogger(this.getClass().getSimpleName());
- logger.info("\tInitilization of " + this.getClass().getSimpleName() + " migration script complete.");
+ logger = LoggerFactory.getLogger(this.getClass().getSimpleName());
+ logger.debug("\tInitilization of " + this.getClass().getSimpleName() + " migration script complete.");
}
/**
@@ -139,7 +139,7 @@ public abstract class Migrator implements Runnable {
logger.error("Unable to generate file with dmaap msgs for " + getMigrationName(), e);
}
} else {
- logger.info("No dmaap msgs detected for " + getMigrationName());
+ logger.debug("No dmaap msgs detected for " + getMigrationName());
}
}
diff --git a/src/main/java/org/onap/aai/migration/NotificationHelper.java b/src/main/java/org/onap/aai/migration/NotificationHelper.java
index ff5c030..f10a824 100644
--- a/src/main/java/org/onap/aai/migration/NotificationHelper.java
+++ b/src/main/java/org/onap/aai/migration/NotificationHelper.java
@@ -37,8 +37,8 @@ import org.onap.aai.serialization.db.DBSerializer;
import org.onap.aai.serialization.engines.TransactionalGraphEngine;
import org.onap.aai.serialization.engines.query.QueryEngine;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.onap.aai.setup.SchemaVersions;
/**
@@ -46,7 +46,7 @@ import org.onap.aai.setup.SchemaVersions;
*/
public class NotificationHelper {
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(NotificationHelper.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(NotificationHelper.class);
protected final DBSerializer serializer;
protected final Loader loader;
protected final TransactionalGraphEngine engine;
diff --git a/src/main/java/org/onap/aai/migration/PropertyMigrator.java b/src/main/java/org/onap/aai/migration/PropertyMigrator.java
index 4599243..15e704d 100644
--- a/src/main/java/org/onap/aai/migration/PropertyMigrator.java
+++ b/src/main/java/org/onap/aai/migration/PropertyMigrator.java
@@ -68,11 +68,11 @@ public abstract class PropertyMigrator extends Migrator {
*/
@Override
public void run() {
- logger.info("-------- Starting PropertyMigrator for node type " + P.within(this.getAffectedNodeTypes().get())
+ logger.debug("-------- Starting PropertyMigrator for node type " + P.within(this.getAffectedNodeTypes().get())
+ " from property " + OLD_FIELD + " to " + NEW_FIELD + " --------");
modifySchema();
executeModifyOperation();
- logger.info(Migrator.MIGRATION_SUMMARY_COUNT + changedVertexCount + " vertices modified.");
+ logger.debug(Migrator.MIGRATION_SUMMARY_COUNT + changedVertexCount + " vertices modified.");
}
protected void modifySchema() {
@@ -94,13 +94,13 @@ public abstract class PropertyMigrator extends Migrator {
}
g.has(OLD_FIELD).sideEffect(t -> {
final Vertex v = t.get();
- logger.info("Migrating property for vertex " + v.toString());
+ logger.debug("Migrating property for vertex " + v.toString());
final String value = v.value(OLD_FIELD);
v.property(OLD_FIELD).remove();
v.property(NEW_FIELD, value);
this.touchVertexProperties(v, false);
this.changedVertexCount += 1;
- logger.info(v.toString() + " : Migrated property " + OLD_FIELD + " to " + NEW_FIELD + " with value = " + value);
+ logger.debug(v.toString() + " : Migrated property " + OLD_FIELD + " to " + NEW_FIELD + " with value = " + value);
}).iterate();
}
@@ -121,11 +121,11 @@ public abstract class PropertyMigrator extends Migrator {
protected Optional<PropertyKey> addProperty() {
if (!graphMgmt.containsPropertyKey(this.NEW_FIELD)) {
- logger.info(" PropertyKey [" + this.NEW_FIELD + "] created in the DB. ");
+ logger.debug(" PropertyKey [" + this.NEW_FIELD + "] created in the DB. ");
return Optional.of(graphMgmt.makePropertyKey(this.NEW_FIELD).dataType(this.fieldType).cardinality(this.cardinality)
.make());
} else {
- logger.info(" PropertyKey [" + this.NEW_FIELD + "] already existed in the DB. ");
+ logger.debug(" PropertyKey [" + this.NEW_FIELD + "] already existed in the DB. ");
return Optional.empty();
}
@@ -136,7 +136,7 @@ public abstract class PropertyMigrator extends Migrator {
if (graphMgmt.containsGraphIndex(key.get().name())) {
logger.debug(" Index [" + key.get().name() + "] already existed in the DB. ");
} else {
- logger.info("Add index for PropertyKey: [" + key.get().name() + "]");
+ logger.debug("Add index for PropertyKey: [" + key.get().name() + "]");
graphMgmt.buildIndex(key.get().name(), Vertex.class).addKey(key.get()).buildCompositeIndex();
}
}
diff --git a/src/main/java/org/onap/aai/migration/RebuildAllEdges.java b/src/main/java/org/onap/aai/migration/RebuildAllEdges.java
index 73f5678..8bc2b8d 100644
--- a/src/main/java/org/onap/aai/migration/RebuildAllEdges.java
+++ b/src/main/java/org/onap/aai/migration/RebuildAllEdges.java
@@ -61,20 +61,20 @@ public class RebuildAllEdges extends EdgeMigrator {
@Override
protected void executeModifyOperation() {
Instant started = Instant.now();
- logger.info("Started at: " + started);
+ logger.debug("Started at: " + started);
GraphTraversalSource graphTraversalSource = engine.asAdmin().getTraversalSource();
Set<Edge> edges = graphTraversalSource.E().toSet();
rebuildEdges(edges);
Instant completed = Instant.now();
- logger.info("Completed at: " + completed + ". Total time taken in ms : "
+ logger.debug("Completed at: " + completed + ". Total time taken in ms : "
+ (completed.toEpochMilli() - started.toEpochMilli()));
- logger.info(MIGRATION_SUMMARY_COUNT + " Total Edges : " + edges.size() + " . Processed count " + processed
+ logger.debug(MIGRATION_SUMMARY_COUNT + " Total Edges : " + edges.size() + " . Processed count " + processed
+ " . Skipped count: " + skipped + ".");
- logger.info(MIGRATION_SUMMARY_COUNT + "Edge Missing Parent Property Count: "
+ logger.debug(MIGRATION_SUMMARY_COUNT + "Edge Missing Parent Property Count: "
+ edgeMissingParentProperty.size());
- logger.info(MIGRATION_ERROR + "Edge Multiplicity Exception Count : "
+ logger.debug(MIGRATION_ERROR + "Edge Multiplicity Exception Count : "
+ edgeMultiplicityExceptionCtr.values().stream().mapToInt(Number::intValue).sum());
- logger.info(MIGRATION_ERROR + "Edge Multiplicity Exception Breakdown : " + edgeMultiplicityExceptionCtr);
+ logger.debug(MIGRATION_ERROR + "Edge Multiplicity Exception Breakdown : " + edgeMultiplicityExceptionCtr);
}
@Override
diff --git a/src/main/java/org/onap/aai/migration/ValueMigrator.java b/src/main/java/org/onap/aai/migration/ValueMigrator.java
index 458796a..f45b20b 100644
--- a/src/main/java/org/onap/aai/migration/ValueMigrator.java
+++ b/src/main/java/org/onap/aai/migration/ValueMigrator.java
@@ -121,12 +121,12 @@ public abstract class ValueMigrator extends Migrator {
this.nodeTotalSuccess.put(nodeType, Integer.toString(this.subTotal));
}
- logger.info ("\n \n ******* Final Summary for " + " " + getMigrationName() +" ********* \n");
+ logger.debug ("\n \n ******* Final Summary for " + " " + getMigrationName() +" ********* \n");
for (Map.Entry<String, String> migratedNode: nodeTotalSuccess.entrySet()) {
- logger.info("Total Migrated Records for " + migratedNode.getKey() +": " + migratedNode.getValue());
+ logger.debug("Total Migrated Records for " + migratedNode.getKey() +": " + migratedNode.getValue());
}
- logger.info(this.MIGRATION_SUMMARY_COUNT + "Total Migrated Records: "+ migrationSuccess);
+ logger.debug(this.MIGRATION_SUMMARY_COUNT + "Total Migrated Records: "+ migrationSuccess);
}
@@ -136,18 +136,18 @@ public abstract class ValueMigrator extends Migrator {
String propertyValue = v.property(property).value().toString();
if (propertyValue.isEmpty()) {
v.property(property, newValue);
- logger.info(String.format("Node Type %s: Property %s is empty, adding value %s",
+ logger.debug(String.format("Node Type %s: Property %s is empty, adding value %s",
nodeType, property, newValue.toString()));
this.touchVertexProperties(v, false);
updateDmaapList(v);
this.migrationSuccess++;
this.subTotal++;
} else {
- logger.info(String.format("Node Type %s: Property %s value already exists - skipping",
+ logger.debug(String.format("Node Type %s: Property %s value already exists - skipping",
nodeType, property));
}
} else {
- logger.info(String.format("Node Type %s: Property %s does not exist or " +
+ logger.debug(String.format("Node Type %s: Property %s does not exist or " +
"updateExistingValues flag is set to True - adding the property with value %s",
nodeType, property, newValue.toString()));
v.property(property, newValue);
@@ -178,7 +178,7 @@ public abstract class ValueMigrator extends Migrator {
private void updateDmaapList(Vertex v){
String dmaapMsg = System.nanoTime() + "_" + v.id().toString() + "_" + v.value("resource-version").toString();
dmaapMsgList.add(dmaapMsg);
- logger.info("\tAdding Updated Vertex " + v.id().toString() + " to dmaapMsgList....");
+ logger.debug("\tAdding Updated Vertex " + v.id().toString() + " to dmaapMsgList....");
}
public boolean isUpdateDmaap(){
diff --git a/src/main/java/org/onap/aai/migration/VertexMerge.java b/src/main/java/org/onap/aai/migration/VertexMerge.java
index abf19be..a2c814e 100644
--- a/src/main/java/org/onap/aai/migration/VertexMerge.java
+++ b/src/main/java/org/onap/aai/migration/VertexMerge.java
@@ -41,8 +41,8 @@ import org.onap.aai.serialization.db.DBSerializer;
import org.onap.aai.edges.enums.EdgeType;
import org.onap.aai.serialization.db.EdgeSerializer;
import org.onap.aai.serialization.engines.TransactionalGraphEngine;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class recursively merges two vertices passed in.
@@ -52,7 +52,7 @@ import com.att.eelf.configuration.EELFManager;
*/
public class VertexMerge {
- private final EELFLogger logger = EELFManager.getInstance().getLogger(this.getClass().getSimpleName());
+ private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName());
private final GraphTraversalSource g;
private final TransactionalGraphEngine engine;
@@ -98,7 +98,7 @@ public class VertexMerge {
Collection<Vertex> primaryCousins = this.engine.getQueryEngine().findCousinVertices(primary);
secondaryCousins.removeAll(primaryCousins);
- logger.info("removing vertex after merge: " + secondary );
+ logger.debug("removing vertex after merge: " + secondary );
if (this.hasNotifications && secondarySnapshot.isPresent()) {
this.notificationHelper.addEvent(secondary, secondarySnapshot.get(), EventAction.DELETE, this.serializer.getURIForVertex(secondary, false), basePath);
}
diff --git a/src/main/java/org/onap/aai/migration/v16/MigrateBooleanDefaultsToFalse.java b/src/main/java/org/onap/aai/migration/v16/MigrateBooleanDefaultsToFalse.java
new file mode 100644
index 0000000..71c1999
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v16/MigrateBooleanDefaultsToFalse.java
@@ -0,0 +1,78 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v16;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Status;
+import org.onap.aai.migration.ValueMigrator;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+//@Enabled
+public class MigrateBooleanDefaultsToFalse extends ValueMigrator {
+ protected static final String CLOUD_REGION_NODE_TYPE = "cloud-region";
+
+ private static Map<String, Map> map;
+ private static Map<String, Boolean> pair1;
+
+
+ public MigrateBooleanDefaultsToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setBooleanDefaultsToFalse(), false);
+
+ }
+
+ private static Map<String, Map> setBooleanDefaultsToFalse(){
+ map = new HashMap<>();
+ pair1 = new HashMap<>();
+
+ pair1.put("orchestration-disabled", false);
+ map.put("cloud-region", pair1);
+
+ return map;
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{CLOUD_REGION_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateBooleanDefaultsToFalse";
+ }
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/migration/v16/MigrateInMaintDefaultToFalse.java b/src/main/java/org/onap/aai/migration/v16/MigrateInMaintDefaultToFalse.java
new file mode 100644
index 0000000..1a485b9
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v16/MigrateInMaintDefaultToFalse.java
@@ -0,0 +1,82 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v16;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Status;
+import org.onap.aai.migration.ValueMigrator;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+//@Enabled
+public class MigrateInMaintDefaultToFalse extends ValueMigrator {
+
+ protected static final String ZONE_NODE_TYPE = "zone";
+ protected static final String CLOUD_REGION_NODE_TYPE = "cloud-region";
+
+ private static Map<String, Map> map;
+ private static Map<String, Boolean> pair;
+
+ public MigrateInMaintDefaultToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setInMaintToFalse(), false);
+ }
+
+ private static Map<String, Map> setInMaintToFalse(){
+ map = new HashMap<>();
+ pair = new HashMap<>();
+
+ pair.put("in-maint", false);
+
+ map.put("zone", pair);
+ map.put("cloud-region", pair);
+
+
+ return map;
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{ZONE_NODE_TYPE,CLOUD_REGION_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateInMaintDefaultToFalse";
+ }
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/rest/QueryConsumer.java b/src/main/java/org/onap/aai/rest/QueryConsumer.java
index 85665da..b709cfa 100644
--- a/src/main/java/org/onap/aai/rest/QueryConsumer.java
+++ b/src/main/java/org/onap/aai/rest/QueryConsumer.java
@@ -19,19 +19,15 @@
*/
package org.onap.aai.rest;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.onap.aai.concurrent.AaiCallable;
-import org.onap.aai.dbmap.DBConnectionType;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.ModelType;
-import org.onap.aai.rest.dsl.DslQueryProcessor;
-import org.onap.aai.logging.LoggingContext;
import org.onap.aai.logging.StopWatch;
import org.onap.aai.rest.db.HttpEntry;
+import org.onap.aai.rest.dsl.DslQueryProcessor;
import org.onap.aai.rest.search.GenericQueryProcessor;
import org.onap.aai.rest.search.QueryProcessorType;
import org.onap.aai.restcore.HttpMethod;
@@ -45,6 +41,8 @@ import org.onap.aai.serialization.queryformats.SubGraphStyle;
import org.onap.aai.setup.SchemaVersion;
import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -54,7 +52,6 @@ import javax.ws.rs.*;
import javax.ws.rs.core.*;
import javax.ws.rs.core.Response.Status;
import java.util.List;
-import java.util.concurrent.TimeUnit;
@Component
@Path("{version: v[1-9][0-9]*|latest}/dbquery")
@@ -66,7 +63,7 @@ public class QueryConsumer extends RESTAPI {
private QueryProcessorType processorType = QueryProcessorType.LOCAL_GROOVY;
private static final String TARGET_ENTITY = "DB";
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(QueryConsumer.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(QueryConsumer.class);
private HttpEntry traversalUriHttpEntry;
@@ -111,15 +108,12 @@ public class QueryConsumer extends RESTAPI {
public Response processExecuteQuery(String content, @PathParam("version")String versionParam, @PathParam("uri") @Encoded String uri, @DefaultValue("graphson") @QueryParam("format") String queryFormat,@DefaultValue("no_op") @QueryParam("subgraph") String subgraph, @Context HttpHeaders headers, @Context UriInfo info, @Context HttpServletRequest req) {
- String methodName = "executeQuery";
String sourceOfTruth = headers.getRequestHeaders().getFirst("X-FromAppId");
- String realTime = headers.getRequestHeaders().getFirst("Real-Time");
String queryProcessor = headers.getRequestHeaders().getFirst("QueryProcessor");
QueryProcessorType processorType = this.processorType;
Response response = null;
TransactionalGraphEngine dbEngine = null;
try {
- LoggingContext.save();
this.checkQueryParams(info.getQueryParameters());
Format format = Format.getFormat(queryFormat);
if (queryProcessor != null) {
@@ -132,13 +126,11 @@ public class QueryConsumer extends RESTAPI {
JsonElement gremlinElement = input.get("gremlin");
JsonElement dslElement = input.get("dsl");
- String queryURI = "";
String gremlin = "";
String dsl = "";
SchemaVersion version = new SchemaVersion(versionParam);
- DBConnectionType type = this.determineConnectionType(sourceOfTruth, realTime);
- traversalUriHttpEntry.setHttpEntryProperties(version, type);
+ traversalUriHttpEntry.setHttpEntryProperties(version);
dbEngine = traversalUriHttpEntry.getDbEngine();
if (gremlinElement != null) {
@@ -147,11 +139,8 @@ public class QueryConsumer extends RESTAPI {
if (dslElement != null) {
dsl = dslElement.getAsString();
}
- GenericQueryProcessor processor = null;
+ GenericQueryProcessor processor;
- LoggingContext.targetEntity(TARGET_ENTITY);
- LoggingContext.targetServiceName(methodName);
- LoggingContext.startTime();
StopWatch.conditionalStart();
if(!dsl.equals("")){
@@ -175,9 +164,6 @@ public class QueryConsumer extends RESTAPI {
result = formater.output(vertices).toString();
- double msecs = StopWatch.stopIfStarted();
- LoggingContext.elapsedTime((long)msecs,TimeUnit.MILLISECONDS);
- LoggingContext.successStatusFields();
LOGGER.info ("Completed");
response = Response.status(Status.OK)
@@ -190,8 +176,6 @@ public class QueryConsumer extends RESTAPI {
AAIException ex = new AAIException("AAI_4000", e);
response = consumerExceptionResponseGenerator(headers, info, HttpMethod.GET, ex);
} finally {
- LoggingContext.restoreIfPossible();
- LoggingContext.successStatusFields();
if (dbEngine != null) {
dbEngine.rollback();
}
@@ -205,7 +189,7 @@ public class QueryConsumer extends RESTAPI {
if (params.containsKey("depth") && params.getFirst("depth").matches("\\d+")) {
String depth = params.getFirst("depth");
- Integer i = Integer.parseInt(depth);
+ int i = Integer.parseInt(depth);
if (i > 1) {
throw new AAIException("AAI_3303");
}
diff --git a/src/main/java/org/onap/aai/rest/dsl/DslListener.java b/src/main/java/org/onap/aai/rest/dsl/DslListener.java
index e41a946..70d8bf8 100644
--- a/src/main/java/org/onap/aai/rest/dsl/DslListener.java
+++ b/src/main/java/org/onap/aai/rest/dsl/DslListener.java
@@ -19,8 +19,8 @@
*/
package org.onap.aai.rest.dsl;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.onap.aai.AAIDslBaseListener;
import org.onap.aai.AAIDslParser;
@@ -40,7 +40,7 @@ import java.util.Map;
*/
public class DslListener extends AAIDslBaseListener {
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DslQueryProcessor.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(DslQueryProcessor.class);
private final EdgeIngestor edgeRules;
//TODO Use StringBuilder to build the query than concat
diff --git a/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java b/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java
index 582f0ea..3e77e6c 100644
--- a/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java
+++ b/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java
@@ -19,8 +19,8 @@
*/
package org.onap.aai.rest.dsl;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
@@ -38,7 +38,7 @@ import java.nio.charset.StandardCharsets;
*/
public class DslQueryProcessor {
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DslQueryProcessor.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(DslQueryProcessor.class);
private DslListener dslListener;
diff --git a/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java b/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java
index 8f83751..0964bc0 100644
--- a/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java
+++ b/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java
@@ -20,7 +20,10 @@
package org.onap.aai.rest.search;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.onap.aai.aailog.logs.AaiDBGraphadminMetricLog;
import org.onap.aai.restcore.search.GremlinGroovyShell;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.GraphAdminConstants;
import java.util.Map;
@@ -32,12 +35,18 @@ public class GroovyShellImpl extends GenericQueryProcessor {
@Override
protected GraphTraversal<?,?> runQuery(String query, Map<String, Object> params) {
-
+
+ AaiDBGraphadminMetricLog metricLog = new AaiDBGraphadminMetricLog (GraphAdminConstants.AAI_GRAPHADMIN_MS);
+ metricLog.pre(uri);
+
params.put("g", this.dbEngine.asAdmin().getTraversalSource());
GremlinGroovyShell shell = new GremlinGroovyShell();
- return shell.executeTraversal(query, params);
+ GraphTraversal<?,?> graphTraversal = shell.executeTraversal(query, params);
+
+ metricLog.post();
+ return graphTraversal;
}
}
diff --git a/src/main/java/org/onap/aai/schema/GenTester.java b/src/main/java/org/onap/aai/schema/GenTester.java
index 04ade0a..84ecf19 100644
--- a/src/main/java/org/onap/aai/schema/GenTester.java
+++ b/src/main/java/org/onap/aai/schema/GenTester.java
@@ -20,8 +20,8 @@
package org.onap.aai.schema;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.janusgraph.core.JanusGraph;
import org.janusgraph.core.schema.JanusGraphManagement;
import org.onap.aai.config.PropertyPasswordConfiguration;
@@ -29,8 +29,6 @@ import org.onap.aai.dbgen.SchemaGenerator;
import org.onap.aai.dbmap.AAIGraph;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.logging.LoggingContext.StatusCode;
import org.onap.aai.util.*;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
@@ -40,7 +38,8 @@ import java.util.UUID;
public class GenTester {
- private static EELFLogger LOGGER;
+ private static Logger LOGGER;
+ private static boolean historyEnabled;
/**
* The main method.
@@ -55,19 +54,9 @@ public class GenTester {
Properties props = System.getProperties();
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
- LOGGER = EELFManager.getInstance().getLogger(GenTester.class);
+ LOGGER = LoggerFactory.getLogger(GenTester.class);
boolean addDefaultCR = true;
- LoggingContext.init();
- LoggingContext.component("DBGenTester");
- LoggingContext.partnerName("AAI-TOOLS");
- LoggingContext.targetEntity("AAI");
- LoggingContext.requestId(UUID.randomUUID().toString());
- LoggingContext.serviceName("AAI");
- LoggingContext.targetServiceName("main");
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
-
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
initializer.initialize(ctx);
@@ -80,13 +69,18 @@ public class GenTester {
} catch (Exception e) {
AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
LOGGER.error("Problems running the tool "+aai.getMessage());
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
throw aai;
}
+ historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false"));
+ if( historyEnabled ) {
+ String amsg = "GenTester may only be used when history.enabled=false. ";
+ System.out.println(amsg);
+ LOGGER.debug(amsg);
+ return;
+ }
try {
- LOGGER.info("GenTester uses either cql jar or Cassandra jar");
+ LOGGER.debug("GenTester uses either cql jar or Cassandra jar");
AAIConfig.init();
if (args != null && args.length > 0 ){
@@ -100,7 +94,7 @@ public class GenTester {
// an HBase copyTable can be used to set up a copy of the db.
String imsg = " ---- NOTE --- about to load a graph without doing any schema processing (takes a little while) -------- ";
System.out.println(imsg);
- LOGGER.info(imsg);
+ LOGGER.debug(imsg);
graph = AAIGraph.getInstance().getGraph();
if( graph == null ){
@@ -110,7 +104,7 @@ public class GenTester {
else {
String amsg = "Successfully loaded a JanusGraph graph without doing any schema work. ";
System.out.println(amsg);
- LOGGER.auditEvent(amsg);
+ LOGGER.debug(amsg);
return;
}
} else if ("GEN_DB_WITH_NO_DEFAULT_CR".equals(args[0])) {
@@ -121,8 +115,6 @@ public class GenTester {
String emsg = "Unrecognized argument passed to GenTester.java: [" + args[0] + "]. ";
System.out.println(emsg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
LOGGER.error(emsg);
emsg = "Either pass no argument for normal processing, or use 'GEN_DB_WITH_NO_SCHEMA'.";
@@ -137,7 +129,7 @@ public class GenTester {
ErrorLogHelper.loadProperties();
String imsg = " ---- NOTE --- about to open graph (takes a little while)--------;";
System.out.println(imsg);
- LOGGER.info(imsg);
+ LOGGER.debug(imsg);
graph = AAIGraph.getInstance().getGraph();
if( graph == null ){
@@ -152,18 +144,18 @@ public class GenTester {
imsg = "-- Loading new schema elements into JanusGraph --";
System.out.println(imsg);
- LOGGER.info(imsg);
+ LOGGER.debug(imsg);
SchemaGenerator.loadSchemaIntoJanusGraph(graph, graphMgt, null);
if( graph != null ){
imsg = "-- graph commit";
System.out.println(imsg);
- LOGGER.info(imsg);
+ LOGGER.debug(imsg);
graph.tx().commit();
imsg = "-- graph shutdown ";
System.out.println(imsg);
- LOGGER.info(imsg);
+ LOGGER.debug(imsg);
graph.close();
}
@@ -174,11 +166,10 @@ public class GenTester {
- LOGGER.auditEvent("-- all done, if program does not exit, please kill.");
+ LOGGER.debug("-- all done, if program does not exit, please kill.");
System.exit(0);
}
-}
-
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/schema/GenTester4Hist.java b/src/main/java/org/onap/aai/schema/GenTester4Hist.java
new file mode 100644
index 0000000..eefb7b0
--- /dev/null
+++ b/src/main/java/org/onap/aai/schema/GenTester4Hist.java
@@ -0,0 +1,175 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.schema;
+
+import com.att.eelf.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.onap.aai.config.PropertyPasswordConfiguration;
+import org.onap.aai.dbgen.SchemaGenerator4Hist;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.util.*;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import java.util.Properties;
+import java.util.UUID;
+
+
+public class GenTester4Hist {
+
+ private static Logger LOGGER;
+ private static boolean historyEnabled;
+
+ /**
+ * The main method.
+ *
+ * @param args the arguments
+ */
+ public static void main(String[] args) throws AAIException{
+
+ JanusGraph graph = null;
+ System.setProperty("aai.service.name", GenTester4Hist.class.getSimpleName());
+ // Set the logging file properties to be used by EELFManager
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+ LOGGER = LoggerFactory.getLogger(GenTester4Hist.class);
+ boolean addDefaultCR = false; // For History, we do not add the default CloudRegion
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
+ PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
+ initializer.initialize(ctx);
+ try {
+ ctx.scan(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+ ctx.refresh();
+ } catch (Exception e) {
+ AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
+ LOGGER.error("Problems running the tool "+aai.getMessage());
+ ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
+ throw aai;
+ }
+
+ historyEnabled = Boolean.parseBoolean(ctx.getEnvironment().getProperty("history.enabled","false"));
+ if( !historyEnabled ) {
+ String amsg = "GenTester4Hist may only be used when history.enabled=true. ";
+ System.out.println(amsg);
+ LOGGER.debug(amsg);
+ return;
+ }
+
+ try {
+ LOGGER.debug("GenTester4Hist uses either cql jar or Cassandra jar");
+
+ AAIConfig.init();
+ if (args != null && args.length > 0 ){
+ if( "genDbRulesOnly".equals(args[0]) ){
+ ErrorLogHelper.logError("AAI_3100",
+ " This option is no longer supported. What was in DbRules is now derived from the OXM files. ");
+ return;
+ }
+ else if ( "GEN_DB_WITH_NO_SCHEMA".equals(args[0]) ){
+ // Note this is done to create an empty DB with no Schema so that
+ // an HBase copyTable can be used to set up a copy of the db.
+ String imsg = " ---- NOTE --- about to load a graph without doing any schema processing (takes a little while) -------- ";
+ System.out.println(imsg);
+ LOGGER.debug(imsg);
+ graph = AAIGraph.getInstance().getGraph();
+
+ if( graph == null ){
+ ErrorLogHelper.logError("AAI_5102", "Error creating JanusGraph graph.");
+ return;
+ }
+ else {
+ String amsg = "Successfully loaded a JanusGraph graph without doing any schema work. ";
+ System.out.println(amsg);
+ LOGGER.debug(amsg);
+ return;
+ }
+ } else if ("GEN_DB_WITH_NO_DEFAULT_CR".equals(args[0])) {
+ addDefaultCR = false;
+ }
+ else {
+ ErrorLogHelper.logError("AAI_3000", "Unrecognized argument passed to GenTester4Hist.java: [" + args[0] + "]. ");
+
+ String emsg = "Unrecognized argument passed to GenTester4Hist.java: [" + args[0] + "]. ";
+ System.out.println(emsg);
+ LOGGER.error(emsg);
+
+ emsg = "Either pass no argument for normal processing, or use 'GEN_DB_WITH_NO_SCHEMA'.";
+ System.out.println(emsg);
+ LOGGER.error(emsg);
+
+ return;
+ }
+ }
+
+ //AAIConfig.init();
+ ErrorLogHelper.loadProperties();
+ String imsg = " ---- NOTE --- about to open graph (takes a little while)--------;";
+ System.out.println(imsg);
+ LOGGER.debug(imsg);
+ graph = AAIGraph.getInstance().getGraph();
+
+ if( graph == null ){
+ ErrorLogHelper.logError("AAI_5102", "Error creating JanusGraph graph. ");
+ return;
+ }
+
+ GraphAdminDBUtils.logConfigs(graph.configuration());
+
+ // Load the propertyKeys, indexes and edge-Labels into the DB
+ JanusGraphManagement graphMgt = graph.openManagement();
+
+ imsg = "-- Loading new schema elements into JanusGraph --";
+ System.out.println(imsg);
+ LOGGER.debug(imsg);
+ SchemaGenerator4Hist.loadSchemaIntoJanusGraph(graph, graphMgt, null);
+
+ if( graph != null ){
+ imsg = "-- graph commit";
+ System.out.println(imsg);
+ LOGGER.debug(imsg);
+ graph.tx().commit();
+
+ imsg = "-- graph shutdown ";
+ System.out.println(imsg);
+ LOGGER.debug(imsg);
+ graph.close();
+ }
+
+ } catch(Exception ex) {
+ ErrorLogHelper.logError("AAI_4000", ex.getMessage());
+ System.exit(1);
+ }
+
+ LOGGER.debug("-- all done, if program does not exit, please kill.");
+ System.exit(0);
+ }
+
+
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/service/AuthorizationService.java b/src/main/java/org/onap/aai/service/AuthorizationService.java
index d2597d0..9b5a614 100644
--- a/src/main/java/org/onap/aai/service/AuthorizationService.java
+++ b/src/main/java/org/onap/aai/service/AuthorizationService.java
@@ -19,8 +19,8 @@
*/
package org.onap.aai.service;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.eclipse.jetty.util.security.Password;
import org.onap.aai.Profiles;
import org.onap.aai.util.AAIConstants;
@@ -41,7 +41,7 @@ import java.util.stream.Stream;
@Service
public class AuthorizationService {
- private static final EELFLogger logger = EELFManager.getInstance().getLogger(AuthorizationService.class);
+ private static final Logger logger = LoggerFactory.getLogger(AuthorizationService.class);
private final Map<String, String> authorizedUsers = new HashMap<>();
@@ -72,7 +72,7 @@ public class AuthorizationService {
String[] usernamePasswordArray = usernamePassword.split(":");
if(usernamePasswordArray == null || usernamePasswordArray.length != 3){
- throw new RuntimeException("Not a valid entry for the realm.properties entry: " + usernamePassword);
+ throw new RuntimeException("This username / pwd is not a valid entry in realm.properties");
}
String username = usernamePasswordArray[0];
diff --git a/src/main/java/org/onap/aai/util/ExceptionTranslator.java b/src/main/java/org/onap/aai/util/ExceptionTranslator.java
index a395c23..17c5667 100644
--- a/src/main/java/org/onap/aai/util/ExceptionTranslator.java
+++ b/src/main/java/org/onap/aai/util/ExceptionTranslator.java
@@ -19,27 +19,33 @@
*/
package org.onap.aai.util;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.onap.aai.GraphAdminApp;
import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.LogFormatTools;
public class ExceptionTranslator {
- private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(ExceptionTranslator.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(ExceptionTranslator.class);
public static AAIException schemaServiceExceptionTranslator(Exception ex) {
AAIException aai = null;
- LOGGER.info("Exception is " + ExceptionUtils.getRootCause(ex).getMessage() + "Root cause is"+ ExceptionUtils.getRootCause(ex).toString());
- if(ExceptionUtils.getRootCause(ex).getMessage().contains("NodeIngestor")){
- aai = new AAIException("AAI_3026","Error reading OXM from SchemaService - Investigate");
- }
- else if(ExceptionUtils.getRootCause(ex).getMessage().contains("EdgeIngestor")){
- aai = new AAIException("AAI_3027","Error reading EdgeRules from SchemaService - Investigate");
- }
- else if(ExceptionUtils.getRootCause(ex).getMessage().contains("Connection refused")){
- aai = new AAIException("AAI_3025","Error connecting to SchemaService - Investigate");
- }else {
- aai = new AAIException("AAI_3025","Error connecting to SchemaService - Please Investigate");
+ if ( ExceptionUtils.getRootCause(ex) == null || ExceptionUtils.getRootCause(ex).getMessage() == null ) {
+ aai = new AAIException("AAI_3025","Error parsing exception - Please Investigate" +
+ LogFormatTools.getStackTop(ex));
+ } else {
+ LOGGER.info("Exception is " + ExceptionUtils.getRootCause(ex).getMessage() + "Root cause is"+ ExceptionUtils.getRootCause(ex).toString());
+ if(ExceptionUtils.getRootCause(ex).getMessage().contains("NodeIngestor")){
+ aai = new AAIException("AAI_3026","Error reading OXM from SchemaService - Investigate");
+ }
+ else if(ExceptionUtils.getRootCause(ex).getMessage().contains("EdgeIngestor")){
+ aai = new AAIException("AAI_3027","Error reading EdgeRules from SchemaService - Investigate");
+ }
+ else if(ExceptionUtils.getRootCause(ex).getMessage().contains("Connection refused")){
+ aai = new AAIException("AAI_3025","Error connecting to SchemaService - Investigate");
+ }else {
+ aai = new AAIException("AAI_3025","Error connecting to SchemaService - Please Investigate");
+ }
}
return aai;
diff --git a/src/main/java/org/onap/aai/util/GraphAdminConstants.java b/src/main/java/org/onap/aai/util/GraphAdminConstants.java
index 017d92e..62ba593 100644
--- a/src/main/java/org/onap/aai/util/GraphAdminConstants.java
+++ b/src/main/java/org/onap/aai/util/GraphAdminConstants.java
@@ -22,6 +22,7 @@ package org.onap.aai.util;
public final class GraphAdminConstants {
public static final int AAI_SNAPSHOT_DEFAULT_THREADS_FOR_CREATE = 15;
+ public static final Long AAI_SNAPSHOT_DEFAULT_MAX_NODES_PER_FILE_FOR_CREATE = 120000L;
public static final int AAI_SNAPSHOT_DEFAULT_MAX_ERRORS_PER_THREAD = 25;
public static final Long AAI_SNAPSHOT_DEFAULT_VERTEX_ADD_DELAY_MS = 1L;
public static final Long AAI_SNAPSHOT_DEFAULT_EDGE_ADD_DELAY_MS = 1L;
@@ -36,7 +37,8 @@ public final class GraphAdminConstants {
public static final int AAI_DUPETOOL_DEFAULT_MAX_FIX = 25;
public static final int AAI_DUPETOOL_DEFAULT_SLEEP_MINUTES = 7;
-
+ /** Micro-service Names */
+ public static final String AAI_GRAPHADMIN_MS = "aai-graphadmin";
/**
diff --git a/src/main/java/org/onap/aai/util/GraphAdminDBUtils.java b/src/main/java/org/onap/aai/util/GraphAdminDBUtils.java
index 992223e..202bc0a 100644
--- a/src/main/java/org/onap/aai/util/GraphAdminDBUtils.java
+++ b/src/main/java/org/onap/aai/util/GraphAdminDBUtils.java
@@ -21,12 +21,12 @@ package org.onap.aai.util;
import java.util.Iterator;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class GraphAdminDBUtils {
- private static EELFLogger LOGGER = EELFManager.getInstance().getLogger(GraphAdminDBUtils.class);
+ private static Logger LOGGER = LoggerFactory.getLogger(GraphAdminDBUtils.class);
public static void logConfigs(org.apache.commons.configuration.Configuration configuration) {
diff --git a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java
index d9615b0..36d01e1 100644
--- a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java
+++ b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java
@@ -20,24 +20,22 @@
package org.onap.aai.util;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.onap.aai.db.props.AAIProperties;
import org.onap.aai.dbmap.AAIGraph;
-import org.onap.aai.dbmap.DBConnectionType;
import org.onap.aai.exceptions.AAIException;
-import org.onap.aai.introspection.*;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
import org.onap.aai.migration.EventAction;
import org.onap.aai.migration.NotificationHelper;
-import org.onap.aai.rest.ueb.UEBNotification;
import org.onap.aai.serialization.db.DBSerializer;
-import org.onap.aai.serialization.engines.QueryStyle;
import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.QueryStyle;
import org.onap.aai.serialization.engines.TransactionalGraphEngine;
-import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import java.io.IOException;
@@ -47,11 +45,9 @@ import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
-import javax.ws.rs.core.Response.Status;
-
public class SendDeleteMigrationNotifications {
- protected EELFLogger logger = EELFManager.getInstance().getLogger(SendDeleteMigrationNotifications.class.getSimpleName());
+ protected Logger logger = LoggerFactory.getLogger(SendDeleteMigrationNotifications.class.getSimpleName());
private String config;
private String path;
@@ -104,7 +100,7 @@ public class SendDeleteMigrationNotifications {
Map<Integer, String> deleteDataMap = processFile();
int count = 0;
for (Map.Entry<Integer, String> entry : deleteDataMap.entrySet()) {
- logger.info("Processing " + entry.getKey() + " :: Data :: " + entry.getValue());
+ logger.debug("Processing " + entry.getKey() + " :: Data :: " + entry.getValue());
String data = entry.getValue();
Introspector obj = null;
if (data.contains("#@#")) {
@@ -118,7 +114,7 @@ public class SendDeleteMigrationNotifications {
count++;
if (count >= this.numToBatch) {
trigger();
- logger.info("Triggered " + entry.getKey());
+ logger.debug("Triggered " + entry.getKey());
count = 0;
Thread.sleep(this.sleepInMilliSecs);
}
@@ -158,7 +154,7 @@ public class SendDeleteMigrationNotifications {
private void initFields() {
this.loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
- this.engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
+ this.engine = new JanusGraphDBEngine(queryStyle, loader);
try {
this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.eventSource);
} catch (AAIException e) {
@@ -176,7 +172,7 @@ public class SendDeleteMigrationNotifications {
protected void logAndPrint(String msg) {
System.out.println(msg);
- logger.info(msg);
+ logger.debug(msg);
}
diff --git a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java
index a94b435..0fbe520 100644
--- a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java
+++ b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java
@@ -26,7 +26,6 @@ import org.onap.aai.dbmap.AAIGraph;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.logging.LoggingContext;
import org.onap.aai.migration.EventAction;
import org.onap.aai.setup.SchemaVersions;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
@@ -40,15 +39,6 @@ public class SendDeleteMigrationNotificationsMain {
Arrays.asList(args).stream().forEach(System.out::println);
String requestId = UUID.randomUUID().toString();
- LoggingContext.init();
- LoggingContext.partnerName("Migration");
- LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.component("SendMigrationNotifications");
- LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.targetServiceName("main");
- LoggingContext.requestId(requestId);
- LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
@@ -61,8 +51,6 @@ public class SendDeleteMigrationNotificationsMain {
ctx.refresh();
} catch (Exception e) {
AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
throw aai;
}
diff --git a/src/main/java/org/onap/aai/util/SendMigrationNotifications.java b/src/main/java/org/onap/aai/util/SendMigrationNotifications.java
index 577f577..c51de1e 100644
--- a/src/main/java/org/onap/aai/util/SendMigrationNotifications.java
+++ b/src/main/java/org/onap/aai/util/SendMigrationNotifications.java
@@ -20,23 +20,25 @@
package org.onap.aai.util;
import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.onap.aai.db.props.AAIProperties;
import org.onap.aai.dbmap.AAIGraph;
-import org.onap.aai.dbmap.DBConnectionType;
import org.onap.aai.exceptions.AAIException;
-import org.onap.aai.introspection.*;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
import org.onap.aai.migration.EventAction;
import org.onap.aai.migration.NotificationHelper;
import org.onap.aai.serialization.db.DBSerializer;
-import org.onap.aai.serialization.engines.QueryStyle;
import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.QueryStyle;
import org.onap.aai.serialization.engines.TransactionalGraphEngine;
-import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import java.io.IOException;
@@ -47,7 +49,7 @@ import java.util.*;
public class SendMigrationNotifications {
- protected EELFLogger logger = EELFManager.getInstance().getLogger(SendMigrationNotifications.class.getSimpleName());
+ protected Logger logger = LoggerFactory.getLogger(SendMigrationNotifications.class.getSimpleName());
private String config;
private String path;
@@ -111,7 +113,7 @@ public class SendMigrationNotifications {
logAndPrint("Vertex " + entry.getKey() + " query returned " + vertexes.size() + " vertexes." );
continue;
} else {
- logger.info("Processing " + entry.getKey() + "resource-version " + entry.getValue());
+ logger.debug("Processing " + entry.getKey() + "resource-version " + entry.getValue());
v = vertexes.get(0);
if (notifyOn.isEmpty() || notifyOn.contains(v.value(AAIProperties.NODE_TYPE).toString())) {
if (entry.getValue().equals(v.value(AAIProperties.RESOURCE_VERSION).toString())) {
@@ -121,7 +123,7 @@ public class SendMigrationNotifications {
count++;
if (count >= this.numToBatch) {
trigger();
- logger.info("Triggered " + entry.getKey());
+ logger.debug("Triggered " + entry.getKey());
count = 0;
Thread.sleep(this.sleepInMilliSecs);
}
@@ -164,7 +166,7 @@ public class SendMigrationNotifications {
private void initFields() {
this.loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
- this.engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
+ this.engine = new JanusGraphDBEngine(queryStyle, loader);
try {
this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.eventSource);
} catch (AAIException e) {
@@ -182,7 +184,7 @@ public class SendMigrationNotifications {
protected void logAndPrint(String msg) {
System.out.println(msg);
- logger.info(msg);
+ logger.debug(msg);
}
diff --git a/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java b/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java
index 17a127a..d3670f2 100644
--- a/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java
+++ b/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java
@@ -26,7 +26,6 @@ import org.onap.aai.dbmap.AAIGraph;
import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.logging.LoggingContext;
import org.onap.aai.migration.EventAction;
import org.onap.aai.setup.SchemaVersions;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
@@ -40,16 +39,7 @@ public class SendMigrationNotificationsMain {
Arrays.asList(args).stream().forEach(System.out::println);
String requestId = UUID.randomUUID().toString();
- LoggingContext.init();
- LoggingContext.partnerName("Migration");
- LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.component("SendMigrationNotifications");
- LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
- LoggingContext.targetServiceName("main");
- LoggingContext.requestId(requestId);
- LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
-
+
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
PropertyPasswordConfiguration initializer = new PropertyPasswordConfiguration();
initializer.initialize(ctx);
@@ -62,8 +52,6 @@ public class SendMigrationNotificationsMain {
} catch (Exception e) {
AAIException aai = ExceptionTranslator.schemaServiceExceptionTranslator(e);
System.out.println("Problems running tool "+aai.getMessage());
- LoggingContext.statusCode(LoggingContext.StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
ErrorLogHelper.logError(aai.getCode(), e.getMessage() + ", resolve and retry");
throw aai;
}
diff --git a/src/main/java/org/onap/aai/util/UniquePropertyCheck.java b/src/main/java/org/onap/aai/util/UniquePropertyCheck.java
index e96c252..2db3dd5 100644
--- a/src/main/java/org/onap/aai/util/UniquePropertyCheck.java
+++ b/src/main/java/org/onap/aai/util/UniquePropertyCheck.java
@@ -1,288 +1,277 @@
-/**
- * ============LICENSE_START=======================================================
- * org.onap.aai
- * ================================================================================
- * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END=========================================================
- */
-package org.onap.aai.util;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.onap.aai.GraphAdminApp;
-import org.onap.aai.exceptions.AAIException;
-import org.onap.aai.logging.LoggingContext;
-import org.onap.aai.logging.LoggingContext.StatusCode;
-import org.slf4j.MDC;
-
-import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-
-import org.janusgraph.core.JanusGraphFactory;
-import org.janusgraph.core.JanusGraph;
-import org.onap.aai.dbmap.AAIGraphConfig;
-
-public class UniquePropertyCheck {
-
-
- private static final String FROMAPPID = "AAI-UTILS";
- private static final String TRANSID = UUID.randomUUID().toString();
- private static final String COMPONENT = "UniquePropertyCheck";
-
- /**
- * The main method.
- *
- * @param args the arguments
- */
- public static void main(String[] args) {
-
-
- Properties props = System.getProperties();
- props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
- props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
- EELFLogger logger = EELFManager.getInstance().getLogger(UniquePropertyCheck.class.getSimpleName());
-
- LoggingContext.init();
- LoggingContext.partnerName(FROMAPPID);
- LoggingContext.serviceName(GraphAdminApp.APP_NAME);
- LoggingContext.component(COMPONENT);
- LoggingContext.targetEntity(GraphAdminApp.APP_NAME);
- LoggingContext.targetServiceName("main");
- LoggingContext.requestId(TRANSID);
- LoggingContext.statusCode(StatusCode.COMPLETE);
- LoggingContext.responseCode(LoggingContext.SUCCESS);
-
- MDC.put("logFilenameAppender", UniquePropertyCheck.class.getSimpleName());
-
- if( args == null || args.length != 1 ){
- String msg = "usage: UniquePropertyCheck propertyName \n";
- System.out.println(msg);
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
- logAndPrint(logger, msg );
- System.exit(1);
- }
- String propertyName = args[0];
- Graph graph = null;
-
- try {
- AAIConfig.init();
- System.out.println(" ---- NOTE --- about to open graph (takes a little while)--------\n");
- JanusGraph tGraph = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(UniquePropertyCheck.class.getSimpleName()).withGraphType("realtime").buildConfiguration());
-
- if( tGraph == null ) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
- logAndPrint(logger, " Error: Could not get JanusGraph ");
- System.exit(1);
- }
-
- graph = tGraph.newTransaction();
- if( graph == null ){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
- logAndPrint(logger, "could not get graph object in UniquePropertyCheck() \n");
- System.exit(0);
- }
- }
- catch (AAIException e1) {
- String msg = "Threw Exception: [" + e1.toString() + "]";
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
- logAndPrint(logger, msg);
- System.exit(0);
- }
- catch (Exception e2) {
- String msg = "Threw Exception: [" + e2.toString() + "]";
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
- logAndPrint(logger, msg);
- System.exit(0);
- }
-
- runTheCheckForUniqueness( TRANSID, FROMAPPID, graph, propertyName, logger );
- System.exit(0);
-
- }// End main()
-
-
- /**
- * Run the check for uniqueness.
- *
- * @param transId the trans id
- * @param fromAppId the from app id
- * @param graph the graph
- * @param propertyName the property name
- * @param logger the logger
- * @return the boolean
- */
- public static Boolean runTheCheckForUniqueness( String transId, String fromAppId, Graph graph,
- String propertyName, EELFLogger logger ){
-
- // Note - property can be found in more than one nodetype
- // our uniqueness constraints are always across the entire db - so this
- // tool looks across all nodeTypes that the property is found in.
- Boolean foundDupesFlag = false;
-
- HashMap <String,String> valuesAndVidHash = new HashMap <String, String> ();
- HashMap <String,String> dupeHash = new HashMap <String, String> ();
-
- int propCount = 0;
- int dupeCount = 0;
- Iterator<Vertex> vertItor = graph.traversal().V().has(propertyName);
- while( vertItor.hasNext() ){
- propCount++;
- Vertex v = vertItor.next();
- String thisVid = v.id().toString();
- Object val = (v.<Object>property(propertyName)).orElse(null);
- if( valuesAndVidHash.containsKey(val) ){
- // We've seen this one before- track it in our dupe hash
- dupeCount++;
- if( dupeHash.containsKey(val) ){
- // This is not the first one being added to the dupe hash for this value
- String updatedDupeList = dupeHash.get(val) + "|" + thisVid;
- dupeHash.put(val.toString(), updatedDupeList);
- }
- else {
- // This is the first time we see this value repeating
- String firstTwoVids = valuesAndVidHash.get(val) + "|" + thisVid;
- dupeHash.put(val.toString(), firstTwoVids);
- }
- }
- else {
- valuesAndVidHash.put(val.toString(), thisVid);
- }
- }
-
-
- String info = "\n Found this property [" + propertyName + "] " + propCount + " times in our db.";
- logAndPrint(logger, info);
- info = " Found " + dupeCount + " cases of duplicate values for this property.\n\n";
- logAndPrint(logger, info);
-
- try {
- if( ! dupeHash.isEmpty() ){
- Iterator <?> dupeItr = dupeHash.entrySet().iterator();
- while( dupeItr.hasNext() ){
- foundDupesFlag = true;
- Map.Entry pair = (Map.Entry) dupeItr.next();
- String dupeValue = pair.getKey().toString();;
- String vidsStr = pair.getValue().toString();
- String[] vidArr = vidsStr.split("\\|");
- logAndPrint(logger, "\n\n -------------- Found " + vidArr.length
- + " nodes with " + propertyName + " of this value: [" + dupeValue + "]. Node details: ");
-
- for( int i = 0; i < vidArr.length; i++ ){
- String vidString = vidArr[i];
- Long idLong = Long.valueOf(vidString);
- Vertex tvx = graph.traversal().V(idLong).next();
- showPropertiesAndEdges( TRANSID, FROMAPPID, tvx, logger );
- }
- }
- }
- }
- catch( Exception e2 ){
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- logAndPrint(logger, "Threw Exception: [" + e2.toString() + "]");
- }
-
-
- return foundDupesFlag;
-
- }// end of runTheCheckForUniqueness()
-
-
- /**
- * Show properties and edges.
- *
- * @param transId the trans id
- * @param fromAppId the from app id
- * @param tVert the t vert
- * @param logger the logger
- */
- private static void showPropertiesAndEdges( String transId, String fromAppId, Vertex tVert,
- EELFLogger logger ){
-
- if( tVert == null ){
- logAndPrint(logger, "Null node passed to showPropertiesAndEdges.");
- }
- else {
- String nodeType = "";
- Object ob = tVert.<String>property("aai-node-type").orElse(null);
- if( ob == null ){
- nodeType = "null";
- }
- else{
- nodeType = ob.toString();
- }
-
- logAndPrint(logger, " AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
- logAndPrint(logger, " Property Detail: ");
- Iterator<VertexProperty<Object>> pI = tVert.properties();
- while( pI.hasNext() ){
- VertexProperty<Object> tp = pI.next();
- Object val = tp.value();
- logAndPrint(logger, "Prop: [" + tp.key() + "], val = [" + val + "] ");
- }
-
- Iterator <Edge> eI = tVert.edges(Direction.BOTH);
- if( ! eI.hasNext() ){
- logAndPrint(logger, "No edges were found for this vertex. ");
- }
- while( eI.hasNext() ){
- Edge ed = eI.next();
- String lab = ed.label();
- Vertex vtx;
- if (tVert.equals(ed.inVertex())) {
- vtx = ed.outVertex();
- } else {
- vtx = ed.inVertex();
- }
- if( vtx == null ){
- logAndPrint(logger, " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
- }
- else {
- String nType = vtx.<String>property("aai-node-type").orElse(null);
- String vid = vtx.id().toString();
- logAndPrint(logger, "Found an edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid);
- }
- }
- }
- } // End of showPropertiesAndEdges()
-
-
- /**
- * Log and print.
- *
- * @param logger the logger
- * @param msg the msg
- */
- protected static void logAndPrint(EELFLogger logger, String msg) {
- System.out.println(msg);
- logger.info(msg);
- }
-
-}
-
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.util;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.onap.aai.aailog.logs.AaiScheduledTaskAuditLog;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.logging.filter.base.ONAPComponents;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import com.att.eelf.configuration.Configuration;
+
+public class UniquePropertyCheck {
+
+
+ private static final String FROMAPPID = "AAI-UTILS";
+ private static final String TRANSID = UUID.randomUUID().toString();
+ private static final String COMPONENT = "UniquePropertyCheck";
+
+ /**
+ * The main method.
+ *
+ * @param args the arguments
+ */
+ public static void main(String[] args) {
+
+
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+ Logger logger = LoggerFactory.getLogger(UniquePropertyCheck.class);
+ MDC.put("logFilenameAppender", UniquePropertyCheck.class.getSimpleName());
+ AaiScheduledTaskAuditLog auditLog = new AaiScheduledTaskAuditLog();
+ auditLog.logBefore("UniquePropertyCheck", ONAPComponents.AAI.toString());
+
+ if( args == null || args.length != 1 ){
+ String msg = "usage: UniquePropertyCheck propertyName \n";
+ System.out.println(msg);
+ //LoggingContext.statusCode(StatusCode.ERROR);
+ //LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logAndPrint(logger, msg );
+ System.exit(1);
+ }
+ String propertyName = args[0];
+ Graph graph = null;
+
+ try {
+ AAIConfig.init();
+ System.out.println(" ---- NOTE --- about to open graph (takes a little while)--------\n");
+ JanusGraph tGraph = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(UniquePropertyCheck.class.getSimpleName()).withGraphType("realtime").buildConfiguration());
+
+ if( tGraph == null ) {
+ //LoggingContext.statusCode(StatusCode.ERROR);
+ //LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logAndPrint(logger, " Error: Could not get JanusGraph ");
+ System.exit(1);
+ }
+
+ graph = tGraph.newTransaction();
+ if( graph == null ){
+ //LoggingContext.statusCode(StatusCode.ERROR);
+ //LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logAndPrint(logger, "could not get graph object in UniquePropertyCheck() \n");
+ System.exit(0);
+ }
+ }
+ catch (AAIException e1) {
+ String msg = "Threw Exception: [" + e1.toString() + "]";
+ //LoggingContext.statusCode(StatusCode.ERROR);
+ //LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
+ logAndPrint(logger, msg);
+ System.exit(0);
+ }
+ catch (Exception e2) {
+ String msg = "Threw Exception: [" + e2.toString() + "]";
+ //LoggingContext.statusCode(StatusCode.ERROR);
+ //LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
+ logAndPrint(logger, msg);
+ System.exit(0);
+ }
+
+ runTheCheckForUniqueness( TRANSID, FROMAPPID, graph, propertyName, logger );
+ auditLog.logAfter();
+ System.exit(0);
+
+ }// End main()
+
+
+ /**
+ * Run the check for uniqueness.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param graph the graph
+ * @param propertyName the property name
+ * @param logger the logger
+ * @return the boolean
+ */
+ public static Boolean runTheCheckForUniqueness( String transId, String fromAppId, Graph graph,
+ String propertyName, Logger logger ){
+
+ // Note - property can be found in more than one nodetype
+ // our uniqueness constraints are always across the entire db - so this
+ // tool looks across all nodeTypes that the property is found in.
+ Boolean foundDupesFlag = false;
+
+ HashMap <String,String> valuesAndVidHash = new HashMap <String, String> ();
+ HashMap <String,String> dupeHash = new HashMap <String, String> ();
+
+ int propCount = 0;
+ int dupeCount = 0;
+ Iterator<Vertex> vertItor = graph.traversal().V().has(propertyName);
+ while( vertItor.hasNext() ){
+ propCount++;
+ Vertex v = vertItor.next();
+ String thisVid = v.id().toString();
+ Object val = (v.<Object>property(propertyName)).orElse(null);
+ if( valuesAndVidHash.containsKey(val) ){
+ // We've seen this one before- track it in our dupe hash
+ dupeCount++;
+ if( dupeHash.containsKey(val) ){
+ // This is not the first one being added to the dupe hash for this value
+ String updatedDupeList = dupeHash.get(val) + "|" + thisVid;
+ dupeHash.put(val.toString(), updatedDupeList);
+ }
+ else {
+ // This is the first time we see this value repeating
+ String firstTwoVids = valuesAndVidHash.get(val) + "|" + thisVid;
+ dupeHash.put(val.toString(), firstTwoVids);
+ }
+ }
+ else {
+ valuesAndVidHash.put(val.toString(), thisVid);
+ }
+ }
+
+
+ String info = "\n Found this property [" + propertyName + "] " + propCount + " times in our db.";
+ logAndPrint(logger, info);
+ info = " Found " + dupeCount + " cases of duplicate values for this property.\n\n";
+ logAndPrint(logger, info);
+
+ try {
+ if( ! dupeHash.isEmpty() ){
+ Iterator <?> dupeItr = dupeHash.entrySet().iterator();
+ while( dupeItr.hasNext() ){
+ foundDupesFlag = true;
+ Map.Entry pair = (Map.Entry) dupeItr.next();
+ String dupeValue = pair.getKey().toString();;
+ String vidsStr = pair.getValue().toString();
+ String[] vidArr = vidsStr.split("\\|");
+ logAndPrint(logger, "\n\n -------------- Found " + vidArr.length
+ + " nodes with " + propertyName + " of this value: [" + dupeValue + "]. Node details: ");
+
+ for( int i = 0; i < vidArr.length; i++ ){
+ String vidString = vidArr[i];
+ Long idLong = Long.valueOf(vidString);
+ Vertex tvx = graph.traversal().V(idLong).next();
+ showPropertiesAndEdges( TRANSID, FROMAPPID, tvx, logger );
+ }
+ }
+ }
+ }
+ catch( Exception e2 ){
+ //LoggingContext.statusCode(StatusCode.ERROR);
+ //LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint(logger, "Threw Exception: [" + e2.toString() + "]");
+ }
+
+
+ return foundDupesFlag;
+
+ }// end of runTheCheckForUniqueness()
+
+
+ /**
+ * Show properties and edges.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param tVert the t vert
+ * @param logger the logger
+ */
+ private static void showPropertiesAndEdges( String transId, String fromAppId, Vertex tVert,
+ Logger logger ){
+
+ if( tVert == null ){
+ logAndPrint(logger, "Null node passed to showPropertiesAndEdges.");
+ }
+ else {
+ String nodeType = "";
+ Object ob = tVert.<String>property("aai-node-type").orElse(null);
+ if( ob == null ){
+ nodeType = "null";
+ }
+ else{
+ nodeType = ob.toString();
+ }
+
+ logAndPrint(logger, " AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
+ logAndPrint(logger, " Property Detail: ");
+ Iterator<VertexProperty<Object>> pI = tVert.properties();
+ while( pI.hasNext() ){
+ VertexProperty<Object> tp = pI.next();
+ Object val = tp.value();
+ logAndPrint(logger, "Prop: [" + tp.key() + "], val = [" + val + "] ");
+ }
+
+ Iterator <Edge> eI = tVert.edges(Direction.BOTH);
+ if( ! eI.hasNext() ){
+ logAndPrint(logger, "No edges were found for this vertex. ");
+ }
+ while( eI.hasNext() ){
+ Edge ed = eI.next();
+ String lab = ed.label();
+ Vertex vtx;
+ if (tVert.equals(ed.inVertex())) {
+ vtx = ed.outVertex();
+ } else {
+ vtx = ed.inVertex();
+ }
+ if( vtx == null ){
+ logAndPrint(logger, " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
+ }
+ else {
+ String nType = vtx.<String>property("aai-node-type").orElse(null);
+ String vid = vtx.id().toString();
+ logAndPrint(logger, "Found an edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid);
+ }
+ }
+ }
+ } // End of showPropertiesAndEdges()
+
+
+ /**
+ * Log and print.
+ *
+ * @param logger the logger
+ * @param msg the msg
+ */
+ protected static void logAndPrint(Logger logger, String msg) {
+ System.out.println(msg);
+ logger.info(msg);
+ }
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/web/JerseyConfiguration.java b/src/main/java/org/onap/aai/web/JerseyConfiguration.java
index 436946c..629d11e 100644
--- a/src/main/java/org/onap/aai/web/JerseyConfiguration.java
+++ b/src/main/java/org/onap/aai/web/JerseyConfiguration.java
@@ -19,25 +19,28 @@
*/
package org.onap.aai.web;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ContainerResponseFilter;
+
+import org.apache.tinkerpop.gremlin.structure.T;
import org.glassfish.jersey.filter.LoggingFilter;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletProperties;
import org.onap.aai.rest.QueryConsumer;
import org.onap.aai.rest.util.EchoResponse;
+import org.onap.logging.filter.base.AuditLogContainerFilter;
import org.reflections.Reflections;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
-import javax.annotation.Priority;
-import javax.ws.rs.container.ContainerRequestFilter;
-import javax.ws.rs.container.ContainerResponseFilter;
-import java.util.List;
-import java.util.Set;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
@Component
public class JerseyConfiguration extends ResourceConfig {
@@ -54,11 +57,11 @@ public class JerseyConfiguration extends ResourceConfig {
register(EchoResponse.class);
- //Request Filters
- registerFiltersForRequests();
- // Response Filters
- registerFiltersForResponses();
-
+ //Filters
+ registerFilters(ContainerRequestFilter.class);
+ registerFilters(ContainerResponseFilter.class);
+ registerFilters(AuditLogContainerFilter.class);
+
property(ServletProperties.FILTER_FORWARD_ON_404, true);
// Following registers the request headers and response headers
@@ -68,13 +71,14 @@ public class JerseyConfiguration extends ResourceConfig {
}
}
- public void registerFiltersForRequests() {
+ public <T> void registerFilters(Class<T> type) {
// Find all the classes within the interceptors package
- Reflections reflections = new Reflections("org.onap.aai.interceptors");
- // Filter them based on the clazz that was passed in
- Set<Class<? extends ContainerRequestFilter>> filters = reflections.getSubTypesOf(ContainerRequestFilter.class);
-
+ Reflections loggingReflections = new Reflections("org.onap.aai.aailog.filter");
+ Reflections reflections = new Reflections("org.onap.aai.interceptors");
+ // Filter them based on the clazz that was passed in
+ Set<Class<? extends T>> filters = loggingReflections.getSubTypesOf(type);
+ filters.addAll(reflections.getSubTypesOf(type));
// Check to ensure that each of the filter has the @Priority annotation and if not throw exception
for (Class filterClass : filters) {
@@ -84,7 +88,7 @@ public class JerseyConfiguration extends ResourceConfig {
}
// Turn the set back into a list
- List<Class<? extends ContainerRequestFilter>> filtersList = filters
+ List<Class<? extends T>> filtersList = filters
.stream()
.filter(f -> {
if (f.isAnnotationPresent(Profile.class)
@@ -101,37 +105,4 @@ public class JerseyConfiguration extends ResourceConfig {
// Then register this to the jersey application
filtersList.forEach(this::register);
}
-
- public void registerFiltersForResponses() {
-
- // Find all the classes within the interceptors package
- Reflections reflections = new Reflections("org.onap.aai.interceptors");
- // Filter them based on the clazz that was passed in
- Set<Class<? extends ContainerResponseFilter>> filters = reflections.getSubTypesOf(ContainerResponseFilter.class);
-
-
- // Check to ensure that each of the filter has the @Priority annotation and if not throw exception
- for (Class filterClass : filters) {
- if (filterClass.getAnnotation(Priority.class) == null) {
- throw new RuntimeException("Container filter " + filterClass.getName() + " does not have @Priority annotation");
- }
- }
-
- // Turn the set back into a list
- List<Class<? extends ContainerResponseFilter>> filtersList = filters.stream()
- .filter(f -> {
- if (f.isAnnotationPresent(Profile.class)
- && !env.acceptsProfiles(f.getAnnotation(Profile.class).value())) {
- return false;
- }
- return true;
- })
- .collect(Collectors.toList());
-
- // Sort them by their priority levels value
- filtersList.sort((c1, c2) -> Integer.valueOf(c1.getAnnotation(Priority.class).value()).compareTo(c2.getAnnotation(Priority.class).value()));
-
- // Then register this to the jersey application
- filtersList.forEach(this::register);
- }
}